Release ha-handler v0.4.2

They were changed anyway.

.build.yml

@@ -1,19 +0,0 @@
-image: alpine/latest
-packages:
-- elixir
-artifacts:
-- ha-handler/_build/prod/rel/ha-handler.tar.gz
-sources:
-- https://code.recycled.cloud/RecycledCloud/ha-handler.git
-tasks:
-- setup: |
-    cd ha-handler
-    mix local.hex --force
-    mix local.rebar --force
-    mix deps.get
-- release: |
-    cd ha-handler
-    MIX_ENV=prod mix compile
-    MIX_ENV=prod mix release
-    cd _build/prod/rel
-    tar cvzf ha-handler.tar.gz ha_handler/

.drone.yml

@@ -0,0 +1,34 @@
+---
+kind: pipeline
+name: default
+
+steps:
+- name: build-release
+  image: alpine:3.15
+  environment:
+    MIX_ENV: prod
+  commands:
+    - apk add elixir git make gcc libc-dev
+    - mix local.hex --force
+    - mix local.rebar --force
+    - mix deps.get
+    - mix compile
+    - mix release
+    - cd _build/prod/rel
+    - tar czf "ha-handler-$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || git rev-parse HEAD).tar.gz" ha_handler/
+- name: publish-release-archive
+  image: alpine:3.15
+  environment:
+    LFTP_PASSWORD:
+     from_secret: ssh_password
+  commands:
+    - apk add git lftp openssh-client
+    - cd _build/prod/rel
+    - mkdir ~/.ssh
+    - ssh-keyscan static.recycled.cloud > ~/.ssh/known_hosts
+    - lftp "sftp://artifacts:$LFTP_PASSWORD@static.recycled.cloud" -e "cd htdocs; put ha-handler-$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || git rev-parse HEAD).tar.gz; bye"
+
+---
+kind: secret
+name: ssh_password
+data: aRTqi4c0MkO0arILOMZOFPeRg4HrCRkR52NcxwWRKI6ju69L

.editorconfig

@@ -0,0 +1,12 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*.ex]
+indent_style = space
+indent_size = 2
+
+[*.eex]
+indent_style = space
+indent_size = 2

changelog.md

@@ -1,4 +1,30 @@
-# 2022-02-?? - v0.2.0
+# 2022-07-04 - v0.4.2
+
+* Fix eventual crash on failed DRBD backend.
+* Appsignal: ignore errors on backends (failed PSQL backend currently generate
+  errors, and floods appsignal).
+
+# 2022-06-13 - v0.4.1
+
+* Fix crash on failed SSHEx / Postgrex connection failure.
+
+# 2022-06-09 - v0.4.0
+
+* Add minimal clustering logic.
+* Fix crash on unavailable HAProxy socket.
+* Fix invalid drbd backend state computation for haproxy.
+
+# 2022-04-20 - v0.3.0
+
+* Add Appsignal support.
+* drbd: initial plumbing.
+
+# 2022-02-23 - v0.2.1
+
+* haproxy: properly close file descriptor after socker query.
+* Add editorconfig, (mix) format codebase.
+
+# 2022-02-21 - v0.2.0
 
 * haproxy: fix invalid deduplication / merging of stat entries.
 * pgsql: refactoring, wire mode of operation (primary, secondary).

config/config.exs

@@ -1,7 +1,50 @@
 import Config
 
 config :ha_handler,
-	http_port: 4000,
+	http_port: 4040,
 	acme_challenge_path: "acme-challenge",
 	haproxy_socket: System.get_env("HAPROXY_SOCKET") || "/var/run/haproxy.sock",
-	pgsql_instances: []
+	handler_instances: [
+		:"ha_handler@fenschel",
+		:"ha_handler2@fenschel"
+	],
+	pgsql_instances: [
+		[
+			hostname: "pgsql.lnth.ch.recycled.cloud",
+			username: "postgres",
+			database: "postgres",
+			haproxy_server: "lnth",
+			password: "secret",
+			socket_options: [:inet6],
+			ssl: true
+		],
+		[
+			hostname: "pgsql.fvil.ch.recycled.cloud",
+			haproxy_server: "fvil",
+			username: "postgres",
+			database: "postgres",
+			password: "secret",
+			socket_options: [:inet6],
+			ssl: true
+		]
+	],
+	drbd_instances: [
+		[
+			hostname: "drbd.lnth.ch.recycled.cloud",
+			password: "secret",
+			haproxy_server: "lnth"
+		],
+		[
+			hostname: "drbd.fvil.ch.recycled.cloud",
+			password: "secret",
+			haproxy_server: "fvil"
+		]
+	]
+
+config :appsignal, :config,
+	active: false,
+	otp_app: :ha_handler,
+	name: "ha-handler",
+	push_api_key: "secret",
+	ignore_namespaces: ["pgsql", "drbd"],
+	env: config_env()

lib/ha_handler.ex

@@ -6,12 +6,14 @@ defmodule HAHandler do
   # Mix is not available in releases, and these things are static
   # anyway (@variables are evaluated at compile time).
   @otp_app Mix.Project.config()[:app]
-  @version Mix.Project.config[:version]
-  @env Mix.env
+  @version Mix.Project.config()[:version]
+  @env Mix.env()
 
   def http_port, do: Application.get_env(@otp_app, :http_port)
   def haproxy_socket, do: Application.get_env(@otp_app, :haproxy_socket)
   def pgsql_instances, do: Application.get_env(@otp_app, :pgsql_instances, [])
+  def drbd_instances, do: Application.get_env(@otp_app, :drbd_instances, [])
+  def handler_instances, do: Application.get_env(@otp_app, :handler_instances, [])
 
   def acme_challenge_path, do: Application.get_env(@otp_app, :acme_challenge_path)
   def static_path(), do: Application.app_dir(@otp_app, "priv/static/")
@@ -26,6 +28,7 @@ defmodule HAHandler do
       from: acme_challenge_path()
     ]
   end
+
   def assets_static_config() do
     [
       at: "/static",

lib/ha_handler/application.ex

@@ -10,8 +10,11 @@ defmodule HAHandler.Application do
   @impl true
   def start(_type, _args) do
     children = [
-      {Plug.Cowboy, scheme: :http, plug: HAHandler.Web.Router, options: [port: HAHandler.http_port()]},
+      {Plug.Cowboy,
+       scheme: :http, plug: HAHandler.Web.Router, options: [port: HAHandler.http_port()]},
       {HAHandler.PGSQL.Supervisor, HAHandler.pgsql_instances()},
+      {HAHandler.DRBD.Supervisor, HAHandler.drbd_instances()},
+      {HAHandler.Cluster, HAHandler.handler_instances()},
       {HAHandler.Control, []}
     ]
 

lib/ha_handler/cluster.ex

@@ -0,0 +1,108 @@
+defmodule HAHandler.Cluster do
+  use GenServer
+  require Logger
+
+  # How much do we wait (ms) between each check/decision-making round?
+  @refresh 30_000
+
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
+  end
+
+  @impl true
+  def init(instances) do
+    if Node.alive?() do
+      Logger.info("Distribution/clustering is ENABLED.")
+      Logger.info("Current handler instance is: #{Node.self()}")
+      Logger.info("Configured handler instances: #{inspect(instances)}")
+      :net_kernel.monitor_nodes(true)
+
+      send(self(), :sync)
+    else
+      Logger.warning("Distribution is DISABLED - skipping clustering logic")
+    end
+
+    {:ok, instances}
+  end
+
+  @impl true
+  def handle_info(:sync, instances) do
+    current_network = Node.list() ++ [Node.self()]
+
+    for node_name <- instances do
+      # Nothing to do if the node is already in our network/cluster.
+      if node_name not in current_network do
+        case Node.connect(node_name) do
+          true ->
+            Logger.info("Connected to handler instance #{node_name}")
+          _ ->
+            Logger.warning("Could not connect to handler instance #{node_name}")
+        end
+      end
+    end
+
+    Process.send_after(self(), :sync, @refresh)
+
+    {:noreply, instances}
+  end
+
+  @impl true
+  def handle_info({:nodedown, node}, instances) do
+    Logger.warning("Node #{node} went down.")
+
+    {:noreply, instances}
+  end
+
+  @impl true
+  def handle_info({:nodeup, node}, instances) do
+    Logger.info("Node #{node} came up.")
+
+    send(self(), :sync)
+
+    {:noreply, instances}
+  end
+
+  @impl true
+  def handle_call(:get_details, _from, instances) do
+    {uptime_ms, _} = :erlang.statistics(:wall_clock)
+
+    local_details = %{
+      node: Node.self(),
+      otp_app: HAHandler.otp_app,
+      version: HAHandler.version,
+      uptime: round(uptime_ms / 1_000 / 60),
+      env: HAHandler.env
+    }
+
+    {:reply, local_details, instances}
+  end
+
+  @impl true
+  def handle_call(:get_instances, _from, instances) do
+    {:reply, instances, instances}
+  end
+
+  def get_instance_details() do
+    known_instances = [Node.self()] ++ Node.list() ++ GenServer.call(__MODULE__, :get_instances)
+
+    known_instances
+    |> Enum.uniq()
+    |> Enum.map(fn node ->
+      try do
+        # FIXME: remote node coud return garbage/another structure!
+        GenServer.call({__MODULE__, node}, :get_details)
+        |> Map.put(:status, :up)
+      catch
+        :exit, _err ->
+          %{
+            node: node,
+            otp_app: :unknown,
+            version: :unknown,
+            uptime: :unknown,
+            env: :unknown,
+            status: :down
+          }
+      end
+    end)
+  end
+end

lib/ha_handler/control.ex

@@ -1,82 +1,189 @@
 defmodule HAHandler.Control do
-	@moduledoc """
-	This module handles the decision-logic and actions to be
-	taken regarding the current state of the infrastructure.
-	"""
+  @moduledoc """
+  This module handles the decision-logic and actions to be
+  taken regarding the current state of the infrastructure.
 
-	@haproxy_pgsql_backend "pgsql"
+  FIXME: POC quickly hacked together, there's a lot of weak code duplicated
+  around.
+  """
 
-	use GenServer
+  @haproxy_pgsql_backend "pgsql"
+  @haproxy_drbd_backend "sshfs"
 
-	require Logger
+  use GenServer
 
-	alias HAHandler.{PGSQL, HAProxy}
+  require Logger
 
-	# How much do we wait (ms) between each check/decision-making round?
-	@refresh 15_000
+  alias HAHandler.{PGSQL, HAProxy, DRBD}
 
-	def start_link(opts) do
-		GenServer.start_link(__MODULE__, opts, name: __MODULE__)
-	end
+  # How much do we wait (ms) between each check/decision-making round?
+  @refresh 15_000
 
-	@impl true
-	def init(_opts) do
-		state = []
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
+  end
 
-		# Let's skip the initial startup round so that other components are all up
-		# and running.
-		Process.send_after self(), :sync, @refresh
+  @impl true
+  def init(_opts) do
+    state = []
 
-		{:ok, state}
-	end
+    # Let's skip the initial startup round so that other components are all up
+    # and running.
+    Process.send_after(self(), :sync, @refresh)
 
-	@impl true
-	def handle_info(:sync, state) do
-		Logger.debug("Executing control logic.")
+    {:ok, state}
+  end
 
-		# Fetch PGSQL state, make sure HAProxy routes to the master
-		# process.
-		pgsql_state = PGSQL.get_instances()
-		|> Enum.map(fn {hostname, pid} = instance->
-			haproxy_server = HAHandler.pgsql_instances()
-										|> Enum.filter(fn opts -> Keyword.get(opts, :hostname) == hostname end)
-										|> Enum.at(0)
-										|> Keyword.get(:haproxy_server)
+  defp process_pgsql() do
+    # Fetch PGSQL state, make sure HAProxy routes to the master
+    # process.
+    pgsql_state =
+      PGSQL.get_instances()
+      |> Enum.map(fn {hostname, pid} = instance ->
+        haproxy_server =
+          HAHandler.pgsql_instances()
+          |> Enum.filter(fn opts -> Keyword.get(opts, :hostname) == hostname end)
+          |> Enum.at(0)
+          |> Keyword.get(:haproxy_server)
 
-			%{
-				haproxy_server: haproxy_server,
-				pgsql_watcher_pid: pid,
-				pgsql_operation_mode: PGSQL.get_operation_mode(instance)
-			}
-		end)
-		haproxy_state = HAProxy.get_stats()
-									|> Map.get("Server", [])
-									|> Enum.filter(fn mapping -> mapping["pxname"] == @haproxy_pgsql_backend end)
-									|> Enum.map(fn mapping -> %{mapping["svname"] => mapping["status"]} end)
-									|> Enum.reduce(&Map.merge/2)
+        %{
+          haproxy_server: haproxy_server,
+          pgsql_watcher_pid: pid,
+          pgsql_operation_mode: PGSQL.get_operation_mode(instance)
+        }
+      end)
 
-		for pgsql_instance <- pgsql_state do
-			haproxy_state = Map.get(haproxy_state, pgsql_instance.haproxy_server)
+    haproxy_state =
+      HAProxy.get_stats()
+      |> Map.get("Server", [])
+      |> Enum.filter(fn mapping -> mapping["pxname"] == @haproxy_pgsql_backend end)
+      |> Enum.map(fn mapping -> %{mapping["svname"] => mapping["status"]} end)
+      |> Enum.reduce(&Map.merge/2)
 
-			case {pgsql_instance.pgsql_operation_mode, haproxy_state} do
-				{:primary, "UP"} ->
-					:noop
-				{:primary, "MAINT"} ->
-					Logger.info("Enabling routing PGSQL to (now) primary #{pgsql_instance.haproxy_server}.")
-					HAProxy.set_server(@haproxy_pgsql_backend, pgsql_instance.haproxy_server, "state", "ready")
-				{:secondary, "UP"} ->
-					Logger.info("Disabling routing PGSQL to (now) secondary #{pgsql_instance.haproxy_server}.")
-					HAProxy.set_server(@haproxy_pgsql_backend, pgsql_instance.haproxy_server, "state", "maint")
-				{:secondary, "MAINT"} ->
-					:noop
-				unknown ->
-					Logger.warning("Unhandled PGSQL/HAProxy state: #{unknown}")
-			end
-		end
+    for pgsql_instance <- pgsql_state do
+      haproxy_state = Map.get(haproxy_state, pgsql_instance.haproxy_server)
 
-		# Schedule next round.
-		Process.send_after self(), :sync, @refresh
+      case {pgsql_instance.pgsql_operation_mode, haproxy_state} do
+        {:primary, "UP"} ->
+          :noop
 
-		{:noreply, state}
-	end
+        {:primary, "MAINT"} ->
+          Logger.info("Enabling routing PGSQL to (now) primary #{pgsql_instance.haproxy_server}.")
+
+          HAProxy.set_server(
+            @haproxy_pgsql_backend,
+            pgsql_instance.haproxy_server,
+            "state",
+            "ready"
+          )
+
+        {:secondary, "UP"} ->
+          Logger.info(
+            "Disabling routing PGSQL to (now) secondary #{pgsql_instance.haproxy_server}."
+          )
+
+          HAProxy.set_server(
+            @haproxy_pgsql_backend,
+            pgsql_instance.haproxy_server,
+            "state",
+            "maint"
+          )
+
+        {:secondary, "MAINT"} ->
+          :noop
+
+        unknown ->
+          Logger.warning("Unhandled PGSQL/HAProxy state: #{inspect(unknown)}")
+      end
+    end
+  end
+
+  defp process_drbd() do
+    drbd_state =
+      DRBD.get_instances()
+      |> Enum.map(fn {hostname, pid} = instance ->
+        haproxy_server =
+          HAHandler.drbd_instances()
+          |> Enum.filter(fn opts -> Keyword.get(opts, :hostname) == hostname end)
+          |> Enum.at(0)
+          |> Keyword.get(:haproxy_server)
+
+        %{
+          haproxy_server: haproxy_server,
+          drbd_watcher_pid: pid,
+          drbd_state: DRBD.get_state(instance)
+        }
+      end)
+
+    haproxy_state =
+      HAProxy.get_stats()
+      |> Map.get("Server", [])
+      |> Enum.filter(fn mapping -> mapping["pxname"] == @haproxy_drbd_backend end)
+      |> Enum.map(fn mapping -> %{mapping["svname"] => mapping["status"]} end)
+      |> Enum.reduce(&Map.merge/2)
+
+    for drbd_instance <- drbd_state do
+      haproxy_state = Map.get(haproxy_state, drbd_instance.haproxy_server)
+
+      case {drbd_instance.drbd_state.mode, haproxy_state} do
+        {"Secondary/Primary", "UP"} ->
+          Logger.info(
+            "Disabling routing SSHFS to (now) secondary #{drbd_instance.haproxy_server}."
+          )
+
+          HAProxy.set_server(
+            @haproxy_drbd_backend,
+            drbd_instance.haproxy_server,
+            "state",
+            "maint"
+          )
+        {"Primary/Secondary", "UP"} ->
+            :noop
+
+        {"Secondary/Primary", "MAINT"} ->
+            :noop
+
+        {"Primary/Secondary", "MAINT"} ->
+          Logger.info("Enabling routing SSHFS to (now) primary #{drbd_instance.haproxy_server}.")
+
+          HAProxy.set_server(
+            @haproxy_drbd_backend,
+            drbd_instance.haproxy_server,
+            "state",
+            "ready"
+          )
+
+        unknown ->
+          Logger.warning("Unknown DRBD/HAProxy state: #{inspect(unknown)}")
+          Logger.info(
+            "Disabling routing SSHFS to (likely) failed #{drbd_instance.haproxy_server}."
+          )
+
+          HAProxy.set_server(
+            @haproxy_drbd_backend,
+            drbd_instance.haproxy_server,
+            "state",
+            "maint"
+          )
+      end
+    end
+  end
+
+  @impl true
+  def handle_info(:sync, state) do
+    Logger.debug("Executing control logic.")
+
+    case HAProxy.get_stats() do
+      %{} ->
+        process_pgsql()
+        process_drbd()
+      {:error, err} ->
+        Logger.error("Unable to fetch HAProxy state (#{inspect(err)}) - skipping control loop.")
+    end
+
+    # Schedule next round.
+    Process.send_after(self(), :sync, @refresh)
+
+    {:noreply, state}
+  end
 end

lib/ha_handler/drbd.ex

@@ -0,0 +1,91 @@
+defmodule HAHandler.DRBD do
+  require Logger
+
+  @supervisor HAHandler.DRBD.Supervisor
+
+  # There might be >1 resources configured in DRBD!
+  @default_resource_id "1"
+
+  # We don't support DRBD 9 for the time being, as /proc/drbd does not have a
+  # stable API.
+  @supported_drbd_major_version "8"
+
+  # Parsing of /proc/drbd, assuming DRBD 8. Splitting the regexes helps humans
+  # wrapping their head around what's going on. And yes, it's fragile: we need
+  # drbd 9 to get a JSON interface to `drbdadm status`.
+  @drbd_proc_cmd "cat /proc/drbd"
+  @block_regex ~r/(?<version_block>(.|\n)*)\n(?<resource_block>\n\s(.|\n)*)/
+  @version_regex ~r/version: (?<full>(?<major>\d+)\.(?<intermediate>\d+)\.(?<minor>\d))/
+  @resource_split_regex ~r{(\n\s(\d+)\:\s)}
+  @id_extraction_regex ~r/\n\s(?<id>\d+)\:\s/
+  @data_extraction_regex ~r/cs:(?<cs>(\w|\/)+)\sro:(?<ro>(\w|\/)+)\sds:(?<ds>(\w|\/)+)\s/
+
+  # Empty state, when backend is not queryable for some reason.
+  @empty_state %{
+    hostname: "unknown",
+    version: "",
+    mode: "",
+    status: "unknown",
+    data: ""
+  }
+
+  def get_instances() do
+    watchers = Supervisor.which_children(@supervisor)
+
+    for {hostname, pid, _type, _modules} <- watchers do
+      {hostname, pid}
+    end
+  end
+
+  def get_stats() do
+    get_instances()
+    |> Enum.map(fn instance -> get_state(instance) end)
+  end
+
+  def get_state({hostname, pid}) do
+    empty_reply = %{@empty_state | hostname: hostname}
+
+    case GenServer.call(pid, {:execute, @drbd_proc_cmd}) do
+      {:ok, raw, 0} ->
+        case Regex.named_captures(@block_regex, raw) do
+          %{"version_block" => version_block, "resource_block" => resource_block} ->
+            version = Regex.named_captures(@version_regex, version_block)
+
+            if Map.get(version, "major") != @supported_drbd_major_version do
+              {:error, "unsupported DRBD version #{inspect(version)}"}
+            else
+              resources  = Regex.split( @resource_split_regex, resource_block,
+                [include_captures: true, trim: true])
+                |> Enum.chunk_every(2)
+                |> Enum.map(fn [raw_id, raw_data] ->
+                  %{}
+                  |> Map.merge(Regex.named_captures(@id_extraction_regex, raw_id))
+                  |> Map.merge(Regex.named_captures(@data_extraction_regex, raw_data))
+                end)
+
+              default_resource = resources
+                                 |> Enum.filter(fn r -> r["id"] == @default_resource_id end)
+                                 |> Enum.at(0)
+
+              processed_reply = %{
+                version: Map.get(version, "full"),
+                mode: Map.get(default_resource, "ro"),
+                status: Map.get(default_resource, "ds"),
+                data: resources
+              }
+              Map.merge(empty_reply, processed_reply)
+            end
+          _ ->
+            Logger.warning("Failed to query DRBD backend: could not parse /proc/drbd.")
+
+        end
+      {:ok, _, posix_err} ->
+        Logger.warning("Failed to query DRBD backend: POSIX #{inspect(posix_err)}.")
+        empty_reply
+
+      {:error, err} ->
+        Logger.warning("Failed to query DRBD backend: #{inspect(err)}.")
+        empty_reply
+    end
+  end
+end

lib/ha_handler/drbd/supervisor.ex

@@ -0,0 +1,25 @@
+defmodule HAHandler.DRBD.Supervisor do
+  use Supervisor
+
+  alias HAHandler.DRBD.Watcher, as: DRBDWatcher
+
+  def start_link(opts) do
+    Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
+  end
+
+  @impl true
+  def init(instances) do
+    children =
+      Enum.map(instances, fn conf ->
+        %{
+          id: Keyword.get(conf, :hostname),
+          start: {DRBDWatcher, :start_link, [conf]}
+        }
+      end)
+
+    opts = [
+      strategy: :one_for_one
+    ]
+    Supervisor.init(children, opts)
+  end
+end

lib/ha_handler/drbd/watcher.ex

@@ -0,0 +1,76 @@
+defmodule HAHandler.DRBD.Watcher do
+  # TODO: add support for SSH public keys authentication.
+
+  use GenServer
+
+  require Logger
+
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts)
+  end
+
+  defp connect(hostname, password) do
+    case :inet.gethostbyname(to_charlist(hostname), :inet6) do
+      {:ok, {:hostent, _name, _aliases, _addrtype, _length, [addr]}} ->
+        SSHEx.connect(
+          ip: addr,
+          user: 'root',
+          password: password,
+          silently_accept_hosts: true
+        )
+      err ->
+        err
+    end
+  end
+
+  @impl true
+  def init(opts) do
+    # Configures this worker's jobs to report in the "drbd" namespace
+    Appsignal.Span.set_namespace(Appsignal.Tracer.root_span(), "drbd")
+
+    state = %{
+      backend: nil,
+      last_reconnect: nil,
+      hostname: Keyword.get(opts, :hostname),
+      password: Keyword.get(opts, :password),
+    }
+
+    # This action will be processed once the GenServer is fully
+    # started/operational. This process handle connection failures by itself,
+    # as we don't want to crash loop into supervisor logic (which is only there
+    # to handle unexpected failure).
+    send self(), :reconnect
+
+    {:ok, state}
+  end
+
+  @impl true
+  def handle_info(:reconnect, state = %{hostname: hostname, password: password}) do
+    case connect(hostname, password) do
+      {:ok, pid} ->
+        {:noreply, %{state | backend: pid}}
+      {:error, _err} ->
+        # Nothing to do, as the next request will trigger the reconnect logic
+        # (see :execute call).
+
+        {:noreply, state}
+    end
+  end
+
+  @impl true
+  def handle_call({:execute, cmd}, _from, %{backend: backend} = state) do
+    case SSHEx.run(backend, cmd) do
+      {:ok, _output, _status} = reply->
+        {:reply, reply, state}
+      {:error, :closed} = reply ->
+        # Asynchroneously tries to reopen the connection to the backend.
+        send self(), :reconnect
+
+        {:reply, reply, state}
+      {:error, _err} = reply ->
+        # Do not take action on unknown error.
+        {:reply, reply, state}
+    end
+  end
+
+end

lib/ha_handler/ha_proxy.ex

@@ -26,31 +26,38 @@ defmodule HAHandler.HAProxy do
   """
   def execute(command) when is_binary(command) do
     case open_socket(haproxy_socket()) do
-      {:ok, socket} ->
-        send(socket, {self(), {:command, command <> "\n"}})
-        read_from_socket(socket)
+      {:ok, fd, port} ->
+        send(port, {self(), {:command, command <> "\n"}})
+        data = read_from_socket(port)
+        # We need to be careful and always close file descriptors, as they can
+        # run out - which will block all operations.
+        close_socket(fd)
+        data
+
       {:error, err} ->
         {:error, err}
     end
   end
 
   defp extract_stats(data) do
-    extracted = for entry <- data do
-      for mapping <- entry do
-        case mapping do
-          %{
-            "id" => id,
-            "proxyId" => proxy_id,
-            "objType" => type,
-            "field" => %{"name" => name},
-            "value" => %{"value" => value},
-          } ->
-            %{:id => id, :proxy_id => proxy_id, :type => type, :field => name, :value => value}
-          _ ->
-            nil
+    extracted =
+      for entry <- data do
+        for mapping <- entry do
+          case mapping do
+            %{
+              "id" => id,
+              "proxyId" => proxy_id,
+              "objType" => type,
+              "field" => %{"name" => name},
+              "value" => %{"value" => value}
+            } ->
+              %{:id => id, :proxy_id => proxy_id, :type => type, :field => name, :value => value}
+
+            _ ->
+              nil
+          end
         end
       end
-    end
 
     extracted
     |> List.flatten()
@@ -58,18 +65,14 @@ defmodule HAHandler.HAProxy do
       fn mapping -> {mapping.type, mapping.id, mapping.proxy_id} end,
       fn mapping -> %{mapping.field => mapping.value} end
     )
-    |> Enum.map(
-      fn {{type, id, proxy_id}, grouped_mappings} ->
-        grouped_mappings
-        |> Enum.reduce(fn l, r -> Map.merge(l,r) end)
-        |> Map.put("type", type)
-        |> Map.put("id", id)
-        |> Map.put("proxy_id", proxy_id)
-      end
-    )
-    |> Enum.group_by(
-      fn entry -> Map.get(entry, "type") end
-    )
+    |> Enum.map(fn {{type, id, proxy_id}, grouped_mappings} ->
+      grouped_mappings
+      |> Enum.reduce(fn l, r -> Map.merge(l, r) end)
+      |> Map.put("type", type)
+      |> Map.put("id", id)
+      |> Map.put("proxy_id", proxy_id)
+    end)
+    |> Enum.group_by(fn entry -> Map.get(entry, "type") end)
   end
 
   @doc """
@@ -77,7 +80,8 @@ defmodule HAHandler.HAProxy do
   a list of Maps.
   """
   def get_stats(opts \\ [])
-  def get_stats([hide_error: true]) do
+
+  def get_stats(hide_error: true) do
     case get_stats() do
       {:error, _err} ->
         %{
@@ -85,9 +89,12 @@ defmodule HAHandler.HAProxy do
           "Backend" => %{},
           "Server" => %{}
         }
-      stats -> stats
+
+      stats ->
+        stats
     end
   end
+
   def get_stats(_opts) do
     case execute("show stat json") do
       {:ok, raw} ->
@@ -113,8 +120,10 @@ defmodule HAHandler.HAProxy do
     case execute("set server #{backend}/#{server} #{key} #{value}") do
       {:ok, ""} ->
         :ok
+
       {:ok, err} ->
         {:error, err}
+
       {:error, err} ->
         {:error, err}
     end
@@ -158,13 +167,16 @@ defmodule HAHandler.HAProxy do
         {fdin, fdout} = {socket, socket}
         port = Port.open({:fd, fdin, fdout}, [{:line, 10_000}, :binary])
 
-        {:ok, port}
+        {:ok, socket, port}
 
       {:error, err} ->
         {:error, err}
     end
   end
 
+  # Close UNIX fd opened with `open_socket/1`.
+  defp close_socket(fd), do: Socket.close(fd)
+
   # Messages may be split due to the `{:line, L}` option specific in
   # `open_socket/1`.
   defp read_from_socket(socket, acc \\ "") do

lib/ha_handler/pgsql.ex

@@ -1,45 +1,52 @@
 defmodule HAHandler.PGSQL do
-	@supervisor HAHandler.PGSQL.Supervisor
-	@version_query "SELECT version();"
-	@is_in_recovery_query "SELECT pg_is_in_recovery();"
+  @supervisor HAHandler.PGSQL.Supervisor
+  @version_query "SELECT version();"
+  @is_in_recovery_query "SELECT pg_is_in_recovery();"
 
-	def get_instances() do
-		watchers = Supervisor.which_children(@supervisor)
-		for {hostname, pid, _type, _modules} <- watchers do
-			{hostname, pid}
-		end
-	end
+  def get_instances() do
+    watchers = Supervisor.which_children(@supervisor)
 
-	def get_version({hostname, pid}) do
-		case GenServer.call(pid, {:execute, @version_query, []}) do
-			{:ok, %Postgrex.Result{rows: [[raw_version_string]]}} ->
-				version = case Regex.run(~r/^PostgreSQL (\d+\.\d+)/, raw_version_string) do
-					[_, version_number] -> version_number
-					_ -> "unknown"
-				end
-				%{hostname: hostname, version: version, status: "up"}
-			{:error, %DBConnection.ConnectionError{message: _msg, reason: err}} ->
-				%{hostname: hostname, version: "unknown", status: err}
-			_ ->
-				%{hostname: hostname, version: "unknown", status: :unknown}
-		end
-	end
+    for {hostname, pid, _type, _modules} <- watchers do
+      {hostname, pid}
+    end
+  end
 
-	def get_operation_mode({_hostname, pid}) do
-		case GenServer.call(pid, {:execute, @is_in_recovery_query, []}) do
-			{:ok, %Postgrex.Result{rows: [[false]]}} ->
-				:primary
-			{:ok, %Postgrex.Result{rows: [[true]]}} ->
-				:secondary
-			_ ->
-				:unknown
-		end
-	end
+  def get_version({hostname, pid}) do
+    case GenServer.call(pid, {:execute, @version_query, []}) do
+      {:ok, %Postgrex.Result{rows: [[raw_version_string]]}} ->
+        version =
+          case Regex.run(~r/^PostgreSQL (\d+\.\d+)/, raw_version_string) do
+            [_, version_number] -> version_number
+            _ -> "unknown"
+          end
 
-	def get_stats() do
-		get_instances()
-		|> Enum.map(fn instance ->
-			get_version(instance) |> Map.put(:mode, get_operation_mode(instance))
-		end)
-	end
+        %{hostname: hostname, version: version, status: "up"}
+
+      {:error, %DBConnection.ConnectionError{message: _msg, reason: err}} ->
+        %{hostname: hostname, version: "unknown", status: err}
+
+      _ ->
+        %{hostname: hostname, version: "unknown", status: :unknown}
+    end
+  end
+
+  def get_operation_mode({_hostname, pid}) do
+    case GenServer.call(pid, {:execute, @is_in_recovery_query, []}) do
+      {:ok, %Postgrex.Result{rows: [[false]]}} ->
+        :primary
+
+      {:ok, %Postgrex.Result{rows: [[true]]}} ->
+        :secondary
+
+      _ ->
+        :unknown
+    end
+  end
+
+  def get_stats() do
+    get_instances()
+    |> Enum.map(fn instance ->
+      get_version(instance) |> Map.put(:mode, get_operation_mode(instance))
+    end)
+  end
 end

lib/ha_handler/pgsql/supervisor.ex

@@ -1,22 +1,23 @@
 defmodule HAHandler.PGSQL.Supervisor do
-	use Supervisor
+  use Supervisor
 
-	alias HAHandler.PGSQL.Watcher, as: PGSQLWatcher
+  alias HAHandler.PGSQL.Watcher, as: PGSQLWatcher
 
-	def start_link(opts) do
-		Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
-	end
+  def start_link(opts) do
+    Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
+  end
 
-	@impl true
-	def init(instances) do
-		children = Enum.map(instances, fn conf ->
-			%{
-				id: Keyword.get(conf, :hostname),
-				start: {PGSQLWatcher, :start_link, [conf]}
-			}
-		end)
+  @impl true
+  def init(instances) do
+    children =
+      Enum.map(instances, fn conf ->
+        %{
+          id: Keyword.get(conf, :hostname),
+          start: {PGSQLWatcher, :start_link, [conf]}
+        }
+      end)
 
-		opts = [ strategy: :one_for_one ]
-		Supervisor.init(children, opts)
-	end
+    opts = [strategy: :one_for_one]
+    Supervisor.init(children, opts)
+  end
 end

lib/ha_handler/pgsql/watcher.ex

@@ -1,29 +1,38 @@
 defmodule HAHandler.PGSQL.Watcher do
-	use GenServer
-	require Logger
+  use GenServer
+  require Logger
 
-	def start_link(opts) do
-		GenServer.start_link(__MODULE__, opts)
-	end
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts)
+  end
 
-	@impl true
-	def init(opts) do
-		# Starts a Postgrex child but does not means the connection was
-		# successful.
-		# TODO: set dbconnections backoff and connect hooks
-		# See https://github.com/elixir-ecto/db_connection/blob/master/lib/db_connection.ex#L343
-		{:ok, pid} = Postgrex.start_link(opts)
+  @impl true
+  def init(opts) do
+    # Configures this worker's jobs to report in the "pgsql" namespace
+    Appsignal.Span.set_namespace(Appsignal.Tracer.root_span(), "pgsql")
 
-		state = %{
-			backend: pid,
-			hostname: Keyword.get(opts, :hostname)
-		}
+    # Starts a Postgrex child but does not means the connection was
+    # successful.
+    # TODO: set dbconnections backoff and connect hooks
+    # See https://github.com/elixir-ecto/db_connection/blob/master/lib/db_connection.ex#L343
+    case Postgrex.start_link(opts) do
+      {:ok, pid} ->
+        state = %{
+          backend: pid,
+          hostname: Keyword.get(opts, :hostname)
+        }
 
-		{:ok, state}
-	end
+        {:ok, state}
+      {:error, err} ->
+        # Will be catched by the supervisor if anything happen. It should not
+        # be triggered even if a PGSQL node down, since Postgrex has its own
+        # surpervision tree.
+        {:error, err}
+    end
+  end
 
-	@impl true
-	def handle_call({:execute, query, params}, _from, %{backend: backend} = state) do
-		{:reply, Postgrex.query(backend, query, params), state}
-	end
+  @impl true
+  def handle_call({:execute, query, params}, _from, %{backend: backend} = state) do
+    {:reply, Postgrex.query(backend, query, params), state}
+  end
 end

lib/ha_handler/web/controller.ex

@@ -1,35 +1,38 @@
 defmodule HAHandler.Web.Controller do
-	import Plug.Conn
+  import Plug.Conn
 
-	alias HAHandler.{HAProxy, PGSQL}
+  alias HAHandler.{HAProxy, PGSQL, DRBD, Cluster}
 
   @template_dir "lib/ha_handler/web/templates"
-	@index_template EEx.compile_file(
-		Path.join(@template_dir, "index.html.eex")
-	)
+  @index_template EEx.compile_file(Path.join(@template_dir, "index.html.eex"))
 
-	defp render(conn, template, assigns) do
-		{body, _binding} = Code.eval_quoted(template, assigns)
+  defp render(conn, template, assigns) do
+    {body, _binding} = Code.eval_quoted(template, assigns)
 
-		conn
-		|> put_resp_content_type("text/html")
-		|> send_resp(200, body)
-	end
+    conn
+    |> put_resp_content_type("text/html")
+    |> send_resp(200, body)
+  end
 
-	def index(conn) do
-		{:ok, hostname} = :net_adm.dns_hostname(:net_adm.localhost)
+  def index(conn) do
+    {:ok, hostname} = :net_adm.dns_hostname(:net_adm.localhost())
 
-		haproxy_stats = HAProxy.get_stats([hide_error: true])
-		pgsql_stats = PGSQL.get_stats()
+    haproxy_stats = HAProxy.get_stats(hide_error: true)
+    pgsql_stats = PGSQL.get_stats()
+    drbd_stats = DRBD.get_stats()
+    handler_stats = Cluster.get_instance_details()
 
-		assigns = [
-			haproxy_stats: haproxy_stats,
-			pgsql_status: pgsql_stats,
-			hostname: hostname,
-			otp_app: HAHandler.otp_app(),
-			version: HAHandler.version(),
-			env: HAHandler.env()
-		]
-		render(conn, @index_template, assigns)
-	end
+    assigns = [
+      haproxy_stats: haproxy_stats,
+      pgsql_status: pgsql_stats,
+      drbd_status: drbd_stats,
+      handler_status: handler_stats,
+      hostname: hostname,
+      otp_app: HAHandler.otp_app(),
+      version: HAHandler.version(),
+      env: HAHandler.env()
+    ]
+
+    render(conn, @index_template, assigns)
+  end
 end

lib/ha_handler/web/router.ex

@@ -1,35 +1,38 @@
 defmodule HAHandler.Web.Router do
-	@moduledoc """
-	This module dispatch incoming HTTP requests to their
-	related logic. Please refer to [1] for details.
+  @moduledoc """
+  This module dispatch incoming HTTP requests to their
+  related logic. Please refer to [1] for details.
 
-	[1] https://hexdocs.pm/plug/Plug.Router.html#content
-	"""
+  [1] https://hexdocs.pm/plug/Plug.Router.html#content
+  """
 
-	use Plug.Router
+  use Plug.Router
 
-	alias HAHandler.Web.Controller
+  alias HAHandler.Web.Controller
 
-	# Note for plugs: oder is important, as a plug may stop
-	# want to stop the pipeline!
+  # Note for plugs: oder is important, as a plug may stop
+  # want to stop the pipeline!
 
-	plug Plug.Logger, log: :debug
+  plug(Plug.Logger, log: :debug)
 
-	# We use replug to allow for runtime configuration in release (as macros such
-	# as the `plug` call ae evaluated are compile-time).
-	plug Replug,
-		plug: Plug.Static,
-		opts: {HAHandler, :acme_challenges_static_config}
-	plug Replug,
-		plug: Plug.Static,
-		opts: {HAHandler, :assets_static_config}
+  # We use replug to allow for runtime configuration in release (as macros such
+  # as the `plug` call ae evaluated are compile-time).
+  plug(Replug,
+    plug: Plug.Static,
+    opts: {HAHandler, :acme_challenges_static_config}
+  )
 
-	plug :match
-	plug :dispatch
+  plug(Replug,
+    plug: Plug.Static,
+    opts: {HAHandler, :assets_static_config}
+  )
 
-	get "/", do: Controller.index(conn)
+  plug(:match)
+  plug(:dispatch)
 
-	match _ do
-		send_resp(conn, 404, "Not found")
-	end
+  get("/", do: Controller.index(conn))
+
+  match _ do
+    send_resp(conn, 404, "Not found")
+  end
 end

lib/ha_handler/web/templates/index.html.eex

@@ -1,124 +1,174 @@
 <!DOCTYPE html>
 <html>
-	<head>
-		<title>[HA] <%= hostname %></title>
-		<link rel="stylesheet" href="/static/app.css">
-	</head>
-	<body>
-		<main>
-			<div>
-				<img id="logo" src="/static/logo.svg" />
-			</div>
+  <head>
+    <title>[HA] <%= hostname %></title>
+    <link rel="stylesheet" href="/static/app.css">
+  </head>
+  <body>
+    <main>
+      <div>
+        <img id="logo" src="/static/logo.svg" />
+      </div>
 
-			<h1>Recycled Cloud HA handler</h1>
+      <h1>Recycled Cloud HA handler</h1>
 
-			<p>
-				This service supervises the various components of
-				the Recycled Cloud's High Availability
-				infrastruture. Documentation and source code can be
-				found on <a
-				href="https://code.recycled.cloud/RecycledCloud/ha-handler">our
-			software forge</a>.
-			</p>
+      <p>
+        This service supervises the various components of
+        the Recycled Cloud's High Availability
+        infrastruture. Documentation and source code can be
+        found on <a
+                   href="https://code.recycled.cloud/RecycledCloud/ha-handler">our
+                   software forge</a>.
+      </p>
 
-			<hr />
+      <hr />
 
-			<h2>Handler</h2>
+      <h2>Handler</h2>
 
-			<%= otp_app %> <b>v<%= version %></b> (<%= env %>) running on <b><%= hostname %></b>
+      <p>
+        <b>Local instance:</b> <%= otp_app %> <b>v<%= version %></b> (<%= env %>) running on <b><%= hostname %></b>
+      </p>
 
-			<hr />
+      <table>
+        <thead>
+          <tr>
+            <th>Instance</th>
+            <th>Version</th>
+            <th>Env</th>
+            <th>Status</th>
+            <th>Uptime</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for instance <- handler_status do %>
+            <tr>
+              <td><%= instance.node %></td>
+              <td><%= instance.version %></td>
+              <td><%= instance.env %></td>
+              <td><%= instance.status %></td>
+              <td><%= instance.uptime %>m</td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
 
-			<h2>HAProxy</h2>
+      <hr />
 
-			<h3>Frontends</h3>
-			<table>
-				<thead>
-					<tr>
-						<th>Name</th>
-						<th>Status</th>
-						<th>Bytes in</th>
-						<th>Bytes out</th>
-					</tr>
-				</thead>
-				<tbody>
-					<%= for entry <- Map.get(haproxy_stats, "Frontend") do %>
-						<tr>
-							<td><%= entry["pxname"] %></td>
-							<td><%= entry["status"] %></td>
-							<td><%= entry["bin"] %></td>
-							<td><%= entry["bout"] %></td>
-						</tr>
-					<% end %>
-				</tbody>
-			</table>
+      <h2>HAProxy</h2>
 
-			<h3>Backends</h3>
-			<table>
-				<thead>
-					<tr>
-						<th>Name</th>
-						<th>Status</th>
-						<th>algo</th>
-					</tr>
-				</thead>
-				<tbody>
-					<%= for entry <- Map.get(haproxy_stats, "Backend") do %>
-						<tr>
-							<td><%= entry["pxname"] %></td>
-							<td><%= entry["status"] %></td>
-							<td><%= entry["algo"] %></td>
-						</tr>
-					<% end %>
-				</tbody>
-			</table>
+      <h3>Frontends</h3>
+      <table>
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Status</th>
+            <th>Bytes in</th>
+            <th>Bytes out</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for entry <- Map.get(haproxy_stats, "Frontend") do %>
+            <tr>
+              <td><%= entry["pxname"] %></td>
+              <td><%= entry["status"] %></td>
+              <td><%= entry["bin"] %></td>
+              <td><%= entry["bout"] %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
 
-			<h3>Servers</h3>
-			<table>
-				<thead>
-					<tr>
-						<th>Name</th>
-						<th>Status</th>
-						<th>Mode</th>
-						<th>Address</th>
-					</tr>
-				</thead>
-				<tbody>
-					<%= for entry <- Map.get(haproxy_stats, "Server") do %>
-						<tr>
-							<td><%= entry["pxname"] %>/<%= entry["svname"] %></td>
-							<td><%= entry["status"] %></td>
-							<td><%= entry["mode"] %></td>
-							<td><%= entry["addr"] %></td>
-						</tr>
-					<% end %>
-				</tbody>
-			</table>
+      <h3>Backends</h3>
+      <table>
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Status</th>
+            <th>algo</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for entry <- Map.get(haproxy_stats, "Backend") do %>
+            <tr>
+              <td><%= entry["pxname"] %></td>
+              <td><%= entry["status"] %></td>
+              <td><%= entry["algo"] %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
 
-			<hr />
+      <h3>Servers</h3>
+      <table>
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Status</th>
+            <th>Mode</th>
+            <th>Address</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for entry <- Map.get(haproxy_stats, "Server") do %>
+            <tr>
+              <td><%= entry["pxname"] %>/<%= entry["svname"] %></td>
+              <td><%= entry["status"] %></td>
+              <td><%= entry["mode"] %></td>
+              <td><%= entry["addr"] %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
 
-			<h2>PostgreSQL</h2>
+      <hr />
 
-			<table>
-				<thead>
-					<tr>
-						<th>Hostname</th>
-						<th>Version</th>
-						<th>Status</th>
-						<th>Operation</th>
-					</tr>
-				</thead>
-				<tbody>
-					<%= for entry <- pgsql_status do %>
-						<tr>
-							<td><%= entry[:hostname] %></td>
-							<td><%= entry[:version] %></td>
-							<td><%= entry[:status] %></td>
-							<td><%= entry[:mode] %></td>
-						</tr>
-					<% end %>
-				</tbody>
-			</table>
-		</main>
-	</body>
+      <h2>PostgreSQL</h2>
+
+      <table>
+        <thead>
+          <tr>
+            <th>Hostname</th>
+            <th>Version</th>
+            <th>Status</th>
+            <th>Operation</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for entry <- pgsql_status do %>
+            <tr>
+              <td><%= entry[:hostname] %></td>
+              <td><%= entry[:version] %></td>
+              <td><%= entry[:status] %></td>
+              <td><%= entry[:mode] %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+
+      <hr />
+
+      <h2>DRBD</h2>
+
+      <table>
+        <thead>
+          <tr>
+            <th>Hostname</th>
+            <th>Version</th>
+            <th>Status</th>
+            <th>Operation</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for entry <- drbd_status do %>
+            <tr>
+              <td><%= entry[:hostname] %></td>
+              <td><%= entry[:version] %></td>
+              <td><%= entry[:status] %></td>
+              <td><%= entry[:mode] %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </main>
+  </body>
 </html>

mix.exs

@@ -4,7 +4,7 @@ defmodule HAHandler.MixProject do
   def project do
     [
       app: :ha_handler,
-      version: "0.2.0",
+      version: "0.4.2",
       elixir: "~> 1.12",
       start_permanent: Mix.env() == :prod,
       deps: deps(),
@@ -28,7 +28,9 @@ defmodule HAHandler.MixProject do
       {:replug, "~> 0.1.0"},
       {:procket, "~> 0.9"},
       {:poison, "~> 5.0"},
-      {:postgrex, "~> 0.16.1"}
+      {:postgrex, "~> 0.16.1"},
+      {:sshex, "~> 2.2.1"},
+      {:appsignal, "~> 2.0"}
     ]
   end
 

mix.lock

@@ -1,12 +1,20 @@
 %{
   "afunix": {:git, "https://github.com/tonyrog/afunix.git", "d7baab77d741d49bce08de2aeb28c5f192ab13d8", []},
+  "appsignal": {:hex, :appsignal, "2.2.10", "faf085bd5130a3f885620daab8baa660683ba2018eb43e9a6763a643e42c02c3", [:make, :mix], [{:decorator, "~> 1.2.3 or ~> 1.3", [hex: :decorator, repo: "hexpm", optional: false]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ca9818ac46efae3d1c61c5f119e8af83ea000903ca26979c0eeafeb9f5df5615"},
+  "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
   "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"},
   "db_connection": {:hex, :db_connection, "2.4.1", "6411f6e23f1a8b68a82fa3a36366d4881f21f47fc79a9efb8c615e62050219da", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea36d226ec5999781a9a8ad64e5d8c4454ecedc7a4d643e4832bf08efca01f00"},
   "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
+  "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"},
+  "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.2", "0b9e1a4c840eafb68d820b0e2158ef5c49385d17fb36855ac6e7e087d4b1dcc5", [:mix], [], "hexpm", "e6a3f76b4c277739e36c2e21a2c640778ba4c3846189d5ab19f97f126df5f9b7"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
   "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
@@ -15,6 +23,9 @@
   "procket": {:hex, :procket, "0.9.5", "ce91c00cb3f4d627fb3827f46ce68e6a969ea265df1afb349c300a07be4bb16c", [:rebar3], [], "hexpm", "1288bba5eb6f08ea5df3dda9e051bdac8f4b1e50adcca5cf4d726e825ad81bed"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "replug": {:hex, :replug, "0.1.0", "61d35f8c873c0078a23c49579a48f36e45789414b1ec0daee3fd5f4e34221f23", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f71f7a57e944e854fe4946060c6964098e53958074c69fb844b96e0bd58cfa60"},
+  "sshex": {:hex, :sshex, "2.2.1", "e1270b8345ea2a66a11c2bb7aed22c93e3bc7bc813486f4ffd0a980e4a898160", [:mix], [], "hexpm", "45b2caa5011dc850e70a2d77e3b62678a3e8bcb903eab6f3e7afb2ea897b13db"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
   "sweet_xml": {:hex, :sweet_xml, "0.7.2", "4729f997286811fabdd8288f8474e0840a76573051062f066c4b597e76f14f9f", [:mix], [], "hexpm", "6894e68a120f454534d99045ea3325f7740ea71260bc315f82e29731d570a6e8"},
   "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
 }

priv/static/app.css

@@ -1,5 +1,5 @@
 main {
-	width: 500px;
+	width: 700px;
 	margin-top: 50px;
 	margin-left: auto;
 	margin-right: auto;

run-ci.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-set -e
-
-BUILDS_INSTANCE=https://builds.sr.ht
-MANIFEST=.build.yml
-
-if [ -z "$SRHT_ACCESS_TOKEN" ]; then
-	echo "Please set SRHT_ACCESS_TOKEN before calling this script." >&2
-	exit 1
-fi
-
-curl \
-	-H Authorization:"token $SRHT_ACCESS_TOKEN" \
-	--data "manifest=$(jq -R -r --slurp < $MANIFEST)" \
-	--data "note=recycledcloud-ha-handler" \
-	-X POST $BUILDS_INSTANCE/api/jobs | jq