Do not crash due to failed backend components

lib/ha_handler/drbd/watcher.ex

@@ -28,15 +28,23 @@ defmodule HAHandler.DRBD.Watcher do
     hostname = Keyword.get(opts, :hostname)
     password = Keyword.get(opts, :password)
 
-    {:ok, pid} = connect(hostname, password)
+    case connect(hostname, password) do
+      {:ok, pid} ->
+        state = %{
+          backend: pid,
+          hostname: hostname,
+          password: password
+        }
 
-    state = %{
+        {:ok, state}
-      backend: pid,
+      {:error, err} ->
-      hostname: hostname,
-      password: password
-    }
 
-    {:ok, state}
+        # Wait for 10 seconds so that the supervisor does not loop on dead node
+        # (and reach the max_restart threshold / stop trying).
+        Process.sleep(10_000)
+
+        {:error, err}
+    end
   end
 
   @impl true

lib/ha_handler/pgsql/watcher.ex

@@ -12,14 +12,19 @@ defmodule HAHandler.PGSQL.Watcher do
     # successful.
     # TODO: set dbconnections backoff and connect hooks
     # See https://github.com/elixir-ecto/db_connection/blob/master/lib/db_connection.ex#L343
-    {:ok, pid} = Postgrex.start_link(opts)
+    case Postgrex.start_link(opts) do
+      {:ok, pid} ->
+        state = %{
+          backend: pid,
+          hostname: Keyword.get(opts, :hostname)
+        }
 
-    state = %{
+        {:ok, state}
-      backend: pid,
+      {:error, err} ->
-      hostname: Keyword.get(opts, :hostname)
+        # Will be catched by the supervisor if anything happen. It should not
-    }
+        # be triggered even if a PGSQL node down, since Postgrex has its own
-
+        # surpervision tree.
-    {:ok, state}
+        {:error, err}
   end
 
   @impl true