Do not crash due to failed backend components

lib/ha_handler/drbd/watcher.ex

@@ -28,8 +28,8 @@ defmodule HAHandler.DRBD.Watcher do
     hostname = Keyword.get(opts, :hostname)
     password = Keyword.get(opts, :password)
 
-    {:ok, pid} = connect(hostname, password)
-
+    case connect(hostname, password) do
+      {:ok, pid} ->
         state = %{
           backend: pid,
           hostname: hostname,
@@ -37,6 +37,14 @@ defmodule HAHandler.DRBD.Watcher do
         }
 
         {:ok, state}
+      {:error, err} ->
+
+        # Wait for 10 seconds so that the supervisor does not loop on dead node
+        # (and reach the max_restart threshold / stop trying).
+        Process.sleep(10_000)
+
+        {:error, err}
+    end
   end
 
   @impl true

lib/ha_handler/pgsql/watcher.ex

@@ -12,14 +12,19 @@ defmodule HAHandler.PGSQL.Watcher do
     # successful.
     # TODO: set dbconnections backoff and connect hooks
     # See https://github.com/elixir-ecto/db_connection/blob/master/lib/db_connection.ex#L343
-    {:ok, pid} = Postgrex.start_link(opts)
-
+    case Postgrex.start_link(opts) do
+      {:ok, pid} ->
         state = %{
           backend: pid,
           hostname: Keyword.get(opts, :hostname)
         }
 
         {:ok, state}
+      {:error, err} ->
+        # Will be catched by the supervisor if anything happen. It should not
+        # be triggered even if a PGSQL node down, since Postgrex has its own
+        # surpervision tree.
+        {:error, err}
   end
 
   @impl true