Configure Systemd Watchdog

that monitors the reachability of Poseidon and automatically restarts Poseidon if required.

.github/workflows/ci.yml

@@ -208,6 +208,10 @@ jobs:
           ./poseidon | tee poseidon.log &
           until curl -s --fail http://localhost:7200/api/v1/health ; do sleep 1; done
           make e2e-test
+      - name: Write Environment Variables to file
+        run: |
+          echo "${{ vars }}"
+        if: ${{ success() || failure() }}
       - name: Setup Poseidon Socket
         run: |
           killall poseidon
@@ -216,13 +220,22 @@ jobs:
           cat ./.github/workflows/resources/poseidon-minimal.service | envsubst > ~/.config/systemd/user/poseidon.service
           systemctl --user daemon-reload
           systemctl --user start poseidon.socket
+        if: ${{ success() || failure() }}
       - name: Print Poseidon Failure logs
         if: failure()
-        run: journalctl -xen --no-pager
+        run: journalctl --user -xen --no-pager
       - name: Run e2e recovery tests
+        run: make e2e-test-recovery
+        if: ${{ success() || failure() }}
+      - name: Print Systemd Failure logs
         run: |
-          tail -f /var/log/syslog &
-          make e2e-test-recovery
+          /usr/bin/systemctl --user show poseidon.service -p NRestarts
+          journalctl --user -xe -u poseidon.service --no-pager
+        if: failure()
+      - name: Stop Poseidon to flush the coverage file
+        run: |
+          systemctl --user stop poseidon.service poseidon.socket
+          ls -lah ${GOCOVERDIR}
         if: ${{ success() || failure() }}
       - name: Convert coverage reports
         run: make convert-run-coverage

.github/workflows/resources/poseidon-minimal.service

@@ -8,5 +8,10 @@ Requires=poseidon.socket
 [Service]
 WorkingDirectory=${GITHUB_WORKSPACE}
 ExecStart=${GITHUB_WORKSPACE}/poseidon
-Restart=always
 Environment="POSEIDON_SERVER_SYSTEMDSOCKETACTIVATION=TRUE"
+
+Restart=always
+StartLimitBurst=0
+
+Type=notify
+WatchdogSec=5