Make pipeline wait for successful Nomad deployment

.gitlab-ci.yml

@@ -85,7 +85,7 @@ dockerimage:
     - nomad validate nomad/api.nomad
     # nomad plan returns 1 if allocation is created or destroyed which is what we want here
     - nomad plan     nomad/api.nomad || [ $? == 1 ]
-    - nomad run      nomad/api.nomad
+    - nomad-run-and-wait nomad/api.nomad
   artifacts:
     paths:
       - nomad/api.nomad

nomad/Dockerfile

@@ -16,3 +16,6 @@ RUN wget "https://releases.hashicorp.com/nomad/1.0.4/nomad_1.0.4_linux_amd64.zip
 
 # Install Nomad
 RUN mv nomad /usr/sbin/ && nomad -version
+
+COPY nomad-run-and-wait /usr/sbin/
+RUN chmod +x /usr/sbin/nomad-run-and-wait

nomad/api.tpl.nomad

@@ -12,11 +12,16 @@ job "${NOMAD_SLUG}" {
       progress_deadline = "10m"
       auto_revert   = true
     }
+
+    // Don't allow rescheduling to fail deployment and pipeline if task fails
+    reschedule {
+      attempts = 0
+      unlimited = false
+    }
+
+    // No restarts to immediately fail the deployment and pipeline on first task fail
     restart {
-      attempts = 3
-      delay    = "15s"
-      interval = "30m"
-      mode     = "fail"
+      attempts = 0
     }
 
     network {

nomad/nomad-run-and-wait

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Script that runs a Nomad job and watches the deployment status until it finished running
+# See https://github.com/hashicorp/nomad/issues/6818
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: $0 <path/to/job.nomad>"
+  exit 1
+fi
+
+output=$(nomad run $1 2>&1)
+
+if [[ $? -ne 0 ]]; then
+  echo $output
+  exit 1
+fi
+
+deployment=$(grep -oP "(?<=Evaluation within deployment: \").*(?=\")" <<<$output)
+
+echo "Monitoring deployment $deployment"
+
+timeout=300
+sleepDuration=4
+iterations=$((timeout / sleepDuration))
+
+for i in $(seq $iterations); do
+  if [[ $i -eq $iterations ]]; then
+    # timeout reached, fail deployment and exit
+    nomad deployment fail $deployment
+    exit 1
+  fi
+  output=$(nomad deployment status $deployment 2>&1)
+  grep -E "Status" <<<$output
+  running=$(grep -E "Status.*=.*running" <<<$output)
+  if [[ -z "$running" ]]; then
+    break
+  fi
+  sleep $sleepDuration
+done
+
+echo "#######"
+echo "$output"
+failed=$(grep -E "Status.*=.*failed" <<<$output)
+if [[ -n "$failed" ]]; then
+  exit 1
+fi
+
+exit 0