Remove "Stage" variable

as we will use distinct Influx and Grafana instances for the different stages/environments.

deploy/grafana-dashboard/panels/poseidon.dashboard.py

@@ -3,16 +3,13 @@ from grafanalib.core import Dashboard, Templating, Time
 from panels.availability_row import availability_panels
 from panels.general_row import general_panels
 from panels.runner_insights_row import runner_insights_panels
-from utils.variables import stage_variable, environment_variable
+from utils.variables import environment_variable
 
 dashboard = Dashboard(
     title="Poseidon autogen",
     timezone="browser",
     panels=availability_panels + general_panels + runner_insights_panels,
-    templating=Templating(list=[
+    templating=Templating(list=[ environment_variable ]),
-        stage_variable,
-        environment_variable,
-    ]),
     editable=True,
     refresh="30s",
     time=Time("now-6h", "now"),

deploy/grafana-dashboard/queries/environment-ids.flux

@@ -1,7 +1,6 @@
 from(bucket: "poseidon")
   |> range(start: -1y)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_environments")
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["id"])
   |> distinct(column: "id")
   |> keep(columns: ["_value"])

deploy/grafana-dashboard/queries/environment-mapping.flux

@@ -6,7 +6,7 @@ envMapping = from(bucket: "poseidon")
   |> last()
   |> keep(columns: ["id", "image", "stage"])
   |> rename(columns: {id: "environment_id"})
-  |> map(fn: (r) => ({ r with image: strings.substring(v: r.stage, start: 0, end: 1) + r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")}))
+  |> map(fn: (r) => ({ r with image: r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")}))
 
 join(tables: {key1: result, key2: envMapping}, on: ["environment_id", "stage"], method: "inner")
   |> keep(columns: ["_value", "image", "_time"])

deploy/grafana-dashboard/queries/execution-duration-hist.flux

@@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_field"] == "duration")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket")
   |> filter(fn: (r) => exists r.environment_id)
   |> keep(columns: ["_time", "_value", "environment_id", "stage"])

deploy/grafana-dashboard/queries/execution-duration.flux

@@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
   |> filter(fn: (r) => r["_field"] == "duration")
   |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => exists r.environment_id)
   |> keep(columns: ["_value", "runner_id", "environment_id", "stage"])
   |> group(columns: ["environment_id", "stage"])

deploy/grafana-dashboard/queries/executions-per-minute-time.flux

@@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
   |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
   |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => r["event_type"] == "creation")
   |> group(columns: ["environment_id", "stage"], mode:"by")
   |> aggregateWindow(every: 1m, fn: count, createEmpty: true)

deploy/grafana-dashboard/queries/executions-per-minute.flux

@@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
   |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
   |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => r["event_type"] == "creation")
   |> group(columns: ["environment_id", "stage"], mode:"by")
   |> aggregateWindow(every: 1m, fn: count, createEmpty: true)

deploy/grafana-dashboard/queries/executions-per-runner-hist.flux

@@ -2,7 +2,6 @@ import "strings"
 
 data = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
 
 runner_deletions = data
   |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")

deploy/grafana-dashboard/queries/executions-per-runner.flux

@@ -2,7 +2,6 @@ import "strings"
 
 data = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
 
 runner_deletions = data
   |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")

deploy/grafana-dashboard/queries/file-download-ratio.flux

@@ -5,7 +5,6 @@ data = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
 
 actual = data |> filter(fn: (r) => r["_field"] == "actual_length")
 expected = data |> filter(fn: (r) => r["_field"] == "expected_length")

deploy/grafana-dashboard/queries/file-download.flux

@@ -6,5 +6,4 @@ result = from(bucket: "poseidon")
   |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download")
   |> filter(fn: (r) => r["_field"] == "actual_length")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_time", "_value", "environment_id", "stage"])

deploy/grafana-dashboard/queries/file-upload.flux

@@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_field"] == "request_size")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_time", "_value", "environment_id", "stage"])
   |> aggregateWindow(every: myWindowPeriod, fn: mean, createEmpty: false)

deploy/grafana-dashboard/queries/idle-runner.flux

@@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners" and r["_field"] == "count")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_value", "_time", "environment_id", "stage"])
   |> aggregateWindow(every: myWindowPeriod, fn: min, createEmpty: false)

deploy/grafana-dashboard/queries/number-of-executions.flux

@@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => r["event_type"] == "creation")
   |> group(columns: ["environment_id", "stage"], mode:"by")
   |> count()

deploy/grafana-dashboard/queries/prewarming-pool-size.flux

@@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
   |> range(start: -1y)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_poolsize")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> group(columns: ["environment_id", "stage"], mode:"by")
   |> last()
   |> keep(columns: ["_value", "environment_id", "stage"])

deploy/grafana-dashboard/queries/request-latency.flux

@@ -2,6 +2,5 @@ from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_field"] == "duration")
   |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_time", "_value"])
   |> aggregateWindow(every: v.windowPeriod, fn: mean)

deploy/grafana-dashboard/queries/requests-per-minute.flux

@@ -4,7 +4,6 @@ data = from(bucket: "poseidon")
   |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
   |> filter(fn: (r) => r._field == "duration")
   |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_time", "_value", "status"])
 
 all = data |> set(key: "status", value: "all")

deploy/grafana-dashboard/queries/runner-per-minute.flux

@@ -6,7 +6,6 @@ result = from(bucket: "poseidon")
   |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
   |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> filter(fn: (r) => r["event_type"] == "creation")
   |> group(columns: ["environment_id", "stage"], mode:"by")
   |> aggregateWindow(every: 1m, fn: count, createEmpty: true)

deploy/grafana-dashboard/queries/runner-startup-duration.flux

@@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
   |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners")
   |> filter(fn: (r) => r["_field"] == "startup_duration")
   |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_value", "_time", "environment_id", "stage"])
   |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)

deploy/grafana-dashboard/queries/service-time.flux

@@ -2,6 +2,5 @@ from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_field"] == "duration")
   |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> keep(columns: ["_time", "_value", "_measurement"])
   |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 10), fn: (tables=<-, column) => tables |> quantile(q: 0.999))

deploy/grafana-dashboard/queries/stages.flux

@@ -1,6 +0,0 @@
-from(bucket: "poseidon")
-  |> range(start: -1y)
-  |> filter(fn: (r) => r["_measurement"] == "poseidon_environments")
-  |> keep(columns: ["stage"])
-  |> distinct(column: "stage")
-  |> keep(columns: ["_value"])

deploy/grafana-dashboard/queries/used-runner.flux

@@ -2,7 +2,6 @@ from(bucket: "poseidon")
   |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
   |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")
   |> filter(fn: (r) => r["_field"] == "count")
-  |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
   |> group(columns: ["stage"], mode:"by")
   |> keep(columns: ["_value", "_time", "stage"])
   |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 5), fn: mean, createEmpty: false)

deploy/grafana-dashboard/utils/color_mapping.py

@@ -31,6 +31,11 @@ colours = [
 
 with open("environments.json") as f:
     environments = json.load(f)
-    environments.sort()
+
-    for environment in environments:
+    environment_identifier = []
+    for environment in environments["executionEnvironments"]:
+        environment_identifier.append(str(environment["id"]) + "/" + environment["image"].removeprefix("openhpi/co_execenv_"))
+
+    environment_identifier.sort()
+    for environment in environment_identifier:
         deep_update_dict(color_mapping_environments, color_mapping(environment, colours.pop(0)))

deploy/grafana-dashboard/utils/variables.py

@@ -2,17 +2,6 @@ from grafanalib.core import Template
 
 from utils.utils import read_query
 
-stage_variable = Template(
-    dataSource="Flux",
-    label="Stage",
-    name="stages",
-    query=read_query("stages"),
-    refresh=1,
-    includeAll=True,
-    multi=True,
-    default="production",
-)
-
 environment_variable = Template(
     dataSource="Flux",
     label="Environment IDs",