diff --git a/deploy/codeocean-terraform b/deploy/codeocean-terraform index d9d3bdb..0b456c7 160000 --- a/deploy/codeocean-terraform +++ b/deploy/codeocean-terraform @@ -1 +1 @@ -Subproject commit d9d3bdbf907613e3fab00817883d0190e293dbe1 +Subproject commit 0b456c7f0446098675a457d31b69eec5e6c5e410 diff --git a/deploy/grafana-dashboard/panels/poseidon.dashboard.py b/deploy/grafana-dashboard/panels/poseidon.dashboard.py index 5a49677..687f182 100644 --- a/deploy/grafana-dashboard/panels/poseidon.dashboard.py +++ b/deploy/grafana-dashboard/panels/poseidon.dashboard.py @@ -3,16 +3,13 @@ from grafanalib.core import Dashboard, Templating, Time from panels.availability_row import availability_panels from panels.general_row import general_panels from panels.runner_insights_row import runner_insights_panels -from utils.variables import stage_variable, environment_variable +from utils.variables import environment_variable dashboard = Dashboard( title="Poseidon autogen", timezone="browser", panels=availability_panels + general_panels + runner_insights_panels, - templating=Templating(list=[ - stage_variable, - environment_variable, - ]), + templating=Templating(list=[ environment_variable ]), editable=True, refresh="30s", time=Time("now-6h", "now"), diff --git a/deploy/grafana-dashboard/queries/environment-ids.flux b/deploy/grafana-dashboard/queries/environment-ids.flux index da9a322..878b88b 100644 --- a/deploy/grafana-dashboard/queries/environment-ids.flux +++ b/deploy/grafana-dashboard/queries/environment-ids.flux @@ -1,7 +1,6 @@ from(bucket: "poseidon") |> range(start: -1y) |> filter(fn: (r) => r["_measurement"] == "poseidon_environments") - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["id"]) |> distinct(column: "id") |> keep(columns: ["_value"]) diff --git a/deploy/grafana-dashboard/queries/environment-mapping.flux b/deploy/grafana-dashboard/queries/environment-mapping.flux index d6b4971..953be73 100644 --- a/deploy/grafana-dashboard/queries/environment-mapping.flux +++ b/deploy/grafana-dashboard/queries/environment-mapping.flux @@ -6,7 +6,7 @@ envMapping = from(bucket: "poseidon") |> last() |> keep(columns: ["id", "image", "stage"]) |> rename(columns: {id: "environment_id"}) - |> map(fn: (r) => ({ r with image: strings.substring(v: r.stage, start: 0, end: 1) + r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")})) + |> map(fn: (r) => ({ r with image: r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")})) join(tables: {key1: result, key2: envMapping}, on: ["environment_id", "stage"], method: "inner") |> keep(columns: ["_value", "image", "_time"]) diff --git a/deploy/grafana-dashboard/queries/execution-duration-hist.flux b/deploy/grafana-dashboard/queries/execution-duration-hist.flux index 667c618..ac78b4b 100644 --- a/deploy/grafana-dashboard/queries/execution-duration-hist.flux +++ b/deploy/grafana-dashboard/queries/execution-duration-hist.flux @@ -4,7 +4,6 @@ result = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_field"] == "duration") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket") |> filter(fn: (r) => exists r.environment_id) |> keep(columns: ["_time", "_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/execution-duration.flux b/deploy/grafana-dashboard/queries/execution-duration.flux index f3811d8..786682b 100644 --- a/deploy/grafana-dashboard/queries/execution-duration.flux +++ b/deploy/grafana-dashboard/queries/execution-duration.flux @@ -5,7 +5,6 @@ result = from(bucket: "poseidon") |> filter(fn: (r) => r["_field"] == "duration") |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => exists r.environment_id) |> keep(columns: ["_value", "runner_id", "environment_id", "stage"]) |> group(columns: ["environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/executions-per-minute-time.flux b/deploy/grafana-dashboard/queries/executions-per-minute-time.flux index 9f03052..91aa92b 100644 --- a/deploy/grafana-dashboard/queries/executions-per-minute-time.flux +++ b/deploy/grafana-dashboard/queries/executions-per-minute-time.flux @@ -5,7 +5,6 @@ result = from(bucket: "poseidon") |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => r["event_type"] == "creation") |> group(columns: ["environment_id", "stage"], mode:"by") |> aggregateWindow(every: 1m, fn: count, createEmpty: true) diff --git a/deploy/grafana-dashboard/queries/executions-per-minute.flux b/deploy/grafana-dashboard/queries/executions-per-minute.flux index 797f548..3ce154e 100644 --- a/deploy/grafana-dashboard/queries/executions-per-minute.flux +++ b/deploy/grafana-dashboard/queries/executions-per-minute.flux @@ -5,7 +5,6 @@ result = from(bucket: "poseidon") |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => r["event_type"] == "creation") |> group(columns: ["environment_id", "stage"], mode:"by") |> aggregateWindow(every: 1m, fn: count, createEmpty: true) diff --git a/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux b/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux index 7b8b4aa..62dce1b 100644 --- a/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux +++ b/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux @@ -2,7 +2,6 @@ import "strings" data = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) runner_deletions = data |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") diff --git a/deploy/grafana-dashboard/queries/executions-per-runner.flux b/deploy/grafana-dashboard/queries/executions-per-runner.flux index 2060164..496908f 100644 --- a/deploy/grafana-dashboard/queries/executions-per-runner.flux +++ b/deploy/grafana-dashboard/queries/executions-per-runner.flux @@ -2,7 +2,6 @@ import "strings" data = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) runner_deletions = data |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") diff --git a/deploy/grafana-dashboard/queries/file-download-ratio.flux b/deploy/grafana-dashboard/queries/file-download-ratio.flux index e5c5262..e053f88 100644 --- a/deploy/grafana-dashboard/queries/file-download-ratio.flux +++ b/deploy/grafana-dashboard/queries/file-download-ratio.flux @@ -5,7 +5,6 @@ data = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) actual = data |> filter(fn: (r) => r["_field"] == "actual_length") expected = data |> filter(fn: (r) => r["_field"] == "expected_length") diff --git a/deploy/grafana-dashboard/queries/file-download.flux b/deploy/grafana-dashboard/queries/file-download.flux index 7ff5b4f..347e2cb 100644 --- a/deploy/grafana-dashboard/queries/file-download.flux +++ b/deploy/grafana-dashboard/queries/file-download.flux @@ -6,5 +6,4 @@ result = from(bucket: "poseidon") |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download") |> filter(fn: (r) => r["_field"] == "actual_length") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_time", "_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/file-upload.flux b/deploy/grafana-dashboard/queries/file-upload.flux index 47af4e3..983c2e5 100644 --- a/deploy/grafana-dashboard/queries/file-upload.flux +++ b/deploy/grafana-dashboard/queries/file-upload.flux @@ -5,6 +5,5 @@ result = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_field"] == "request_size") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_time", "_value", "environment_id", "stage"]) |> aggregateWindow(every: myWindowPeriod, fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/idle-runner.flux b/deploy/grafana-dashboard/queries/idle-runner.flux index faa2a68..fd559f1 100644 --- a/deploy/grafana-dashboard/queries/idle-runner.flux +++ b/deploy/grafana-dashboard/queries/idle-runner.flux @@ -5,6 +5,5 @@ result = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners" and r["_field"] == "count") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_value", "_time", "environment_id", "stage"]) |> aggregateWindow(every: myWindowPeriod, fn: min, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/number-of-executions.flux b/deploy/grafana-dashboard/queries/number-of-executions.flux index 094955a..f0babbc 100644 --- a/deploy/grafana-dashboard/queries/number-of-executions.flux +++ b/deploy/grafana-dashboard/queries/number-of-executions.flux @@ -4,7 +4,6 @@ result = from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => r["event_type"] == "creation") |> group(columns: ["environment_id", "stage"], mode:"by") |> count() diff --git a/deploy/grafana-dashboard/queries/prewarming-pool-size.flux b/deploy/grafana-dashboard/queries/prewarming-pool-size.flux index d36f906..37da6a7 100644 --- a/deploy/grafana-dashboard/queries/prewarming-pool-size.flux +++ b/deploy/grafana-dashboard/queries/prewarming-pool-size.flux @@ -4,7 +4,6 @@ result = from(bucket: "poseidon") |> range(start: -1y) |> filter(fn: (r) => r["_measurement"] == "poseidon_poolsize") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> group(columns: ["environment_id", "stage"], mode:"by") |> last() |> keep(columns: ["_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/request-latency.flux b/deploy/grafana-dashboard/queries/request-latency.flux index 7f24174..7c89f7d 100644 --- a/deploy/grafana-dashboard/queries/request-latency.flux +++ b/deploy/grafana-dashboard/queries/request-latency.flux @@ -2,6 +2,5 @@ from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_field"] == "duration") |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_time", "_value"]) |> aggregateWindow(every: v.windowPeriod, fn: mean) diff --git a/deploy/grafana-dashboard/queries/requests-per-minute.flux b/deploy/grafana-dashboard/queries/requests-per-minute.flux index cf9b412..9c2eff6 100644 --- a/deploy/grafana-dashboard/queries/requests-per-minute.flux +++ b/deploy/grafana-dashboard/queries/requests-per-minute.flux @@ -4,7 +4,6 @@ data = from(bucket: "poseidon") |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) |> filter(fn: (r) => r._field == "duration") |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_time", "_value", "status"]) all = data |> set(key: "status", value: "all") diff --git a/deploy/grafana-dashboard/queries/runner-per-minute.flux b/deploy/grafana-dashboard/queries/runner-per-minute.flux index 429d348..b6ba5ad 100644 --- a/deploy/grafana-dashboard/queries/runner-per-minute.flux +++ b/deploy/grafana-dashboard/queries/runner-per-minute.flux @@ -6,7 +6,6 @@ result = from(bucket: "poseidon") |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> filter(fn: (r) => r["event_type"] == "creation") |> group(columns: ["environment_id", "stage"], mode:"by") |> aggregateWindow(every: 1m, fn: count, createEmpty: true) diff --git a/deploy/grafana-dashboard/queries/runner-startup-duration.flux b/deploy/grafana-dashboard/queries/runner-startup-duration.flux index 990972f..f7c0b70 100644 --- a/deploy/grafana-dashboard/queries/runner-startup-duration.flux +++ b/deploy/grafana-dashboard/queries/runner-startup-duration.flux @@ -5,6 +5,5 @@ result = from(bucket: "poseidon") |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners") |> filter(fn: (r) => r["_field"] == "startup_duration") |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_value", "_time", "environment_id", "stage"]) |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/service-time.flux b/deploy/grafana-dashboard/queries/service-time.flux index e887e25..b53c50f 100644 --- a/deploy/grafana-dashboard/queries/service-time.flux +++ b/deploy/grafana-dashboard/queries/service-time.flux @@ -2,6 +2,5 @@ from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_field"] == "duration") |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> keep(columns: ["_time", "_value", "_measurement"]) |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 10), fn: (tables=<-, column) => tables |> quantile(q: 0.999)) diff --git a/deploy/grafana-dashboard/queries/stages.flux b/deploy/grafana-dashboard/queries/stages.flux deleted file mode 100644 index 9f2e1e1..0000000 --- a/deploy/grafana-dashboard/queries/stages.flux +++ /dev/null @@ -1,6 +0,0 @@ -from(bucket: "poseidon") - |> range(start: -1y) - |> filter(fn: (r) => r["_measurement"] == "poseidon_environments") - |> keep(columns: ["stage"]) - |> distinct(column: "stage") - |> keep(columns: ["_value"]) diff --git a/deploy/grafana-dashboard/queries/used-runner.flux b/deploy/grafana-dashboard/queries/used-runner.flux index b31dc2d..bf89139 100644 --- a/deploy/grafana-dashboard/queries/used-runner.flux +++ b/deploy/grafana-dashboard/queries/used-runner.flux @@ -2,7 +2,6 @@ from(bucket: "poseidon") |> range(start: v.timeRangeStart, stop: v.timeRangeStop) |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") |> filter(fn: (r) => r["_field"] == "count") - |> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json})) |> group(columns: ["stage"], mode:"by") |> keep(columns: ["_value", "_time", "stage"]) |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 5), fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/utils/color_mapping.py b/deploy/grafana-dashboard/utils/color_mapping.py index 9eff657..8e01da2 100644 --- a/deploy/grafana-dashboard/utils/color_mapping.py +++ b/deploy/grafana-dashboard/utils/color_mapping.py @@ -31,6 +31,11 @@ colours = [ with open("environments.json") as f: environments = json.load(f) - environments.sort() - for environment in environments: + + environment_identifier = [] + for environment in environments["executionEnvironments"]: + environment_identifier.append(str(environment["id"]) + "/" + environment["image"].removeprefix("openhpi/co_execenv_")) + + environment_identifier.sort() + for environment in environment_identifier: deep_update_dict(color_mapping_environments, color_mapping(environment, colours.pop(0))) diff --git a/deploy/grafana-dashboard/utils/variables.py b/deploy/grafana-dashboard/utils/variables.py index f5c2d6b..45f9c10 100644 --- a/deploy/grafana-dashboard/utils/variables.py +++ b/deploy/grafana-dashboard/utils/variables.py @@ -2,17 +2,6 @@ from grafanalib.core import Template from utils.utils import read_query -stage_variable = Template( - dataSource="Flux", - label="Stage", - name="stages", - query=read_query("stages"), - refresh=1, - includeAll=True, - multi=True, - default="production", -) - environment_variable = Template( dataSource="Flux", label="Environment IDs",