htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove "Stage" variable

Browse Source 
as we will use distinct Influx and Grafana instances for the different stages/environments.
This commit is contained in:
Maximilian Paß
2023-01-17 17:10:03 +00:00
parent dc562ba7fd
commit 1aced0a130
25 changed files with 11 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/codeocean-terraform

Submodule deploy/codeocean-terraform updated: d9d3bdbf90...0b456c7f04

7
deploy/grafana-dashboard/panels/poseidon.dashboard.py
View File

@ -3,16 +3,13 @@ from grafanalib.core import Dashboard, Templating, Time
from panels.availability_row import availability_panels
from panels.general_row import general_panels
from panels.runner_insights_row import runner_insights_panels
from utils.variables import stage_variable, environment_variable
from utils.variables import environment_variable
dashboard = Dashboard(
title="Poseidon autogen",
timezone="browser",
panels=availability_panels + general_panels + runner_insights_panels,
templating=Templating(list=[
stage_variable,
environment_variable,
]),
templating=Templating(list=[ environment_variable ]),
editable=True,
refresh="30s",
time=Time("now-6h", "now"),

1
deploy/grafana-dashboard/queries/environment-ids.flux
View File

@ -1,7 +1,6 @@
from(bucket: "poseidon")
|> range(start: -1y)
|> filter(fn: (r) => r["_measurement"] == "poseidon_environments")
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["id"])
|> distinct(column: "id")
|> keep(columns: ["_value"])

2
deploy/grafana-dashboard/queries/environment-mapping.flux
View File

@ -6,7 +6,7 @@ envMapping = from(bucket: "poseidon")
|> last()
|> keep(columns: ["id", "image", "stage"])
|> rename(columns: {id: "environment_id"})
|> map(fn: (r) => ({ r with image: strings.substring(v: r.stage, start: 0, end: 1) + r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")}))
|> map(fn: (r) => ({ r with image: r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")}))
join(tables: {key1: result, key2: envMapping}, on: ["environment_id", "stage"], method: "inner")
|> keep(columns: ["_value", "image", "_time"])

1
deploy/grafana-dashboard/queries/execution-duration-hist.flux
View File

@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_field"] == "duration")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket")
|> filter(fn: (r) => exists r.environment_id)
|> keep(columns: ["_time", "_value", "environment_id", "stage"])

1
deploy/grafana-dashboard/queries/execution-duration.flux
View File

@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
|> filter(fn: (r) => r["_field"] == "duration")
|> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => exists r.environment_id)
|> keep(columns: ["_value", "runner_id", "environment_id", "stage"])
|> group(columns: ["environment_id", "stage"])

1
deploy/grafana-dashboard/queries/executions-per-minute-time.flux
View File

@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
|> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
|> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => r["event_type"] == "creation")
|> group(columns: ["environment_id", "stage"], mode:"by")
|> aggregateWindow(every: 1m, fn: count, createEmpty: true)

1
deploy/grafana-dashboard/queries/executions-per-minute.flux
View File

@ -5,7 +5,6 @@ result = from(bucket: "poseidon")
|> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
|> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => r["event_type"] == "creation")
|> group(columns: ["environment_id", "stage"], mode:"by")
|> aggregateWindow(every: 1m, fn: count, createEmpty: true)

1
deploy/grafana-dashboard/queries/executions-per-runner-hist.flux
View File

@ -2,7 +2,6 @@ import "strings"
data = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
runner_deletions = data
|> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")

1
deploy/grafana-dashboard/queries/executions-per-runner.flux
View File

@ -2,7 +2,6 @@ import "strings"
data = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
runner_deletions = data
|> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")

1
deploy/grafana-dashboard/queries/file-download-ratio.flux
View File

@ -5,7 +5,6 @@ data = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_measurement"] == "poseidon_file_download")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
actual = data |> filter(fn: (r) => r["_field"] == "actual_length")
expected = data |> filter(fn: (r) => r["_field"] == "expected_length")

1
deploy/grafana-dashboard/queries/file-download.flux
View File

@ -6,5 +6,4 @@ result = from(bucket: "poseidon")
|> filter(fn: (r) => r["_measurement"] == "poseidon_file_download")
|> filter(fn: (r) => r["_field"] == "actual_length")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_time", "_value", "environment_id", "stage"])

1
deploy/grafana-dashboard/queries/file-upload.flux
View File

@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_field"] == "request_size")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_time", "_value", "environment_id", "stage"])
|> aggregateWindow(every: myWindowPeriod, fn: mean, createEmpty: false)

1
deploy/grafana-dashboard/queries/idle-runner.flux
View File

@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners" and r["_field"] == "count")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_value", "_time", "environment_id", "stage"])
|> aggregateWindow(every: myWindowPeriod, fn: min, createEmpty: false)

1
deploy/grafana-dashboard/queries/number-of-executions.flux
View File

@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => r["event_type"] == "creation")
|> group(columns: ["environment_id", "stage"], mode:"by")
|> count()

1
deploy/grafana-dashboard/queries/prewarming-pool-size.flux
View File

@ -4,7 +4,6 @@ result = from(bucket: "poseidon")
|> range(start: -1y)
|> filter(fn: (r) => r["_measurement"] == "poseidon_poolsize")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> group(columns: ["environment_id", "stage"], mode:"by")
|> last()
|> keep(columns: ["_value", "environment_id", "stage"])

1
deploy/grafana-dashboard/queries/request-latency.flux
View File

@ -2,6 +2,5 @@ from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_field"] == "duration")
|> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_time", "_value"])
|> aggregateWindow(every: v.windowPeriod, fn: mean)

1
deploy/grafana-dashboard/queries/requests-per-minute.flux
View File

@ -4,7 +4,6 @@ data = from(bucket: "poseidon")
|> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
|> filter(fn: (r) => r._field == "duration")
|> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_time", "_value", "status"])
all = data |> set(key: "status", value: "all")

1
deploy/grafana-dashboard/queries/runner-per-minute.flux
View File

@ -6,7 +6,6 @@ result = from(bucket: "poseidon")
|> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m))
|> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> filter(fn: (r) => r["event_type"] == "creation")
|> group(columns: ["environment_id", "stage"], mode:"by")
|> aggregateWindow(every: 1m, fn: count, createEmpty: true)

1
deploy/grafana-dashboard/queries/runner-startup-duration.flux
View File

@ -5,6 +5,5 @@ result = from(bucket: "poseidon")
|> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners")
|> filter(fn: (r) => r["_field"] == "startup_duration")
|> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_value", "_time", "environment_id", "stage"])
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)

1
deploy/grafana-dashboard/queries/service-time.flux
View File

@ -2,6 +2,5 @@ from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_field"] == "duration")
|> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json}))
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> keep(columns: ["_time", "_value", "_measurement"])
|> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 10), fn: (tables=<-, column) => tables |> quantile(q: 0.999))

6
deploy/grafana-dashboard/queries/stages.flux
View File

@ -1,6 +0,0 @@
from(bucket: "poseidon")
|> range(start: -1y)
|> filter(fn: (r) => r["_measurement"] == "poseidon_environments")
|> keep(columns: ["stage"])
|> distinct(column: "stage")
|> keep(columns: ["_value"])

1
deploy/grafana-dashboard/queries/used-runner.flux
View File

@ -2,7 +2,6 @@ from(bucket: "poseidon")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners")
|> filter(fn: (r) => r["_field"] == "count")
|> filter(fn: (r) => (not exists r.stage) or contains(value: r["stage"], set: ${stages:json}))
|> group(columns: ["stage"], mode:"by")
|> keep(columns: ["_value", "_time", "stage"])
|> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 5), fn: mean, createEmpty: false)

9
deploy/grafana-dashboard/utils/color_mapping.py
View File

@ -31,6 +31,11 @@ colours = [
with open("environments.json") as f:
environments = json.load(f)
environments.sort()
for environment in environments:
environment_identifier = []
for environment in environments["executionEnvironments"]:
environment_identifier.append(str(environment["id"]) + "/" + environment["image"].removeprefix("openhpi/co_execenv_"))
environment_identifier.sort()
for environment in environment_identifier:
deep_update_dict(color_mapping_environments, color_mapping(environment, colours.pop(0)))

11
deploy/grafana-dashboard/utils/variables.py
View File

@ -2,17 +2,6 @@ from grafanalib.core import Template
from utils.utils import read_query
stage_variable = Template(
dataSource="Flux",
label="Stage",
name="stages",
query=read_query("stages"),
refresh=1,
includeAll=True,
multi=True,
default="production",
)
environment_variable = Template(
dataSource="Flux",
label="Environment IDs",