diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 658e7d1..0000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: "2" - -checks: - similar-code: - config: - threshold: 110 # Golang default: 100 - identical-code: - config: - threshold: 110 # Golang default: 100 - -plugins: - golint: - enabled: false - govet: - enabled: false - gofmt: - enabled: false - -exclude_patterns: - - "**/*_mock.go" - - "**/*_test.go" diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d1f9fe3..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "deploy/codeocean-terraform"] - path = deploy/codeocean-terraform - url = git@lab.xikolo.de:codeocean/codeocean-terraform.git diff --git a/Dockerfile b/Dockerfile index 342b1b9..d4b0326 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,11 +10,11 @@ RUN apt-get update #apt-get install -y git && \ #git clone https://github.com/openHPI/poseidon.git . -COPY . . - # Install make (required for building) RUN apt-get install -y make +COPY . . + # Install required project libraries RUN make bootstrap diff --git a/Makefile b/Makefile index a6c6f7f..4484be7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PROJECT_NAME = poseidon +ROJECT_NAME = poseidon REPOSITORY_OWNER = openHPI PKG = github.com/$(REPOSITORY_OWNER)/$(PROJECT_NAME)/cmd/$(PROJECT_NAME) UNIT_TESTS = $(shell go list ./... | grep -v /e2e | grep -v /recovery) @@ -43,10 +43,13 @@ tidy-deps: ## Remove unused dependencies .PHONY: git-hooks -git-hooks: .git/hooks/pre-commit ## Install the git-hooks -.git/hooks/%: git_hooks/% - cp $^ $@ - chmod 755 $@ +GIT_HOOKS_DIR := $(shell if [ -f .git ]; then echo $$(cat .git | sed 's/gitdir: //')/hooks; else echo .git/hooks; fi) +git-hooks: $(GIT_HOOKS_DIR)/pre-commit ## Install the git-hooks +$(GIT_HOOKS_DIR)/%: git_hooks/% + @if [ -d "$(GIT_HOOKS_DIR)" ]; then \ + cp $^ $@; \ + chmod 755 $@; \ + fi .PHONY: build build: deps ## Build the binary diff --git a/cmd/poseidon/main.go b/cmd/poseidon/main.go index 46bfb29..4ca2e4f 100644 --- a/cmd/poseidon/main.go +++ b/cmd/poseidon/main.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "errors" + "flag" "fmt" "github.com/coreos/go-systemd/v22/activation" "github.com/coreos/go-systemd/v22/daemon" @@ -14,12 +15,17 @@ import ( "github.com/openHPI/poseidon/internal/api" "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/environment" + kubernetes2 "github.com/openHPI/poseidon/internal/kubernetes" "github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/pkg/logging" "github.com/openHPI/poseidon/pkg/monitoring" - "golang.org/x/sys/unix" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "path/filepath" + // Import the unix package for Unix signals like SIGUSR1 "net" "net/http" "os" @@ -31,6 +37,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" ) @@ -343,13 +350,48 @@ func createNomadManager(ctx context.Context) (runner.Manager, environment.Manage environmentManager, err := environment. NewNomadEnvironmentManager(runnerManager, nomadAPIClient, config.Config.Server.TemplateJobFile) if err != nil { - log.WithError(err).Fatal("Error initializing environment manager") + log.WithError(err).Fatal("Error initializing nomad environment manager") } synchronizeNomad(ctx, environmentManager, runnerManager) return runnerManager, environmentManager } +func createKubernetesManager(ctx context.Context) (runner.Manager, environment.ManagerHandler) { + kubernetesConfig := k8sConfig() + + executorAPI, err := kubernetes2.NewExecutorAPI(kubernetesConfig) + if err != nil { + log.WithError(err).Fatal("Error creating Kubernetes API client configuration") + } + + runnerManager := runner.NewKubernetesRunnerManager(&executorAPI, ctx) + environmentManager, err := environment.NewKubernetesEnvironmentManager(runnerManager, &executorAPI, config.Config.Server.TemplateJobFile) + if err != nil { + log.WithError(err).Fatal("Error initializing kubernetes environment manager") + } + return runnerManager, environmentManager + +} + +func k8sConfig() *rest.Config { + var kubeconfig *string + if home := homedir.HomeDir(); home != "" { + kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") + } + flag.Parse() + + // use the current context in kubeconfig + configFromFlags, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + if err != nil { + panic(err.Error()) + } + + return configFromFlags +} + // synchronizeNomad starts the asynchronous synchronization background task and waits for the first environment and runner recovery. func synchronizeNomad(ctx context.Context, environmentManager *environment.NomadEnvironmentManager, runnerManager *runner.NomadRunnerManager) { firstRecoveryDone := make(chan struct{}) @@ -382,18 +424,13 @@ func synchronizeNomad(ctx context.Context, environmentManager *environment.Nomad } } -func createAWSManager(ctx context.Context) ( - runnerManager runner.Manager, environmentManager environment.ManagerHandler) { - runnerManager = runner.NewAWSRunnerManager(ctx) - return runnerManager, environment.NewAWSEnvironmentManager(runnerManager) -} - // initRouter builds a router that serves the API with the chain of responsibility for multiple managers. func initRouter(ctx context.Context) *mux.Router { runnerManager, environmentManager := createManagerHandler(createNomadManager, config.Config.Nomad.Enabled, nil, nil, ctx) - runnerManager, environmentManager = createManagerHandler(createAWSManager, config.Config.AWS.Enabled, - runnerManager, environmentManager, ctx) + + runnerManager, environmentManager = createManagerHandler(createKubernetesManager, config.Config.Kubernetes.Enabled, + nil, nil, ctx) return api.NewRouter(runnerManager, environmentManager) } @@ -421,11 +458,11 @@ func initServer(router *mux.Router) *http.Server { func shutdownOnOSSignal(server *http.Server, ctx context.Context, stopProfiling func()) { // wait for SIGINT shutdownSignals := make(chan os.Signal, 1) - signal.Notify(shutdownSignals, unix.SIGINT, unix.SIGTERM, unix.SIGABRT) + signal.Notify(shutdownSignals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGABRT) // wait for SIGUSR1 writeProfileSignal := make(chan os.Signal, 1) - signal.Notify(writeProfileSignal, unix.SIGUSR1) + signal.Notify(writeProfileSignal, syscall.Signal(0xa)) // syscall.SIGUSR1 select { case <-ctx.Done(): diff --git a/cmd/poseidon/main_test.go b/cmd/poseidon/main_test.go deleted file mode 100644 index 60e1c0d..0000000 --- a/cmd/poseidon/main_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "context" - "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/environment" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "golang.org/x/sys/unix" - "testing" - "time" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestAWSDisabledUsesNomadManager() { - disableRecovery, cancel := context.WithCancel(context.Background()) - cancel() - - runnerManager, environmentManager := createManagerHandler(createNomadManager, true, - runner.NewAbstractManager(s.TestCtx), &environment.AbstractManager{}, disableRecovery) - awsRunnerManager, awsEnvironmentManager := createManagerHandler(createAWSManager, false, - runnerManager, environmentManager, s.TestCtx) - s.Equal(runnerManager, awsRunnerManager) - s.Equal(environmentManager, awsEnvironmentManager) -} - -func (s *MainTestSuite) TestAWSEnabledWrappesNomadManager() { - disableRecovery, cancel := context.WithCancel(context.Background()) - cancel() - - runnerManager, environmentManager := createManagerHandler(createNomadManager, true, - runner.NewAbstractManager(s.TestCtx), &environment.AbstractManager{}, disableRecovery) - awsRunnerManager, awsEnvironmentManager := createManagerHandler(createAWSManager, - true, runnerManager, environmentManager, s.TestCtx) - s.NotEqual(runnerManager, awsRunnerManager) - s.NotEqual(environmentManager, awsEnvironmentManager) -} - -func (s *MainTestSuite) TestShutdownOnOSSignal_Profiling() { - called := false - disableRecovery, cancel := context.WithCancel(context.Background()) - cancel() - - s.ExpectedGoroutineIncrease++ // The shutdownOnOSSignal waits for an exit after stopping the profiling. - s.ExpectedGoroutineIncrease++ // The shutdownOnOSSignal triggers a os.Signal Goroutine. - - server := initServer(initRouter(disableRecovery)) - go shutdownOnOSSignal(server, context.Background(), func() { - called = true - }) - - <-time.After(tests.ShortTimeout) - err := unix.Kill(unix.Getpid(), unix.SIGUSR1) - s.Require().NoError(err) - <-time.After(tests.ShortTimeout) - - s.True(called) -} - -func (s *MainTestSuite) TestLoadNomadEnvironmentsBeforeStartingWebserver() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadEnvironmentJobs").Return([]*api.Job{}, nil) - apiMock.On("WatchEventStream", mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - <-s.TestCtx.Done() - }).Return(nil).Maybe() - - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - environmentManager, err := environment.NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - synchronizeNomad(s.TestCtx, environmentManager, runnerManager) - apiMock.AssertExpectations(s.T()) -} diff --git a/deploy/api.tpl.nomad b/deploy/api.tpl.nomad deleted file mode 100644 index fb4843c..0000000 --- a/deploy/api.tpl.nomad +++ /dev/null @@ -1,90 +0,0 @@ -job "${NOMAD_SLUG}" { - datacenters = ["dc1"] - namespace = "${NOMAD_NAMESPACE}" - - group "api" { - count = 1 - - update { - // https://learn.hashicorp.com/tutorials/nomad/job-rolling-update - max_parallel = 1 - min_healthy_time = "30s" - healthy_deadline = "5m" - progress_deadline = "10m" - auto_revert = true - } - - // Don't allow rescheduling to fail deployment and pipeline if task fails - reschedule { - attempts = 0 - unlimited = false - } - - // No restarts to immediately fail the deployment and pipeline on first task fail - restart { - attempts = 0 - } - - network { - mode = "cni/secure-bridge" - - port "http" { - to = 7200 - } - } - - service { - # urlprefix- tag allows Fabio to discover this service and proxy traffic correctly - tags = ["urlprefix-${HOSTNAME}:80/"] - name = "${NOMAD_SLUG}" - port = "http" - - // Health check to let Consul know we are alive - check { - name = "health-check" - type = "http" - port = "http" - path = "/api/v1/health" - interval = "10s" - timeout = "2s" - check_restart { - limit = 3 // auto-restart task when health check fails 3x in a row - } - - } - } - - task "api" { - driver = "docker" - - config { - image = "${IMAGE_NAME_ENV}" - } - - template { - data = < - 4.0.0 - poseidon - java11Exec - 1.0 - jar - A Java executor created for openHPI/Poseidon. - - 11 - 11 - UTF-8 - - - - - com.amazonaws - aws-lambda-java-core - 1.2.3 - - - com.amazonaws - aws-java-sdk-apigatewaymanagementapi - 1.12.753 - - - com.amazonaws - aws-lambda-java-events - 3.11.6 - - - com.google.code.gson - gson - 2.11.0 - - - junit - junit - 4.13.2 - - - - org.hamcrest - hamcrest-core - 1.3 - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.6.0 - - - - - package - - shade - - - - - - - diff --git a/deploy/aws/java11Exec/src/main/java/poseidon/App.java b/deploy/aws/java11Exec/src/main/java/poseidon/App.java deleted file mode 100644 index 7beb110..0000000 --- a/deploy/aws/java11Exec/src/main/java/poseidon/App.java +++ /dev/null @@ -1,191 +0,0 @@ -package poseidon; - -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApi; -import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApiClientBuilder; -import com.amazonaws.services.apigatewaymanagementapi.model.PostToConnectionRequest; -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent; -import com.google.gson.Gson; -import com.google.gson.JsonObject; - -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.Base64; -import java.util.Map; - -// AwsFunctionRequest contains the java files that needs to be executed. -class AwsFunctionRequest { - String[] cmd; - Map files; -} - -// WebSocketMessageType are the types of messages that are being sent back over the WebSocket connection. -enum WebSocketMessageType { - WebSocketOutputStdout("stdout"), - WebSocketOutputStderr("stderr"), - WebSocketOutputError("error"), - WebSocketExit("exit"); - - private final String typeName; - - WebSocketMessageType(String name) { - this.typeName = name; - } - - public String toString() { - return typeName; - } -} - -/** - * Handler for requests to Lambda function. - * This Lambda function executes the passed command with the provided files in an isolated Java environment. - */ -public class App implements RequestHandler { - - // gson helps parse the json objects. - private static final Gson gson = new Gson(); - - // gwClient is used to send messages back via the WebSocket connection. - private AmazonApiGatewayManagementApi gwClient; - - // connectionID helps to identify the WebSocket connection that has called this function. - private String connectionID; - - public static final String disableOutputHeaderKey = "disableOutput"; - - // disableOutput: If set to true, no output will be sent over the WebSocket connection. - private boolean disableOutput = false; - - // Unwrapps the passed command. We expect a "sh -c" wrapped command. - public static String unwrapCommand(String[] cmd) { - return cmd[cmd.length - 1]; - } - - // Replaces the last element of the command with the new shell command. - public static String[] wrapCommand(String[] originalCommand, String shellCommand) { - originalCommand[originalCommand.length - 1] = shellCommand; - return originalCommand; - } - - public APIGatewayProxyResponseEvent handleRequest(final APIGatewayV2WebSocketEvent input, final Context context) { - APIGatewayV2WebSocketEvent.RequestContext ctx = input.getRequestContext(); - String[] domains = ctx.getDomainName().split("\\."); - String region = domains[domains.length-3]; - this.gwClient = AmazonApiGatewayManagementApiClientBuilder.standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://" + ctx.getDomainName() + "/" + ctx.getStage(), region)) - .build(); - this.connectionID = ctx.getConnectionId(); - this.disableOutput = input.getHeaders() != null && input.getHeaders().containsKey(disableOutputHeaderKey) && Boolean.parseBoolean(input.getHeaders().get(disableOutputHeaderKey)); - AwsFunctionRequest execution = gson.fromJson(input.getBody(), AwsFunctionRequest.class); - - try { - File workingDirectory = this.writeFS(execution.files); - - String[] cmd = execution.cmd; - try { - SimpleMakefile make = new SimpleMakefile(execution.files); - cmd = wrapCommand(cmd, make.parseCommand(unwrapCommand(execution.cmd))); - } catch (NoMakefileFoundException | NoMakeCommandException | InvalidMakefileException ignored) {} - - ProcessBuilder pb = new ProcessBuilder(cmd); - pb.directory(workingDirectory); - Process p = pb.start(); - InputStream stdout = p.getInputStream(), stderr = p.getErrorStream(); - this.forwardOutput(p, stdout, stderr); - p.destroy(); - return new APIGatewayProxyResponseEvent().withStatusCode(200); - } catch (Exception e) { - this.sendMessage(WebSocketMessageType.WebSocketOutputError, e.toString(), null); - return new APIGatewayProxyResponseEvent().withBody(e.toString()).withStatusCode(500); - } - } - - // writeFS writes the files to the local filesystem. - private File writeFS(Map files) throws IOException { - File workspace = Files.createTempDirectory("workspace").toFile(); - for (Map.Entry entry : files.entrySet()) { - try { - File f = new File(entry.getKey()); - - if (!f.isAbsolute()) { - f = new File(workspace, entry.getKey()); - } - - f.getParentFile().mkdirs(); - if (!f.getParentFile().exists()) { - throw new IOException("Cannot create parent directories."); - } - - f.createNewFile(); - if (!f.exists()) { - throw new IOException("Cannot create file."); - } - - Files.write(f.toPath(), Base64.getDecoder().decode(entry.getValue())); - } catch (IOException e) { - this.sendMessage(WebSocketMessageType.WebSocketOutputError, e.toString(), null); - } - } - return workspace; - } - - // forwardOutput sends the output of the process to the WebSocket connection. - private void forwardOutput(Process p, InputStream stdout, InputStream stderr) throws InterruptedException { - Thread output = new Thread(() -> scanForOutput(p, stdout, WebSocketMessageType.WebSocketOutputStdout)); - Thread error = new Thread(() -> scanForOutput(p, stderr, WebSocketMessageType.WebSocketOutputStderr)); - output.start(); - error.start(); - - output.join(); - error.join(); - this.sendMessage(WebSocketMessageType.WebSocketExit, null, p.waitFor()); - } - - // scanForOutput reads the passed stream and forwards it via the WebSocket connection. - private void scanForOutput(Process p, InputStream stream, WebSocketMessageType type) { - BufferedReader br = new BufferedReader(new InputStreamReader(stream)); - StringBuilder s = new StringBuilder(); - int nextByte; - - try { - while ((nextByte = br.read()) != -1) { - char c = (char) nextByte; - s.append(c); - - if (c == '\n') { - this.sendMessage(type, s.toString(), null); - s = new StringBuilder(); - } - } - } catch (IOException ignored) {} - - if (!s.toString().isEmpty()) { - this.sendMessage(type, s.toString(), null); - } - } - - // sendMessage sends WebSocketMessage objects back to the requester of this Lambda function. - private void sendMessage(WebSocketMessageType type, String data, Integer exitCode) { - JsonObject msg = new JsonObject(); - msg.addProperty("type", type.toString()); - if (type == WebSocketMessageType.WebSocketExit) { - msg.addProperty("data", exitCode); - } else { - msg.addProperty("data", data); - } - - if (this.disableOutput) { - System.out.println(gson.toJson(msg)); - } else { - this.gwClient.postToConnection(new PostToConnectionRequest() - .withConnectionId(this.connectionID) - .withData(ByteBuffer.wrap(gson.toJson(msg).getBytes(StandardCharsets.UTF_8)))); - } - } -} diff --git a/deploy/aws/java11Exec/src/main/java/poseidon/SimpleMakefile.java b/deploy/aws/java11Exec/src/main/java/poseidon/SimpleMakefile.java deleted file mode 100644 index 7e5a16c..0000000 --- a/deploy/aws/java11Exec/src/main/java/poseidon/SimpleMakefile.java +++ /dev/null @@ -1,152 +0,0 @@ -package poseidon; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -// NoMakefileFoundException is thrown if no makefile could be found. -class NoMakefileFoundException extends Exception {} - -// NoMakeCommandException is thrown if no make command is called. -class NoMakeCommandException extends Exception {} - -// InvalidMakefileException is thrown if there is no valid rule to be executed. -class InvalidMakefileException extends Exception {} - -// SimpleMakefile adds limited support for the execution of a makefile as passed command. The default Java image does not contain make. -class SimpleMakefile { - - // This pattern validates if a command is a make command. - private static final Pattern isMakeCommand = Pattern.compile("^(?.* && )?make(?:\\s+(?\\w*))?(?(?:.*?=.*?)+)?(? && .*)?$"); - - // This pattern identifies the rules in a makefile. - private static final Pattern makeRules = Pattern.compile("(?.*):\\r?\\n(?(?:\\t.+\\r?\\n?)*)"); - - // The first rule of the makefile. - private String firstRule = null; - - // The rules included in the makefile. - private final Map rules = new HashMap<>(); - - - private static String concatCommands(String[] commands) { - return String.join(" && ", commands); - } - - // getMakefile returns the makefile out of the passed files map. - private static String getMakefile(Map files) throws NoMakefileFoundException { - String makefileB64; - if (files.containsKey("Makefile")) { - makefileB64 = files.get("Makefile"); - } else if (files.containsKey("makefile")) { - makefileB64 = files.get("makefile"); - } else { - throw new NoMakefileFoundException(); - } - - return new String(Base64.getDecoder().decode(makefileB64), StandardCharsets.UTF_8); - } - - public SimpleMakefile(Map files) throws NoMakefileFoundException { - this.parseRules(getMakefile(files)); - } - - // parseRules uses the passed makefile to parse rules into the objet's map "rules". - private void parseRules(String makefile) { - Matcher makeRuleMatcher = makeRules.matcher(makefile); - while (makeRuleMatcher.find()) { - String ruleName = makeRuleMatcher.group("name"); - if (firstRule == null) { - firstRule = ruleName; - } - - String[] ruleCommands = makeRuleMatcher.group("commands").split("\n"); - String[] trimmedCommands = Arrays.stream(ruleCommands) - .map(String::trim) - .map(s -> s.startsWith("@") ? s.substring(1) : s) - .map(s -> s.contains("#") ? s.substring(0, s.indexOf("#")) : s) - .filter(s -> !s.isEmpty()) - .map(s -> s.replaceAll("/usr/java/lib/hamcrest-core-1\\.3\\.jar", "/var/task/lib/org.hamcrest.hamcrest-core-1.3.jar")) - .map(s -> s.replaceAll("/usr/java/lib/junit-4\\.13\\.jar", "/var/task/lib/junit.junit-4.13.2.jar")) - .toArray(String[]::new); - - rules.put(ruleName, trimmedCommands); - } - } - - // getCommand returns a bash line of commands that would be executed by the passed rule. - public String getCommand(String rule) { - if (rule == null || rule.isEmpty()) { - rule = this.firstRule; - } - - return concatCommands(rules.get(rule)); - } - - // updateCommand injects the passed make assignments and before and after statements to the command. - private String updateCommand(Matcher makeCommandMatcher, String command) { - String assignments = makeCommandMatcher.group("assignments"); - if (assignments != null) { - command = injectAssignments(command, assignments); - } - - if (makeCommandMatcher.group("before") != null) { - command = makeCommandMatcher.group("before") + command; - } - if (makeCommandMatcher.group("after") != null) { - command = command + makeCommandMatcher.group("after"); - } - - return command; - } - - // getAssignmentPart returns the key or value of the passed assignment depending on the flag firstPart. - private String getAssignmentPart(String assignment, boolean firstPart) { - String[] parts = assignment.split("="); - - if (firstPart) { - return parts[0]; - } else { - return parts[1].replaceAll("^\\\"|\\\"$", ""); - } - } - - // injectAssignments applies all set assignments in the command string. - private String injectAssignments(String command, String assignments) { - String result = command; - Map map = Arrays.stream(assignments.split(" ")) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toMap(s -> getAssignmentPart(s, true), s -> getAssignmentPart(s, false))); - - for (Map.Entry entry : map.entrySet()) { - result = result.replaceAll("\\$\\{" + entry.getKey() + "\\}", entry.getValue()); - } - - return result; - } - - // parseCommand returns a bash line of commands that would be executed by the passed command. - public String parseCommand(String shellCommand) throws InvalidMakefileException, NoMakeCommandException { - Matcher makeCommandMatcher = isMakeCommand.matcher(shellCommand); - if (!makeCommandMatcher.find()) { - throw new NoMakeCommandException(); - } - - String ruleArgument = makeCommandMatcher.group("startRule"); - if (ruleArgument.isEmpty()) { - ruleArgument = this.firstRule; - } - - if ((this.firstRule == null) || !rules.containsKey(ruleArgument)) { - throw new InvalidMakefileException(); - } - - String command = getCommand(ruleArgument); - return updateCommand(makeCommandMatcher, command); - } -} diff --git a/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java b/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java deleted file mode 100644 index 12b2b2c..0000000 --- a/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package poseidon; - -import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; -import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - - -public class AppTest { - - static final String RecursiveMathContent = Base64.getEncoder().encodeToString( - ("package org.example;\n" + - "\n" + - "public class RecursiveMath {\n" + - "\n" + - " public static void main(String[] args) {\n" + - " System.out.println(\"Mein Text\");\n" + - " }\n" + - "\n" + - " public static double power(int base, int exponent) {\n" + - " return 42;\n" + - " }\n" + - "}").getBytes(StandardCharsets.UTF_8)); - - static final String MultilineMathContent = Base64.getEncoder().encodeToString( - ("package org.example;\n" + - "\n" + - "public class RecursiveMath {\n" + - "\n" + - " public static void main(String[] args) {\n" + - " System.out.println(\"Mein Text\");\n" + - " System.out.println(\"Mein Text\");\n" + - " }\n" + - "}").getBytes(StandardCharsets.UTF_8)); - - static final String MathContentWithoutTrailingNewline = Base64.getEncoder().encodeToString( - ("package org.example;\n" + - "\n" + - "public class RecursiveMath {\n" + - "\n" + - " public static void main(String[] args) {\n" + - " System.out.print(\"Mein Text\");\n" + - " }\n" + - "}").getBytes(StandardCharsets.UTF_8)); - - @Test - public void successfulResponse() { - APIGatewayProxyResponseEvent result = getApiGatewayProxyResponseRecursiveMath(RecursiveMathContent); - assertEquals(200, result.getStatusCode().intValue()); - } - - // Scanner.nextLine() consumes the new line. Therefore, we need to add it again. - @Test - public void successfulMultilineResponse() { - ByteArrayOutputStream out = setupStdOutLogs(); - APIGatewayProxyResponseEvent result = getApiGatewayProxyResponseRecursiveMath(MultilineMathContent); - restoreStdOutLogs(); - - assertEquals(200, result.getStatusCode().intValue()); - String expectedOutput = - "{\"type\":\"stdout\",\"data\":\"Mein Text\\n\"}\n" + - "{\"type\":\"stdout\",\"data\":\"Mein Text\\n\"}\n" + - "{\"type\":\"exit\",\"data\":0}\n"; - assertEquals(expectedOutput, out.toString()); - } - - @Test - public void outputWithoutTrailingNewline() { - ByteArrayOutputStream out = setupStdOutLogs(); - APIGatewayProxyResponseEvent result = getApiGatewayProxyResponseRecursiveMath(MathContentWithoutTrailingNewline); - restoreStdOutLogs(); - - assertEquals(200, result.getStatusCode().intValue()); - String expectedOutput = - "{\"type\":\"stdout\",\"data\":\"Mein Text\"}\n" + - "{\"type\":\"exit\",\"data\":0}\n"; - assertEquals(expectedOutput, out.toString()); - } - - @Test - public void makefileJustReplacesShellCommand() { - ByteArrayOutputStream out = setupStdOutLogs(); - APIGatewayProxyResponseEvent result = getApiGatewayProxyResponse("{\"action\":\"java11Exec\"," + - "\"cmd\":[\"env\", \"TEST_VAR=42\", \"sh\",\"-c\",\"make run\"]," + - "\"files\":{\"Makefile\":\"" + Base64.getEncoder().encodeToString(("run:\n\t@echo $TEST_VAR\n").getBytes(StandardCharsets.UTF_8)) + "\"}}"); - restoreStdOutLogs(); - - assertEquals(200, result.getStatusCode().intValue()); - String expectedOutput = - "{\"type\":\"stdout\",\"data\":\"42\\n\"}\n" + - "{\"type\":\"exit\",\"data\":0}\n"; - assertEquals(expectedOutput, out.toString()); - } - - - private PrintStream originalOut; - private ByteArrayOutputStream setupStdOutLogs() { - originalOut = System.out; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); - return out; - } - - private void restoreStdOutLogs() { - System.setOut(originalOut); - } - - private APIGatewayProxyResponseEvent getApiGatewayProxyResponse(String body) { - App app = new App(); - APIGatewayV2WebSocketEvent input = new APIGatewayV2WebSocketEvent(); - APIGatewayV2WebSocketEvent.RequestContext ctx = new APIGatewayV2WebSocketEvent.RequestContext(); - ctx.setDomainName("abcdef1234.execute-api.eu-central-1.amazonaws.com"); - ctx.setConnectionId("myUUID"); - input.setRequestContext(ctx); - Map headers = new HashMap<>(); - headers.put(App.disableOutputHeaderKey, "True"); - input.setHeaders(headers); - input.setBody(body); - return app.handleRequest(input, null); - } - - private APIGatewayProxyResponseEvent getApiGatewayProxyResponseRecursiveMath(String content) { - return getApiGatewayProxyResponse("{\"action\":\"java11Exec\",\"cmd\":[\"sh\",\"-c\",\"javac org/example/RecursiveMath.java && java org/example/RecursiveMath\"]," + - "\"files\":{\"org/example/RecursiveMath.java\":\"" + content + "\"}}"); - } -} diff --git a/deploy/aws/java11Exec/src/test/java/poseidon/SimpleMakefileTest.java b/deploy/aws/java11Exec/src/test/java/poseidon/SimpleMakefileTest.java deleted file mode 100644 index f367941..0000000 --- a/deploy/aws/java11Exec/src/test/java/poseidon/SimpleMakefileTest.java +++ /dev/null @@ -1,167 +0,0 @@ -package poseidon; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static poseidon.AppTest.RecursiveMathContent; - - -public class SimpleMakefileTest { - static final String SuccessfulMakefile = Base64.getEncoder().encodeToString( - ("run:\n" + - "\tjavac org/example/RecursiveMath.java\n" + - "\tjava org/example/RecursiveMath\n" + - "\n" + - "test:\n" + - "\techo Hi\n" - ).getBytes(StandardCharsets.UTF_8)); - - static final String SuccessfulWindowsMakefile = Base64.getEncoder().encodeToString( - ("run:\r\n" + - "\tjavac org/example/RecursiveMath.java\r\n" + - "\tjava org/example/RecursiveMath\r\n" + - "\r\n" + - "test:\r\n" + - "\techo Hi\r\n" - ).getBytes(StandardCharsets.UTF_8)); - - static final String SuccessfulMakefileWithAtSymbol = Base64.getEncoder().encodeToString( - ("run:\r\n" + - "\t@javac org/example/RecursiveMath.java\r\n" + - "\t@java org/example/RecursiveMath\r\n" - ).getBytes(StandardCharsets.UTF_8)); - - static final String SuccessfulMakefileWithAssignments = Base64.getEncoder().encodeToString( - ("test:\n" + - "\tjavac -encoding utf8 -cp .:/usr/java/lib/hamcrest-core-1.3.jar:/usr/java/lib/junit-4.13.jar ${FILENAME}\n" + - "\tjava -Dfile.encoding=UTF8 -cp .:/usr/java/lib/hamcrest-core-1.3.jar:/usr/java/lib/junit-4.13.jar org.junit.runner.JUnitCore ${CLASS_NAME}\n" - ).getBytes(StandardCharsets.UTF_8)); - - static final String SuccessfulMakefileWithComment = Base64.getEncoder().encodeToString( - ("run:\r\n" + - "\t@javac org/example/RecursiveMath.java\r\n" + - "\t@java org/example/RecursiveMath\r\n" + - "\t#exit\r\n" - ).getBytes(StandardCharsets.UTF_8)); - - static final String NotSupportedMakefile = Base64.getEncoder().encodeToString( - ("run: test\n" + - "\tjavac org/example/RecursiveMath.java\n" + - "\tjava org/example/RecursiveMath\n" + - "\n" + - "test:\n" + - "\techo Hi\n" - ).getBytes(StandardCharsets.UTF_8)); - - @Test - public void sucessfullMake() { - parseRunCommandOfMakefile(SuccessfulMakefile); - } - - @Test - public void sucessfullMakeWithCR() { - parseRunCommandOfMakefile(SuccessfulWindowsMakefile); - } - - // We remove [the @ Symbol](https://www.gnu.org/software/make/manual/make.html#Echoing) - // as the command itself is never written to stdout with this implementation. - @Test - public void sucessfullMakeWithAtSymbol() { - parseRunCommandOfMakefile(SuccessfulMakefileWithAtSymbol); - } - - // We remove [any comments with #](https://www.gnu.org/software/make/manual/make.html#Recipe-Syntax) - // as they are normally ignored / echoed with most shells. - @Test - public void sucessfullMakeWithComment() { - parseRunCommandOfMakefile(SuccessfulMakefileWithComment); - } - - private void parseRunCommandOfMakefile(String makefileB64) { - Map files = new HashMap<>(); - files.put("Makefile", makefileB64); - files.put("org/example/RecursiveMath.java", RecursiveMathContent); - - try { - String command = "make run"; - SimpleMakefile makefile = new SimpleMakefile(files); - String cmd = makefile.parseCommand(command); - - assertEquals("javac org/example/RecursiveMath.java && java org/example/RecursiveMath", cmd); - } catch (NoMakefileFoundException | InvalidMakefileException | NoMakeCommandException ignored) { - fail(); - } - } - - @Test - public void withoutMake() { - Map files = new HashMap<>(); - files.put("Makefile", SuccessfulMakefile); - files.put("org/example/RecursiveMath.java", RecursiveMathContent); - - try { - String command = "javac org/example/RecursiveMath.java"; - SimpleMakefile make = new SimpleMakefile(files); - make.parseCommand(command); - fail(); - } catch (NoMakefileFoundException | InvalidMakefileException ignored) { - fail(); - } catch (NoMakeCommandException ignored) {} - } - - @Test - public void sucessfullMakeWithAssignments() { - Map files = new HashMap<>(); - files.put("Makefile", SuccessfulMakefileWithAssignments); - files.put("org/example/RecursiveMath.java", RecursiveMathContent); - - try { - String command = "make test CLASS_NAME=\"RecursiveMath\" FILENAME=\"RecursiveMath-Test.java\""; - SimpleMakefile make = new SimpleMakefile(files); - String cmd = make.parseCommand(command); - - assertEquals("javac -encoding utf8 -cp .:/var/task/lib/org.hamcrest.hamcrest-core-1.3.jar:/var/task/lib/junit.junit-4.13.2.jar RecursiveMath-Test.java && " + - "java -Dfile.encoding=UTF8 -cp .:/var/task/lib/org.hamcrest.hamcrest-core-1.3.jar:/var/task/lib/junit.junit-4.13.2.jar org.junit.runner.JUnitCore RecursiveMath", cmd); - } catch (NoMakefileFoundException | InvalidMakefileException | NoMakeCommandException ignored) { - fail(); - } - } - - @Test - public void withNotSupportedMakefile() { - Map files = new HashMap<>(); - files.put("Makefile", NotSupportedMakefile); - files.put("org/example/RecursiveMath.java", RecursiveMathContent); - - try { - String command = "make run"; - SimpleMakefile makefile = new SimpleMakefile(files); - makefile.parseCommand(command); - fail(); - } catch (NoMakefileFoundException | NoMakeCommandException ignored) { - fail(); - } catch (InvalidMakefileException ignored) {} - } - - @Test - public void withBeforeAndAfterStatements() { - Map files = new HashMap<>(); - files.put("Makefile", Base64.getEncoder().encodeToString(("run:\n\t@echo TRAAAIIN\n").getBytes(StandardCharsets.UTF_8))); - - try { - String command = "echo \"Look it's a\" && sl && make run && echo WOW"; - SimpleMakefile makefile = new SimpleMakefile(files); - String cmd = makefile.parseCommand(command); - - assertEquals("echo \"Look it's a\" && sl && echo TRAAAIIN && echo WOW", cmd); - } catch (NoMakefileFoundException | InvalidMakefileException | NoMakeCommandException ignored) { - fail(); - } - } -} diff --git a/deploy/aws/template.yaml b/deploy/aws/template.yaml deleted file mode 100644 index 5e7c344..0000000 --- a/deploy/aws/template.yaml +++ /dev/null @@ -1,105 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: > - PoseidonExecutors - - Execute untrusted code in AWS functions. - -# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst -Globals: - Function: - Timeout: 15 - -Resources: - PoseidonExecWebSocket: - Type: AWS::ApiGatewayV2::Api - Properties: - Name: PoseidonExecWebSocket - ProtocolType: WEBSOCKET - RouteSelectionExpression: "$request.body.action" - - Deployment: - Type: AWS::ApiGatewayV2::Deployment - DependsOn: - - java11ExecRoute - Properties: - ApiId: !Ref PoseidonExecWebSocket - - Stage: - Type: AWS::ApiGatewayV2::Stage - Properties: - StageName: production - Description: Production Stage - DeploymentId: !Ref Deployment - ApiId: !Ref PoseidonExecWebSocket - - java11ExecRoute: # More info about Routes: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigatewayv2-route.html - Type: AWS::ApiGatewayV2::Route - Properties: - ApiId: !Ref PoseidonExecWebSocket - RouteKey: java11Exec - AuthorizationType: NONE - OperationName: java11ExecRoute - Target: !Join - - '/' - - - 'integrations' - - !Ref java11ExecInteg - - java11ExecInteg: # More info about Integrations: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html - Type: AWS::ApiGatewayV2::Integration - Properties: - ApiId: !Ref PoseidonExecWebSocket - Description: Java 11 Exec Integration - IntegrationType: AWS_PROXY - IntegrationUri: - Fn::Sub: - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${java11ExecFunction.Arn}/invocations - - java11ExecFunction: - Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction - Properties: - CodeUri: java11Exec/ - Handler: poseidon.App::handleRequest - Runtime: java11 - Architectures: - - arm64 - MemorySize: 2048 - Policies: - - Statement: - - Effect: Allow - Action: - - 'execute-api:*' - Resource: "*" - - Effect: Allow - Action: - - 'logs:CreateLogGroup' - Resource: - - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*' - - Effect: Allow - Action: - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: - - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${PoseidonExecWebSocket}:*' - - java11ExecPermission: - Type: AWS::Lambda::Permission - DependsOn: - - PoseidonExecWebSocket - Properties: - Action: lambda:InvokeFunction - FunctionName: !Ref java11ExecFunction - Principal: apigateway.amazonaws.com - -Outputs: - WebSocketURI: - Description: "The WSS Protocol URI to connect to" - Value: !Join [ '', [ 'wss://', !Ref PoseidonExecWebSocket, '.execute-api.',!Ref 'AWS::Region','.amazonaws.com/',!Ref 'Stage' ] ] - - java11ExecFunctionArn: - Description: "Java 11 Execution Lambda Function ARN" - Value: !GetAtt java11ExecFunction.Arn - - java11ExecFunctionIamRole: - Description: "Implicit IAM Role created for the Java 11 Execution function" - Value: !GetAtt java11ExecFunctionRole.Arn diff --git a/deploy/codeocean-terraform b/deploy/codeocean-terraform deleted file mode 160000 index 2717dd9..0000000 --- a/deploy/codeocean-terraform +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2717dd9ad672988980c30dce5619639aeb4570d4 diff --git a/deploy/docker-make/Dockerfile b/deploy/docker-make/Dockerfile deleted file mode 100644 index 1a040a5..0000000 --- a/deploy/docker-make/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM docker:latest - -RUN apk update && apk add make diff --git a/deploy/grafana-dashboard/.gitignore b/deploy/grafana-dashboard/.gitignore deleted file mode 100644 index 2061a6f..0000000 --- a/deploy/grafana-dashboard/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Ignore the generated json encoded dashboard. -main.json - -# Ignore the current environments required for generating the dashboard. -environments.json - -# Ignore the Grafana dashboard deployment files. -dashboards/ diff --git a/deploy/grafana-dashboard/Pipfile b/deploy/grafana-dashboard/Pipfile deleted file mode 100644 index 5d2c345..0000000 --- a/deploy/grafana-dashboard/Pipfile +++ /dev/null @@ -1,9 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -grafanalib = "*" - -[dev-packages] diff --git a/deploy/grafana-dashboard/Pipfile.lock b/deploy/grafana-dashboard/Pipfile.lock deleted file mode 100644 index 1b46622..0000000 --- a/deploy/grafana-dashboard/Pipfile.lock +++ /dev/null @@ -1,35 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "6b655eb9342c7315ed456ce5fe58b653a36f3726736f0cecf6252e9a2247d9ee" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "attrs": { - "hashes": [ - "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", - "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" - ], - "markers": "python_version >= '3.7'", - "version": "==23.2.0" - }, - "grafanalib": { - "hashes": [ - "sha256:3d92bb4e92ae78fe4e21c5b252ab51f4fdcacd8523ba5a44545b897b2a375b83", - "sha256:6fab5d7b837a1f2d1322ef762cd52e565ec0422707a7512765c59f668bdceb58" - ], - "index": "pypi", - "version": "==0.7.1" - } - }, - "develop": {} -} diff --git a/deploy/grafana-dashboard/Readme.md b/deploy/grafana-dashboard/Readme.md deleted file mode 100644 index 21d7972..0000000 --- a/deploy/grafana-dashboard/Readme.md +++ /dev/null @@ -1,16 +0,0 @@ -# Grafana Dashboard Deployment - -## Grafanalib - -We use [Grafanalib](https://github.com/weaveworks/grafanalib) for the definition of our dashboard. -You need to install the python package: `pip install grafanalib`. - -### Generation - -Generate the Grafana dashboard by running `main.py`. -The generated Json definition can be imported in the Grafana UI. - -### Deployment - -You can copy the generated json into the field under the dashboards setting -> "JSON Model". -The version number needs to match! diff --git a/deploy/grafana-dashboard/dashboards/.keep b/deploy/grafana-dashboard/dashboards/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/grafana-dashboard/environments.json.sample b/deploy/grafana-dashboard/environments.json.sample deleted file mode 100644 index 57886c6..0000000 --- a/deploy/grafana-dashboard/environments.json.sample +++ /dev/null @@ -1,15 +0,0 @@ -{ - "executionEnvironments": [ - {"image": "openhpi/co_execenv_java:8-antlr", "id": 10}, - {"image": "openhpi/co_execenv_r:4", "id": 28}, - {"image": "openhpi/co_execenv_python:3.8", "id": 29}, - {"image": "openhpi/co_execenv_java:17", "id": 31}, - {"image": "openhpi/docker_exec_phusion", "id": 33}, - {"image": "openhpi/co_execenv_java:8-antlr", "id": 11}, - {"image": "openhpi/co_execenv_python:3.4", "id": 14}, - {"image": "openhpi/co_execenv_node:0.12", "id": 18}, - {"image": "openhpi/co_execenv_python:3.4-rpi-web", "id": 22}, - {"image": "openhpi/co_execenv_ruby:2.5", "id": 25}, - {"image": "openhpi/co_execenv_python:3.7-ml", "id": 30} - ] -} diff --git a/deploy/grafana-dashboard/main.py b/deploy/grafana-dashboard/main.py deleted file mode 100644 index 1b5fac1..0000000 --- a/deploy/grafana-dashboard/main.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/python3 -from grafanalib._gen import generate_dashboard - -if __name__ == "__main__": - generate_dashboard(args=["-o", "main.json", "panels/poseidon.dashboard.py"]) diff --git a/deploy/grafana-dashboard/panels/availability_row.py b/deploy/grafana-dashboard/panels/availability_row.py deleted file mode 100644 index 05f130b..0000000 --- a/deploy/grafana-dashboard/panels/availability_row.py +++ /dev/null @@ -1,62 +0,0 @@ -from grafanalib.core import RowPanel, BarGauge, GridPos, TimeSeries, ORIENTATION_VERTICAL, \ - GAUGE_DISPLAY_MODE_BASIC -from grafanalib.influxdb import InfluxDBTarget - -from utils.color_mapping import color_mapping_environments -from utils.utils import read_query - -prewarming_pool_size = BarGauge( - title="Prewarming Pool Size", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("prewarming-pool-size", "environment-mapping"))], - gridPos=GridPos(h=10, w=11, x=0, y=1), - allValues=True, - orientation=ORIENTATION_VERTICAL, - displayMode=GAUGE_DISPLAY_MODE_BASIC, - max=None, - extraJson=color_mapping_environments, -) - -idle_runner = TimeSeries( - title="Idle Runner", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("idle-runner", "environment-mapping"))], - gridPos=GridPos(h=10, w=13, x=11, y=1), - lineInterpolation="stepAfter", - maxDataPoints=None, - extraJson=color_mapping_environments, -) - -runner_startup_duration = TimeSeries( - title="Runner startup duration", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("runner-startup-duration", "environment-mapping"))], - gridPos=GridPos(h=10, w=12, x=0, y=11), - unit="ns", - maxDataPoints=None, - lineInterpolation="smooth", - extraJson=color_mapping_environments, -) - -used_runner = TimeSeries( - title="Used Runner", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("used-runner"))], - gridPos=GridPos(h=10, w=12, x=12, y=11), - maxDataPoints=None, - lineInterpolation="smooth", -) - -availability_row = RowPanel( - title="Availability", - collapsed=False, - gridPos=GridPos(h=1, w=24, x=0, y=0), -) - -availability_panels = [ - availability_row, - prewarming_pool_size, - idle_runner, - runner_startup_duration, - used_runner, -] diff --git a/deploy/grafana-dashboard/panels/general_row.py b/deploy/grafana-dashboard/panels/general_row.py deleted file mode 100644 index d09e77a..0000000 --- a/deploy/grafana-dashboard/panels/general_row.py +++ /dev/null @@ -1,127 +0,0 @@ -from grafanalib.core import RowPanel, GridPos, Stat, TimeSeries, Heatmap, BarGauge, GAUGE_DISPLAY_MODE_GRADIENT, \ - ORIENTATION_VERTICAL, GAUGE_DISPLAY_MODE_BASIC -from grafanalib.influxdb import InfluxDBTarget - -from utils.color_mapping import grey_all_mapping, color_mapping_environments -from utils.utils import read_query - -requests_per_minute = TimeSeries( - title="Requests per minute", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("requests-per-minute"))], - gridPos=GridPos(h=9, w=8, x=0, y=22), - scaleDistributionType="log", - extraJson=grey_all_mapping, - lineInterpolation="smooth", -) - -request_latency = Heatmap( - title="Request Latency", - dataSource="Flux", - dataFormat="timeseries", - targets=[InfluxDBTarget(query=read_query("request-latency"))], - gridPos=GridPos(h=9, w=8, x=8, y=22), - maxDataPoints=None, - extraJson={ - "options": {}, - "yAxis": { - "format": "ns" - } - }, -) - -service_time = TimeSeries( - title="Service time (99.9%)", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("service-time"))], - gridPos=GridPos(h=9, w=8, x=16, y=22), - scaleDistributionType="log", - scaleDistributionLog=10, - unit="ns", - maxDataPoints=None, - lineInterpolation="smooth", -) - -current_environment_count = Stat( - title="Current environment count", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("current-environment-count"))], - gridPos=GridPos(h=6, w=8, x=0, y=31), - alignment="center", -) - -currently_used_runners = Stat( - title="Currently used runners", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("currently-used-runners"))], - gridPos=GridPos(h=6, w=8, x=8, y=31), - alignment="center", -) - -number_of_executions = BarGauge( - title="Number of Executions", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("number-of-executions", "environment-mapping"))], - gridPos=GridPos(h=6, w=8, x=16, y=31), - allValues=True, - orientation=ORIENTATION_VERTICAL, - displayMode=GAUGE_DISPLAY_MODE_BASIC, - max=None, - extraJson=color_mapping_environments, -) - -execution_duration = BarGauge( - title="Execution duration", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("execution-duration", "environment-mapping"))], - gridPos=GridPos(h=11, w=8, x=0, y=37), - allValues=True, - displayMode=GAUGE_DISPLAY_MODE_GRADIENT, - format="ns", - max=None, - decimals=2, - extraJson=color_mapping_environments, -) - -executions_per_runner = BarGauge( - title="Executions per runner", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("executions-per-runner", "environment-mapping"))], - gridPos=GridPos(h=11, w=8, x=8, y=37), - allValues=True, - displayMode=GAUGE_DISPLAY_MODE_GRADIENT, - max=None, - decimals=2, - extraJson=color_mapping_environments, -) - -executions_per_minute = BarGauge( - title="Executions per minute", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("executions-per-minute", "environment-mapping"))], - gridPos=GridPos(h=11, w=8, x=16, y=37), - allValues=True, - displayMode=GAUGE_DISPLAY_MODE_GRADIENT, - max=None, - decimals=2, - extraJson=color_mapping_environments, -) - -general_row = RowPanel( - title="General", - collapsed=False, - gridPos=GridPos(h=1, w=24, x=0, y=21), -) - -general_panels = [ - general_row, - requests_per_minute, - request_latency, - service_time, - current_environment_count, - currently_used_runners, - number_of_executions, - execution_duration, - executions_per_runner, - executions_per_minute, -] diff --git a/deploy/grafana-dashboard/panels/poseidon.dashboard.py b/deploy/grafana-dashboard/panels/poseidon.dashboard.py deleted file mode 100644 index 687f182..0000000 --- a/deploy/grafana-dashboard/panels/poseidon.dashboard.py +++ /dev/null @@ -1,18 +0,0 @@ -from grafanalib.core import Dashboard, Templating, Time - -from panels.availability_row import availability_panels -from panels.general_row import general_panels -from panels.runner_insights_row import runner_insights_panels -from utils.variables import environment_variable - -dashboard = Dashboard( - title="Poseidon autogen", - timezone="browser", - panels=availability_panels + general_panels + runner_insights_panels, - templating=Templating(list=[ environment_variable ]), - editable=True, - refresh="30s", - time=Time("now-6h", "now"), - uid="P21Bh1SVk", - version=1, -).auto_panel_ids() diff --git a/deploy/grafana-dashboard/panels/runner_insights_row.py b/deploy/grafana-dashboard/panels/runner_insights_row.py deleted file mode 100644 index c5c06cf..0000000 --- a/deploy/grafana-dashboard/panels/runner_insights_row.py +++ /dev/null @@ -1,115 +0,0 @@ -from grafanalib.core import RowPanel, GridPos, Histogram, TimeSeries, BarGauge, ORIENTATION_VERTICAL, \ - GAUGE_DISPLAY_MODE_BASIC, PERCENT_UNIT_FORMAT, GAUGE_CALC_MEAN -from grafanalib.influxdb import InfluxDBTarget - -from utils.color_mapping import color_mapping_environments -from utils.utils import read_query, deep_update_dict - -execution_duration_extra_json = { - "fieldConfig": { - "defaults": { - "unit": "ns" - } - } -} -deep_update_dict(execution_duration_extra_json, color_mapping_environments) -execution_duration = Histogram( - title="Execution duration", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("execution-duration-hist", "environment-mapping"))], - gridPos=GridPos(h=8, w=24, x=0, y=49), - bucketSize=100000000, - colorMode="palette-classic", - fillOpacity=50, - lineWidth=1, - maxDataPoints=None, - extraJson=execution_duration_extra_json, -) - -executions_per_runner = Histogram( - title="Executions per runner", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("executions-per-runner-hist", "environment-mapping"))], - gridPos=GridPos(h=10, w=11, x=0, y=57), - bucketSize=1, - colorMode="palette-classic", - fillOpacity=50, - lineWidth=1, - maxDataPoints=None, - extraJson=color_mapping_environments, -) - -executions_per_minute = TimeSeries( - title="Executions per minute", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("executions-per-minute-time", "environment-mapping"))], - gridPos=GridPos(h=10, w=13, x=11, y=57), - maxDataPoints=None, - lineInterpolation="smooth", - extraJson=color_mapping_environments, -) - -file_upload = TimeSeries( - title="File Upload", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("file-upload", "environment-mapping"))], - gridPos=GridPos(h=10, w=11, x=0, y=67), - scaleDistributionType="log", - unit="bytes", - maxDataPoints=None, - lineInterpolation="smooth", - extraJson=color_mapping_environments, -) - -runner_per_minute = TimeSeries( - title="Runner per minute", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("runner-per-minute", "environment-mapping"))], - gridPos=GridPos(h=10, w=13, x=11, y=67), - maxDataPoints=None, - lineInterpolation="smooth", - extraJson=color_mapping_environments, -) - -file_download = TimeSeries( - title="File Download", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("file-download", "environment-mapping"))], - gridPos=GridPos(h=10, w=11, x=0, y=77), - scaleDistributionType="log", - unit="bytes", - maxDataPoints=None, - lineInterpolation="smooth", - extraJson=color_mapping_environments, -) - -file_download_ratio = BarGauge( - title="File Download Ratio", - dataSource="Flux", - targets=[InfluxDBTarget(query=read_query("file-download-ratio", "environment-mapping"))], - gridPos=GridPos(h=10, w=13, x=11, y=77), - max=1, - allValues=False, - calc=GAUGE_CALC_MEAN, - orientation=ORIENTATION_VERTICAL, - displayMode=GAUGE_DISPLAY_MODE_BASIC, - format=PERCENT_UNIT_FORMAT, - extraJson=color_mapping_environments, -) - -runner_insights_row = RowPanel( - title="Runner Insights", - collapsed=False, - gridPos=GridPos(h=1, w=24, x=0, y=48), -) - -runner_insights_panels = [ - runner_insights_row, - execution_duration, - executions_per_runner, - executions_per_minute, - file_upload, - runner_per_minute, - file_download, - file_download_ratio, -] diff --git a/deploy/grafana-dashboard/queries/current-environment-count.flux b/deploy/grafana-dashboard/queries/current-environment-count.flux deleted file mode 100644 index de5d569..0000000 --- a/deploy/grafana-dashboard/queries/current-environment-count.flux +++ /dev/null @@ -1,19 +0,0 @@ -import "date" - -// The need for the date truncation is caused by Poseidon sending all influx events at the same time when starting up. This way not the last but a random value is displayed. -// Since in this startup process the highest value is the correct one, we choose the highest value of the last events. - -data = from(bucket: "poseidon") - |> range(start: -1y) - |> filter(fn: (r) => r["_measurement"] == "poseidon_environments") - |> group(columns: ["stage"], mode:"by") - |> map(fn: (r) => ({ r with _time: date.truncate(t: r._time, unit: 1m) })) - -deploy_times = data - |> last() - |> keep(columns: ["stage", "_time"]) - -join(tables: {key1: data, key2: deploy_times}, on: ["stage", "_time"], method: "inner") - |> max() - |> keep(columns: ["stage", "_value"]) - |> rename(columns: {_value: ""}) diff --git a/deploy/grafana-dashboard/queries/currently-used-runners.flux b/deploy/grafana-dashboard/queries/currently-used-runners.flux deleted file mode 100644 index 92e85d8..0000000 --- a/deploy/grafana-dashboard/queries/currently-used-runners.flux +++ /dev/null @@ -1,8 +0,0 @@ -from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") - |> filter(fn: (r) => r["_field"] == "count") - |> group(columns: ["stage"], mode:"by") - |> last() - |> keep(columns: ["_value", "stage"]) - |> rename(columns: {_value: ""}) diff --git a/deploy/grafana-dashboard/queries/environment-ids.flux b/deploy/grafana-dashboard/queries/environment-ids.flux deleted file mode 100644 index 878b88b..0000000 --- a/deploy/grafana-dashboard/queries/environment-ids.flux +++ /dev/null @@ -1,6 +0,0 @@ -from(bucket: "poseidon") - |> range(start: -1y) - |> filter(fn: (r) => r["_measurement"] == "poseidon_environments") - |> keep(columns: ["id"]) - |> distinct(column: "id") - |> keep(columns: ["_value"]) diff --git a/deploy/grafana-dashboard/queries/environment-mapping.flux b/deploy/grafana-dashboard/queries/environment-mapping.flux deleted file mode 100644 index 953be73..0000000 --- a/deploy/grafana-dashboard/queries/environment-mapping.flux +++ /dev/null @@ -1,14 +0,0 @@ -envMapping = from(bucket: "poseidon") - |> range(start: -1y) - |> filter(fn: (r) => r["_measurement"] == "poseidon_environments") - |> filter(fn: (r) => r["event_type"] == "creation") - |> group(columns: ["id", "stage"], mode:"by") - |> last() - |> keep(columns: ["id", "image", "stage"]) - |> rename(columns: {id: "environment_id"}) - |> map(fn: (r) => ({ r with image: r.environment_id + "/" + strings.trimPrefix(v: r.image, prefix: "openhpi/co_execenv_")})) - -join(tables: {key1: result, key2: envMapping}, on: ["environment_id", "stage"], method: "inner") - |> keep(columns: ["_value", "image", "_time"]) - |> group(columns: ["image"], mode: "by") - |> rename(columns: {_value: ""}) diff --git a/deploy/grafana-dashboard/queries/execution-duration-hist.flux b/deploy/grafana-dashboard/queries/execution-duration-hist.flux deleted file mode 100644 index ac78b4b..0000000 --- a/deploy/grafana-dashboard/queries/execution-duration-hist.flux +++ /dev/null @@ -1,11 +0,0 @@ -import "strings" - -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_field"] == "duration") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket") - |> filter(fn: (r) => exists r.environment_id) - |> keep(columns: ["_time", "_value", "environment_id", "stage"]) - |> aggregateWindow(every: v.windowPeriod, fn: mean) - |> map(fn: (r) => ({r with _value: r._value * 3.0})) // Each execution has three requests diff --git a/deploy/grafana-dashboard/queries/execution-duration.flux b/deploy/grafana-dashboard/queries/execution-duration.flux deleted file mode 100644 index 786682b..0000000 --- a/deploy/grafana-dashboard/queries/execution-duration.flux +++ /dev/null @@ -1,12 +0,0 @@ -import "strings" - -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_field"] == "duration") - |> filter(fn: (r) => r["_measurement"] == "poseidon_/execute" or r["_measurement"] == "poseidon_/files" or r["_measurement"] == "poseidon_/websocket") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => exists r.environment_id) - |> keep(columns: ["_value", "runner_id", "environment_id", "stage"]) - |> group(columns: ["environment_id", "stage"]) - |> mean() - |> map(fn: (r) => ({r with _value: r._value * 3.0})) // Each execution has three requests diff --git a/deploy/grafana-dashboard/queries/executions-per-minute-time.flux b/deploy/grafana-dashboard/queries/executions-per-minute-time.flux deleted file mode 100644 index 91aa92b..0000000 --- a/deploy/grafana-dashboard/queries/executions-per-minute-time.flux +++ /dev/null @@ -1,11 +0,0 @@ -import "strings" -import "date" - -result = from(bucket: "poseidon") - |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) - |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => r["event_type"] == "creation") - |> group(columns: ["environment_id", "stage"], mode:"by") - |> aggregateWindow(every: 1m, fn: count, createEmpty: true) - |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 8), fn: mean, createEmpty: true) diff --git a/deploy/grafana-dashboard/queries/executions-per-minute.flux b/deploy/grafana-dashboard/queries/executions-per-minute.flux deleted file mode 100644 index 3ce154e..0000000 --- a/deploy/grafana-dashboard/queries/executions-per-minute.flux +++ /dev/null @@ -1,12 +0,0 @@ -import "date" -import "strings" - -result = from(bucket: "poseidon") - |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) - |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => r["event_type"] == "creation") - |> group(columns: ["environment_id", "stage"], mode:"by") - |> aggregateWindow(every: 1m, fn: count, createEmpty: true) - |> keep(columns: ["_value", "environment_id", "stage"]) - |> mean() diff --git a/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux b/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux deleted file mode 100644 index 62dce1b..0000000 --- a/deploy/grafana-dashboard/queries/executions-per-runner-hist.flux +++ /dev/null @@ -1,21 +0,0 @@ -import "strings" - -data = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - -runner_deletions = data - |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") - |> filter(fn: (r) => r["event_type"] == "deletion") - |> keep(columns: ["_time", "id", "stage"]) - |> rename(columns: {id: "runner_id"}) - -executions = data - |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_executions" or r["_measurement"] == "poseidon_aws_executions") - |> filter(fn: (r) => r["event_type"] == "creation") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_value", "environment_id", "runner_id"]) - |> count() - -result = join(tables: {key1: executions, key2: runner_deletions}, on: ["runner_id"], method: "inner") - |> keep(columns: ["_value", "_time", "environment_id", "stage"]) - |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/executions-per-runner.flux b/deploy/grafana-dashboard/queries/executions-per-runner.flux deleted file mode 100644 index 496908f..0000000 --- a/deploy/grafana-dashboard/queries/executions-per-runner.flux +++ /dev/null @@ -1,21 +0,0 @@ -import "strings" - -data = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - -runner_deletions = data - |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") - |> filter(fn: (r) => r["event_type"] == "deletion") - |> keep(columns: ["id", "stage"]) - |> rename(columns: {id: "runner_id"}) - -executions = data - |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_executions" or r["_measurement"] == "poseidon_aws_executions") - |> filter(fn: (r) => r["event_type"] == "creation") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_value", "environment_id", "runner_id"]) - |> count() - -result = join(tables: {key1: executions, key2: runner_deletions}, on: ["runner_id"], method: "inner") - |> keep(columns: ["_value", "environment_id", "stage"]) - |> mean() diff --git a/deploy/grafana-dashboard/queries/file-download-ratio.flux b/deploy/grafana-dashboard/queries/file-download-ratio.flux deleted file mode 100644 index e053f88..0000000 --- a/deploy/grafana-dashboard/queries/file-download-ratio.flux +++ /dev/null @@ -1,12 +0,0 @@ -import "strings" - -myWindowPeriod = if int(v: v.windowPeriod) > int(v: 1m) then duration(v: int(v: v.windowPeriod) * 10) else duration(v: int(v: v.windowPeriod) * 5) -data = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - -actual = data |> filter(fn: (r) => r["_field"] == "actual_length") -expected = data |> filter(fn: (r) => r["_field"] == "expected_length") -result = join(tables: {key1: actual, key2: expected}, on: ["_time", "environment_id", "runner_id", "stage"], method: "inner") - |> map(fn: (r) => ({ _value: if r._value_key2 == 0 then 1.0 else float(v: r._value_key1) / float(v: r._value_key2), environment_id: r.environment_id, runner_id: r.runner_id, stage: r.stage })) diff --git a/deploy/grafana-dashboard/queries/file-download.flux b/deploy/grafana-dashboard/queries/file-download.flux deleted file mode 100644 index 347e2cb..0000000 --- a/deploy/grafana-dashboard/queries/file-download.flux +++ /dev/null @@ -1,9 +0,0 @@ -import "strings" - -myWindowPeriod = if int(v: v.windowPeriod) > int(v: 1m) then duration(v: int(v: v.windowPeriod) * 100) else duration(v: int(v: v.windowPeriod) * 5) -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_file_download") - |> filter(fn: (r) => r["_field"] == "actual_length") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_time", "_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/file-upload.flux b/deploy/grafana-dashboard/queries/file-upload.flux deleted file mode 100644 index 983c2e5..0000000 --- a/deploy/grafana-dashboard/queries/file-upload.flux +++ /dev/null @@ -1,9 +0,0 @@ -import "strings" - -myWindowPeriod = if int(v: v.windowPeriod) > int(v: 1m) then duration(v: int(v: v.windowPeriod) * 10) else duration(v: int(v: v.windowPeriod) * 5) -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_field"] == "request_size") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_time", "_value", "environment_id", "stage"]) - |> aggregateWindow(every: myWindowPeriod, fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/idle-runner.flux b/deploy/grafana-dashboard/queries/idle-runner.flux deleted file mode 100644 index fd559f1..0000000 --- a/deploy/grafana-dashboard/queries/idle-runner.flux +++ /dev/null @@ -1,9 +0,0 @@ -import "strings" - -myWindowPeriod = if int(v: v.windowPeriod) >= int(v: 30s) then duration(v: int(v: v.windowPeriod) * 5) else v.windowPeriod -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners" and r["_field"] == "count") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_value", "_time", "environment_id", "stage"]) - |> aggregateWindow(every: myWindowPeriod, fn: min, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/number-of-executions.flux b/deploy/grafana-dashboard/queries/number-of-executions.flux deleted file mode 100644 index f0babbc..0000000 --- a/deploy/grafana-dashboard/queries/number-of-executions.flux +++ /dev/null @@ -1,10 +0,0 @@ -import "strings" - -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_aws_executions" or r["_measurement"] == "poseidon_nomad_executions") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => r["event_type"] == "creation") - |> group(columns: ["environment_id", "stage"], mode:"by") - |> count() - |> keep(columns: ["_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/prewarming-pool-size.flux b/deploy/grafana-dashboard/queries/prewarming-pool-size.flux deleted file mode 100644 index 37da6a7..0000000 --- a/deploy/grafana-dashboard/queries/prewarming-pool-size.flux +++ /dev/null @@ -1,9 +0,0 @@ -import "strings" - -result = from(bucket: "poseidon") - |> range(start: -1y) - |> filter(fn: (r) => r["_measurement"] == "poseidon_poolsize") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> group(columns: ["environment_id", "stage"], mode:"by") - |> last() - |> keep(columns: ["_value", "environment_id", "stage"]) diff --git a/deploy/grafana-dashboard/queries/request-latency.flux b/deploy/grafana-dashboard/queries/request-latency.flux deleted file mode 100644 index 7c89f7d..0000000 --- a/deploy/grafana-dashboard/queries/request-latency.flux +++ /dev/null @@ -1,6 +0,0 @@ -from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_field"] == "duration") - |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_time", "_value"]) - |> aggregateWindow(every: v.windowPeriod, fn: mean) diff --git a/deploy/grafana-dashboard/queries/requests-per-minute.flux b/deploy/grafana-dashboard/queries/requests-per-minute.flux deleted file mode 100644 index 9c2eff6..0000000 --- a/deploy/grafana-dashboard/queries/requests-per-minute.flux +++ /dev/null @@ -1,16 +0,0 @@ -import "date" - -data = from(bucket: "poseidon") - |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) - |> filter(fn: (r) => r._field == "duration") - |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_time", "_value", "status"]) - -all = data |> set(key: "status", value: "all") - -result = union(tables: [data, all]) - |> aggregateWindow(every: 1m, fn: count, createEmpty: true) - -if int(v: v.windowPeriod) > int(v: 1m) - then result |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 2), fn: mean, createEmpty: true) - else result |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 5), fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/runner-per-minute.flux b/deploy/grafana-dashboard/queries/runner-per-minute.flux deleted file mode 100644 index b6ba5ad..0000000 --- a/deploy/grafana-dashboard/queries/runner-per-minute.flux +++ /dev/null @@ -1,13 +0,0 @@ -import "strings" -import "date" - -myWindowPeriod = if int(v: v.windowPeriod) > int(v: 2m) then duration(v: int(v: v.windowPeriod) * 30) else duration(v: int(v: v.windowPeriod) * 15) -result = from(bucket: "poseidon") - |> range(start: date.truncate(t: v.timeRangeStart, unit: 1m), stop: date.truncate(t: v.timeRangeStop, unit: 1m)) - |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> filter(fn: (r) => r["event_type"] == "creation") - |> group(columns: ["environment_id", "stage"], mode:"by") - |> aggregateWindow(every: 1m, fn: count, createEmpty: true) - |> keep(columns: ["_value", "_time", "environment_id", "stage"]) - |> aggregateWindow(every: myWindowPeriod, fn: mean, createEmpty: true) diff --git a/deploy/grafana-dashboard/queries/runner-startup-duration.flux b/deploy/grafana-dashboard/queries/runner-startup-duration.flux deleted file mode 100644 index f7c0b70..0000000 --- a/deploy/grafana-dashboard/queries/runner-startup-duration.flux +++ /dev/null @@ -1,9 +0,0 @@ -import "strings" - -result = from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_nomad_idle_runners") - |> filter(fn: (r) => r["_field"] == "startup_duration") - |> filter(fn: (r) => contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_value", "_time", "environment_id", "stage"]) - |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/queries/service-time.flux b/deploy/grafana-dashboard/queries/service-time.flux deleted file mode 100644 index b53c50f..0000000 --- a/deploy/grafana-dashboard/queries/service-time.flux +++ /dev/null @@ -1,6 +0,0 @@ -from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_field"] == "duration") - |> filter(fn: (r) => (not exists r.environment_id) or contains(value: r["environment_id"], set: ${environment_ids:json})) - |> keep(columns: ["_time", "_value", "_measurement"]) - |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 10), fn: (tables=<-, column) => tables |> quantile(q: 0.999)) diff --git a/deploy/grafana-dashboard/queries/used-runner.flux b/deploy/grafana-dashboard/queries/used-runner.flux deleted file mode 100644 index bf89139..0000000 --- a/deploy/grafana-dashboard/queries/used-runner.flux +++ /dev/null @@ -1,7 +0,0 @@ -from(bucket: "poseidon") - |> range(start: v.timeRangeStart, stop: v.timeRangeStop) - |> filter(fn: (r) => r["_measurement"] == "poseidon_used_runners") - |> filter(fn: (r) => r["_field"] == "count") - |> group(columns: ["stage"], mode:"by") - |> keep(columns: ["_value", "_time", "stage"]) - |> aggregateWindow(every: duration(v: int(v: v.windowPeriod) * 5), fn: mean, createEmpty: false) diff --git a/deploy/grafana-dashboard/utils/color_mapping.py b/deploy/grafana-dashboard/utils/color_mapping.py deleted file mode 100644 index 8e01da2..0000000 --- a/deploy/grafana-dashboard/utils/color_mapping.py +++ /dev/null @@ -1,41 +0,0 @@ -from utils.utils import deep_update_dict -import json - - -def color_mapping(name, color): - return { - "fieldConfig": { - "overrides": [{ - "matcher": { - "id": "byName", - "options": name - }, - "properties": [{ - "id": "color", - "value": { - "fixedColor": color, - "mode": "fixed" - } - }] - }] - } - } - - -grey_all_mapping = color_mapping("all", "#4c4b5a") -color_mapping_environments = {} -colours = [ - "#E41A1C", "#377EB8", "#4DAF4A", "#984EA3", "#FF7F00", "#FFFF33", "#A65628", "#F781BF", "#999999", # R ColorBrewer Set1 - "#8DD3C7", "#FFFFB3", "#BEBADA", "#FB8072", "#80B1D3", "#FDB462", "#B3DE69", "#FCCDE5", "#D9D9D9", "#BC80BD", "#CCEBC5", "#FFED6F" # R ColorBrewer Set3 -] - -with open("environments.json") as f: - environments = json.load(f) - - environment_identifier = [] - for environment in environments["executionEnvironments"]: - environment_identifier.append(str(environment["id"]) + "/" + environment["image"].removeprefix("openhpi/co_execenv_")) - - environment_identifier.sort() - for environment in environment_identifier: - deep_update_dict(color_mapping_environments, color_mapping(environment, colours.pop(0))) diff --git a/deploy/grafana-dashboard/utils/utils.py b/deploy/grafana-dashboard/utils/utils.py deleted file mode 100644 index 0062e34..0000000 --- a/deploy/grafana-dashboard/utils/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -def read_query(*names): - result = "" - for name in names: - with open("queries/" + name + ".flux", "r") as file: - result += file.read() - result += "\n" - return result - - -def deep_update_dict(base_dict, extra_dict): - if extra_dict is None: - return base_dict - - for k, v in extra_dict.items(): - update_dict_entry(base_dict, k, v) - - -def update_dict_entry(base_dict, k, v): - if k in base_dict and hasattr(base_dict[k], "to_json_data"): - base_dict[k] = base_dict[k].to_json_data() - if k in base_dict and isinstance(base_dict[k], dict): - deep_update_dict(base_dict[k], v) - elif k in base_dict and isinstance(base_dict[k], list): - base_dict[k].extend(v) - else: - base_dict[k] = v diff --git a/deploy/grafana-dashboard/utils/variables.py b/deploy/grafana-dashboard/utils/variables.py deleted file mode 100644 index 45f9c10..0000000 --- a/deploy/grafana-dashboard/utils/variables.py +++ /dev/null @@ -1,14 +0,0 @@ -from grafanalib.core import Template - -from utils.utils import read_query - -environment_variable = Template( - dataSource="Flux", - label="Environment IDs", - name="environment_ids", - query=read_query("environment-ids"), - refresh=1, - includeAll=True, - multi=True, - default="$__all", -) diff --git a/deploy/nomad-ci/Dockerfile b/deploy/nomad-ci/Dockerfile deleted file mode 100644 index 543ce77..0000000 --- a/deploy/nomad-ci/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Simple image containing the Nomad binary to deploy Nomad jobs - -FROM golang:latest - -# Install prerequisites, gettext contains envsubst used in the CI -RUN apt-get update && \ - apt install -y \ - unzip \ - wget \ - gettext \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg \ - lsb-release && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists - -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update && \ - apt-get install -y docker-ce docker-ce-cli containerd.io && \ - rm -rf /var/lib/apt/lists - -ENV NOMAD_VERSION="1.1.2" - -# Download Nomad -RUN wget "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip" && \ - wget "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_SHA256SUMS" && \ - grep "nomad_${NOMAD_VERSION}_linux_amd64.zip" nomad_${NOMAD_VERSION}_SHA256SUMS | sha256sum -c - && \ - unzip nomad_${NOMAD_VERSION}_linux_amd64.zip - -# Install Nomad -RUN mv nomad /usr/sbin/ && nomad -version diff --git a/deploy/nomad-ci/README.md b/deploy/nomad-ci/README.md deleted file mode 100644 index 8bcced1..0000000 --- a/deploy/nomad-ci/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Nomad-in-Docker Image - -The [`Dockerfile`](Dockerfile) in this folder creates a Docker image that contains Docker and Nomad. - -Running the image requires the following Docker options: - -- Allow Nomad to use mount: `--cap-add=SYS_ADMIN` -- Allow Nomad to use bind mounts: `--security-opt apparmor=unconfined` -- Add access to Docker daemon: `-v /var/run/docker.sock:/var/run/docker.sock` -- Map port to host: `-p 4646:4646` - -A complete command to run the container is as follows. - -```shell -docker run --rm --name nomad \ - --cap-add=SYS_ADMIN \ - --security-opt apparmor=unconfined \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -p 4646:4646 \ - nomad-ci -``` diff --git a/deploy/nomad-run-env-job.sh b/deploy/nomad-run-env-job.sh deleted file mode 100755 index 01f9184..0000000 --- a/deploy/nomad-run-env-job.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# This script substitutes environment variables in the given Nomad job and runs it afterwards. - -if [[ "$#" -ne 2 ]]; then - echo "Usage: $0 path/to/infile path/to/outfile" -fi - -envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $1 > $2 -nomad validate $2 -# nomad plan returns 1 if allocations are created or destroyed which is what we want here -# https://www.nomadproject.io/docs/commands/job/plan#usage -nomad plan $2 || [ $? == 1 ] -nomad run $2 diff --git a/deploy/poseidon/Dockerfile b/deploy/poseidon/Dockerfile deleted file mode 100644 index 11777e5..0000000 --- a/deploy/poseidon/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM alpine:latest - -RUN adduser --disabled-password api -USER api -COPY poseidon /home/api/ - -EXPOSE 7200 -CMD ["/home/api/poseidon"] diff --git a/docker-compose.yml b/docker-compose.yml index 8bac884..00f481a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,8 @@ services: dockerfile: Dockerfile ports: - "7200:7200" - network_mode: host + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ./configuration.yaml:/go/src/app/configuration.yaml restart: unless-stopped diff --git a/go.mod b/go.mod index 802075d..7191248 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.5 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 - golang.org/x/sys v0.22.0 + golang.org/x/sys v0.25.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -35,14 +35,23 @@ require ( github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fatih/color v1.17.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/gojuno/minimock/v3 v3.3.13 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/hashicorp/consul/api v1.29.1 // indirect github.com/hashicorp/cronexpr v1.1.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -73,8 +82,12 @@ require ( github.com/hashicorp/serf v0.10.2-0.20240320153621-5d32001edfaa // indirect github.com/hashicorp/vault/api v1.14.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/miekg/dns v1.1.61 // indirect @@ -85,16 +98,22 @@ require ( github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/mitchellh/pointerstructure v1.2.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zclconf/go-cty v1.14.4 // indirect github.com/zclconf/go-cty-yaml v1.0.3 // indirect @@ -102,12 +121,26 @@ require ( golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect + golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/api v0.31.1 // indirect + k8s.io/apimachinery v0.31.1 // indirect + k8s.io/client-go v0.31.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect oss.indeed.com/go/libtime v1.6.0 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 140f7b6..6376c8b 100644 --- a/go.sum +++ b/go.sum @@ -31,18 +31,23 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -55,14 +60,25 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gojuno/minimock/v3 v3.0.4/go.mod h1:HqeqnwV8mAABn3pO5hqF+RE7gjA0jsN8cbbSogoGrzI= github.com/gojuno/minimock/v3 v3.0.6/go.mod h1:v61ZjAKHr+WnEkND63nQPCZ/DTfQgJdvbCi3IuoMblY= github.com/gojuno/minimock/v3 v3.3.13 h1:sXFO7RbB4JnZiKhgMO4BU4RLYcfhcOSepfiv4wPgGNY= @@ -75,12 +91,16 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -170,19 +190,28 @@ github.com/hashicorp/vault/api v1.14.0/go.mod h1:pV9YLxBGSz+cItFDd8Ii4G17waWOQ32 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hexdigest/gowrap v1.1.7/go.mod h1:Z+nBFUDLa01iaNM+/jzoOA1JJ7sm51rnYFauKFUB5fs= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/influxdata/influxdb-client-go/v2 v2.13.0 h1:ioBbLmR5NMbAjP4UVA5r9b5xGjpABD7j65pI8kFphDM= github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0sr5D8LolXHqAAOfPw9v/RIRHl4= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -191,6 +220,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -224,9 +255,14 @@ github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= @@ -273,9 +309,13 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -283,7 +323,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= @@ -292,6 +335,10 @@ github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYg github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -307,7 +354,9 @@ go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= @@ -315,6 +364,8 @@ golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5D golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= @@ -324,17 +375,22 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -351,6 +407,7 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -368,11 +425,15 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -389,12 +450,16 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= @@ -408,12 +473,38 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= +k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= +k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oss.indeed.com/go/libtime v1.6.0 h1:XQyczJihse/wQGo59OfPF3f4f+Sywv4R8vdGB3S9BfU= oss.indeed.com/go/libtime v1.6.0/go.mod h1:B2sdEcuzB0zhTKkAuHy4JInKRc7Al3tME4qWam6R7mA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/api/api_test.go b/internal/api/api_test.go deleted file mode 100644 index 38f0f65..0000000 --- a/internal/api/api_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package api - -import ( - "github.com/gorilla/mux" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/environment" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "net/http" - "net/http/httptest" - "testing" -) - -func mockHTTPHandler(writer http.ResponseWriter, _ *http.Request) { - writer.WriteHeader(http.StatusOK) -} - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestNewRouterV1WithAuthenticationDisabled() { - config.Config.Server.Token = "" - router := mux.NewRouter() - m := &environment.ManagerHandlerMock{} - m.On("Statistics").Return(make(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData)) - configureV1Router(router, nil, m) - - s.Run("health route is accessible", func() { - request, err := http.NewRequest(http.MethodGet, "/api/v1/health", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - s.Equal(http.StatusNoContent, recorder.Code) - }) - - s.Run("added route is accessible", func() { - router.HandleFunc("/api/v1/test", mockHTTPHandler) - request, err := http.NewRequest(http.MethodGet, "/api/v1/test", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - s.Equal(http.StatusOK, recorder.Code) - }) -} - -func (s *MainTestSuite) TestNewRouterV1WithAuthenticationEnabled() { - config.Config.Server.Token = "TestToken" - router := mux.NewRouter() - m := &environment.ManagerHandlerMock{} - m.On("Statistics").Return(make(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData)) - configureV1Router(router, nil, m) - - s.Run("health route is accessible", func() { - request, err := http.NewRequest(http.MethodGet, "/api/v1/health", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - s.Equal(http.StatusNoContent, recorder.Code) - }) - - s.Run("protected route is not accessible", func() { - // request an available API route that should be guarded by authentication. - // (which one, in particular, does not matter here) - request, err := http.NewRequest(http.MethodPost, "/api/v1/runners", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - s.Equal(http.StatusUnauthorized, recorder.Code) - }) - config.Config.Server.Token = "" -} diff --git a/internal/api/auth/auth_test.go b/internal/api/auth/auth_test.go deleted file mode 100644 index 206d96c..0000000 --- a/internal/api/auth/auth_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package auth - -import ( - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/tests" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "net/http" - "net/http/httptest" - "testing" -) - -const testToken = "C0rr3ctT0k3n" - -type AuthenticationMiddlewareTestSuite struct { - tests.MemoryLeakTestSuite - request *http.Request - recorder *httptest.ResponseRecorder - httpAuthenticationMiddleware http.Handler -} - -func (s *AuthenticationMiddlewareTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - correctAuthenticationToken = []byte(testToken) - s.recorder = httptest.NewRecorder() - request, err := http.NewRequest(http.MethodGet, "/api/v1/test", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - s.request = request - s.httpAuthenticationMiddleware = HTTPAuthenticationMiddleware( - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) -} - -func (s *AuthenticationMiddlewareTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - correctAuthenticationToken = []byte(nil) -} - -func (s *AuthenticationMiddlewareTestSuite) TestReturns401WhenHeaderUnset() { - s.httpAuthenticationMiddleware.ServeHTTP(s.recorder, s.request) - assert.Equal(s.T(), http.StatusUnauthorized, s.recorder.Code) -} - -func (s *AuthenticationMiddlewareTestSuite) TestReturns401WhenTokenWrong() { - s.request.Header.Set(TokenHeader, "Wr0ngT0k3n") - s.httpAuthenticationMiddleware.ServeHTTP(s.recorder, s.request) - assert.Equal(s.T(), http.StatusUnauthorized, s.recorder.Code) -} - -func (s *AuthenticationMiddlewareTestSuite) TestWarnsWhenUnauthorized() { - var hook *test.Hook - logger, hook := test.NewNullLogger() - log = logger.WithField("pkg", "api/auth") - - s.request.Header.Set(TokenHeader, "Wr0ngT0k3n") - s.httpAuthenticationMiddleware.ServeHTTP(s.recorder, s.request) - - assert.Equal(s.T(), http.StatusUnauthorized, s.recorder.Code) - assert.Equal(s.T(), logrus.WarnLevel, hook.LastEntry().Level) - assert.Equal(s.T(), hook.LastEntry().Data["token"], "Wr0ngT0k3n") -} - -func (s *AuthenticationMiddlewareTestSuite) TestPassesWhenTokenCorrect() { - s.request.Header.Set(TokenHeader, testToken) - s.httpAuthenticationMiddleware.ServeHTTP(s.recorder, s.request) - - assert.Equal(s.T(), http.StatusOK, s.recorder.Code) -} - -func TestHTTPAuthenticationMiddleware(t *testing.T) { - suite.Run(t, new(AuthenticationMiddlewareTestSuite)) -} - -func TestInitializeAuthentication(t *testing.T) { - t.Run("if token unset", func(t *testing.T) { - config.Config.Server.Token = "" - initialized := InitializeAuthentication() - assert.Equal(t, false, initialized) - assert.Equal(t, []byte(nil), correctAuthenticationToken, "it should not set correctAuthenticationToken") - }) - t.Run("if token set", func(t *testing.T) { - config.Config.Server.Token = testToken - initialized := InitializeAuthentication() - assert.Equal(t, true, initialized) - assert.Equal(t, []byte(testToken), correctAuthenticationToken, "it should set correctAuthenticationToken") - config.Config.Server.Token = "" - correctAuthenticationToken = []byte(nil) - }) -} diff --git a/internal/api/environments_test.go b/internal/api/environments_test.go deleted file mode 100644 index d49d392..0000000 --- a/internal/api/environments_test.go +++ /dev/null @@ -1,308 +0,0 @@ -package api - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/gorilla/mux" - "github.com/openHPI/poseidon/internal/environment" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "math" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "testing" -) - -const jobHCLBasicFormat = "job \"%s\" {}" - -type EnvironmentControllerTestSuite struct { - tests.MemoryLeakTestSuite - manager *environment.ManagerHandlerMock - router *mux.Router -} - -func TestEnvironmentControllerTestSuite(t *testing.T) { - suite.Run(t, new(EnvironmentControllerTestSuite)) -} - -func (s *EnvironmentControllerTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.manager = &environment.ManagerHandlerMock{} - s.router = NewRouter(nil, s.manager) -} - -func (s *EnvironmentControllerTestSuite) TestList() { - call := s.manager.On("List", mock.AnythingOfType("bool")) - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]runner.ExecutionEnvironment{}, nil} - }) - path, err := s.router.Get(listRouteName).URL() - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodGet, path.String(), http.NoBody) - s.Require().NoError(err) - - s.Run("with no Environments", func() { - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusOK, recorder.Code) - - var environmentsResponse ExecutionEnvironmentsResponse - err = json.NewDecoder(recorder.Result().Body).Decode(&environmentsResponse) - s.Require().NoError(err) - _ = recorder.Result().Body.Close() - - s.Empty(environmentsResponse.ExecutionEnvironments) - }) - s.manager.Calls = []mock.Call{} - - s.Run("with fetch", func() { - recorder := httptest.NewRecorder() - query := path.Query() - query.Set("fetch", "true") - path.RawQuery = query.Encode() - request, err := http.NewRequest(http.MethodGet, path.String(), http.NoBody) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusOK, recorder.Code) - s.manager.AssertCalled(s.T(), "List", true) - }) - s.manager.Calls = []mock.Call{} - - s.Run("with bad fetch", func() { - recorder := httptest.NewRecorder() - query := path.Query() - query.Set("fetch", "YouDecide") - path.RawQuery = query.Encode() - request, err := http.NewRequest(http.MethodGet, path.String(), http.NoBody) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusBadRequest, recorder.Code) - s.manager.AssertNotCalled(s.T(), "List") - }) - - s.Run("returns multiple environments", func() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - var firstEnvironment, secondEnvironment *environment.NomadEnvironment - call.Run(func(args mock.Arguments) { - firstEnvironment, err = environment.NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, - fmt.Sprintf(jobHCLBasicFormat, nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger))) - s.Require().NoError(err) - secondEnvironment, err = environment.NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, - fmt.Sprintf(jobHCLBasicFormat, nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger))) - s.Require().NoError(err) - call.ReturnArguments = mock.Arguments{[]runner.ExecutionEnvironment{firstEnvironment, secondEnvironment}, nil} - }) - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusOK, recorder.Code) - - paramMap := make(map[string]interface{}) - err := json.NewDecoder(recorder.Result().Body).Decode(¶mMap) - s.Require().NoError(err) - environmentsInterface, ok := paramMap["executionEnvironments"] - s.Require().True(ok) - environments, ok := environmentsInterface.([]interface{}) - s.Require().True(ok) - s.Equal(2, len(environments)) - - err = firstEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.NoError(err) - err = secondEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.NoError(err) - }) -} - -func (s *EnvironmentControllerTestSuite) TestGet() { - call := s.manager.On("Get", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("bool")) - path, err := s.router.Get(getRouteName).URL(executionEnvironmentIDKey, tests.DefaultEnvironmentIDAsString) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodGet, path.String(), http.NoBody) - s.Require().NoError(err) - - s.Run("with unknown environment", func() { - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{nil, runner.ErrUnknownExecutionEnvironment} - }) - - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusNotFound, recorder.Code) - s.manager.AssertCalled(s.T(), "Get", dto.EnvironmentID(0), false) - }) - s.manager.Calls = []mock.Call{} - - s.Run("not found with fetch", func() { - recorder := httptest.NewRecorder() - query := path.Query() - query.Set("fetch", "true") - path.RawQuery = query.Encode() - request, err := http.NewRequest(http.MethodGet, path.String(), http.NoBody) - s.Require().NoError(err) - - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{nil, runner.ErrUnknownExecutionEnvironment} - }) - - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusNotFound, recorder.Code) - s.manager.AssertCalled(s.T(), "Get", dto.EnvironmentID(0), true) - }) - s.manager.Calls = []mock.Call{} - - s.Run("returns environment", func() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - var testEnvironment *environment.NomadEnvironment - call.Run(func(args mock.Arguments) { - testEnvironment, err = environment.NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, - fmt.Sprintf(jobHCLBasicFormat, nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger))) - s.Require().NoError(err) - call.ReturnArguments = mock.Arguments{testEnvironment, nil} - }) - - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusOK, recorder.Code) - - var environmentParams map[string]interface{} - err := json.NewDecoder(recorder.Result().Body).Decode(&environmentParams) - s.Require().NoError(err) - idInterface, ok := environmentParams["id"] - s.Require().True(ok) - idFloat, ok := idInterface.(float64) - s.Require().True(ok) - s.Equal(tests.DefaultEnvironmentIDAsInteger, int(idFloat)) - - err = testEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.NoError(err) - }) -} - -func (s *EnvironmentControllerTestSuite) TestDelete() { - call := s.manager.On("Delete", mock.AnythingOfType("dto.EnvironmentID")) - path, err := s.router.Get(deleteRouteName).URL(executionEnvironmentIDKey, tests.DefaultEnvironmentIDAsString) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodDelete, path.String(), http.NoBody) - s.Require().NoError(err) - - s.Run("environment not found", func() { - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{false, nil} - }) - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusNotFound, recorder.Code) - }) - - s.Run("environment deleted", func() { - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{true, nil} - }) - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusNoContent, recorder.Code) - }) - - s.manager.Calls = []mock.Call{} - s.Run("with bad environment id", func() { - _, err := s.router.Get(deleteRouteName).URL(executionEnvironmentIDKey, "MagicNonNumberID") - s.Error(err) - }) -} - -type CreateOrUpdateEnvironmentTestSuite struct { - EnvironmentControllerTestSuite - path string - id dto.EnvironmentID - body []byte -} - -func TestCreateOrUpdateEnvironmentTestSuite(t *testing.T) { - suite.Run(t, new(CreateOrUpdateEnvironmentTestSuite)) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) SetupTest() { - s.EnvironmentControllerTestSuite.SetupTest() - s.id = tests.DefaultEnvironmentIDAsInteger - testURL, err := s.router.Get(createOrUpdateRouteName).URL(executionEnvironmentIDKey, strconv.Itoa(int(s.id))) - if err != nil { - s.T().Fatal(err) - } - s.path = testURL.String() - s.body, err = json.Marshal(dto.ExecutionEnvironmentRequest{}) - if err != nil { - s.T().Fatal(err) - } -} - -func (s *CreateOrUpdateEnvironmentTestSuite) recordRequest() *httptest.ResponseRecorder { - recorder := httptest.NewRecorder() - request, err := http.NewRequest(http.MethodPut, s.path, bytes.NewReader(s.body)) - if err != nil { - s.T().Fatal(err) - } - s.router.ServeHTTP(recorder, request) - return recorder -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestReturnsBadRequestWhenBadBody() { - s.body = []byte{} - recorder := s.recordRequest() - s.Equal(http.StatusBadRequest, recorder.Code) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestReturnsInternalServerErrorWhenManagerReturnsError() { - testError := tests.ErrDefault - s.manager. - On("CreateOrUpdate", s.id, mock.AnythingOfType("dto.ExecutionEnvironmentRequest"), mock.Anything). - Return(false, testError) - - recorder := s.recordRequest() - s.Equal(http.StatusInternalServerError, recorder.Code) - s.Contains(recorder.Body.String(), testError.Error()) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestReturnsCreatedIfNewEnvironment() { - s.manager. - On("CreateOrUpdate", s.id, mock.AnythingOfType("dto.ExecutionEnvironmentRequest"), mock.Anything). - Return(true, nil) - - recorder := s.recordRequest() - s.Equal(http.StatusCreated, recorder.Code) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestReturnsNoContentIfNotNewEnvironment() { - s.manager. - On("CreateOrUpdate", s.id, mock.AnythingOfType("dto.ExecutionEnvironmentRequest"), mock.Anything). - Return(false, nil) - - recorder := s.recordRequest() - s.Equal(http.StatusNoContent, recorder.Code) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestReturnsNotFoundOnNonIntegerID() { - s.path = strings.Join([]string{BasePath, EnvironmentsPath, "/", "invalid-id"}, "") - recorder := s.recordRequest() - s.Equal(http.StatusNotFound, recorder.Code) -} - -func (s *CreateOrUpdateEnvironmentTestSuite) TestFailsOnTooLargeID() { - tooLargeIntStr := strconv.Itoa(math.MaxInt64) + "0" - s.path = strings.Join([]string{BasePath, EnvironmentsPath, "/", tooLargeIntStr}, "") - recorder := s.recordRequest() - s.Equal(http.StatusBadRequest, recorder.Code) -} diff --git a/internal/api/health_test.go b/internal/api/health_test.go deleted file mode 100644 index 89e2b3a..0000000 --- a/internal/api/health_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package api - -import ( - "encoding/json" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/environment" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "io" - "net/http" - "net/http/httptest" -) - -func (s *MainTestSuite) TestHealth() { - s.Run("returns StatusNoContent as default", func() { - request, err := http.NewRequest(http.MethodGet, "/health", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - manager := &environment.ManagerHandlerMock{} - manager.On("Statistics").Return(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData{}) - - Health(manager).ServeHTTP(recorder, request) - s.Equal(http.StatusNoContent, recorder.Code) - }) - s.Run("returns InternalServerError for warnings and errors", func() { - s.Run("Prewarming Pool Alert", func() { - request, err := http.NewRequest(http.MethodGet, "/health", http.NoBody) - if err != nil { - s.T().Fatal(err) - } - recorder := httptest.NewRecorder() - manager := &environment.ManagerHandlerMock{} - manager.On("Statistics").Return(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData{ - tests.DefaultEnvironmentIDAsInteger: { - ID: tests.DefaultEnvironmentIDAsInteger, - PrewarmingPoolSize: 3, - IdleRunners: 1, - }, - }) - config.Config.Server.Alert.PrewarmingPoolThreshold = 0.5 - - Health(manager).ServeHTTP(recorder, request) - s.Equal(http.StatusServiceUnavailable, recorder.Code) - - b, err := io.ReadAll(recorder.Body) - s.Require().NoError(err) - var details dto.InternalServerError - err = json.Unmarshal(b, &details) - s.Require().NoError(err) - s.Contains(details.Message, ErrorPrewarmingPoolDepleting.Error()) - }) - }) -} diff --git a/internal/api/runners_test.go b/internal/api/runners_test.go deleted file mode 100644 index 613ab18..0000000 --- a/internal/api/runners_test.go +++ /dev/null @@ -1,501 +0,0 @@ -package api - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/gorilla/mux" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/monitoring" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "net/http" - "net/http/httptest" - "net/url" - "strconv" - "strings" - "testing" -) - -const invalidID = "some-invalid-runner-id" - -type MiddlewareTestSuite struct { - tests.MemoryLeakTestSuite - manager *runner.ManagerMock - router *mux.Router - runner runner.Runner - capturedRunner runner.Runner - runnerRequest func(string) *http.Request -} - -func (s *MiddlewareTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.manager = &runner.ManagerMock{} - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.runner = runner.NewNomadJob(tests.DefaultRunnerID, nil, apiMock, nil) - s.capturedRunner = nil - s.runnerRequest = func(runnerId string) *http.Request { - path, err := s.router.Get("test-runner-id").URL(RunnerIDKey, runnerId) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPost, path.String(), http.NoBody) - s.Require().NoError(err) - return request - } - runnerRouteHandler := func(writer http.ResponseWriter, request *http.Request) { - var ok bool - s.capturedRunner, ok = runner.FromContext(request.Context()) - if ok { - writer.WriteHeader(http.StatusOK) - } else { - writer.WriteHeader(http.StatusInternalServerError) - } - } - s.router = mux.NewRouter() - runnerController := &RunnerController{s.manager, s.router} - s.router.Use(monitoring.InfluxDB2Middleware) - s.router.Use(runnerController.findRunnerMiddleware) - s.router.HandleFunc(fmt.Sprintf("/test/{%s}", RunnerIDKey), runnerRouteHandler).Name("test-runner-id") -} - -func (s *MiddlewareTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - err := s.runner.Destroy(nil) - s.Require().NoError(err) -} - -func TestMiddlewareTestSuite(t *testing.T) { - suite.Run(t, new(MiddlewareTestSuite)) -} - -func (s *MiddlewareTestSuite) TestFindRunnerMiddlewareIfRunnerExists() { - s.manager.On("Get", s.runner.ID()).Return(s.runner, nil) - - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, s.runnerRequest(s.runner.ID())) - - s.Equal(http.StatusOK, recorder.Code) - s.Equal(s.runner, s.capturedRunner) -} - -func (s *MiddlewareTestSuite) TestFindRunnerMiddlewareIfRunnerDoesNotExist() { - s.manager.On("Get", invalidID).Return(nil, runner.ErrRunnerNotFound) - - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, s.runnerRequest(invalidID)) - - s.Equal(http.StatusGone, recorder.Code) -} - -func (s *MiddlewareTestSuite) TestFindRunnerMiddlewareDoesNotEarlyRespond() { - body := strings.NewReader(strings.Repeat("A", 798968)) - - path, err := s.router.Get("test-runner-id").URL(RunnerIDKey, invalidID) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPost, path.String(), body) - s.Require().NoError(err) - - s.manager.On("Get", mock.AnythingOfType("string")).Return(nil, runner.ErrRunnerNotFound) - recorder := httptest.NewRecorder() - s.router.ServeHTTP(recorder, request) - - s.Equal(http.StatusGone, recorder.Code) - s.Equal(0, body.Len()) // No data should be unread -} - -func TestRunnerRouteTestSuite(t *testing.T) { - suite.Run(t, new(RunnerRouteTestSuite)) -} - -type RunnerRouteTestSuite struct { - tests.MemoryLeakTestSuite - runnerManager *runner.ManagerMock - router *mux.Router - runner runner.Runner - executionID string -} - -func (s *RunnerRouteTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.runnerManager = &runner.ManagerMock{} - s.router = NewRouter(s.runnerManager, nil) - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.runner = runner.NewNomadJob("some-id", nil, apiMock, func(_ runner.Runner) error { return nil }) - s.executionID = "execution" - s.runner.StoreExecution(s.executionID, &dto.ExecutionRequest{}) - s.runnerManager.On("Get", s.runner.ID()).Return(s.runner, nil) -} - -func (s *RunnerRouteTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - s.Require().NoError(s.runner.Destroy(nil)) -} - -func TestProvideRunnerTestSuite(t *testing.T) { - suite.Run(t, new(ProvideRunnerTestSuite)) -} - -type ProvideRunnerTestSuite struct { - RunnerRouteTestSuite - defaultRequest *http.Request - path string -} - -func (s *ProvideRunnerTestSuite) SetupTest() { - s.RunnerRouteTestSuite.SetupTest() - - path, err := s.router.Get(ProvideRoute).URL() - s.Require().NoError(err) - s.path = path.String() - - runnerRequest := dto.RunnerRequest{ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger} - body, err := json.Marshal(runnerRequest) - s.Require().NoError(err) - s.defaultRequest, err = http.NewRequest(http.MethodPost, s.path, bytes.NewReader(body)) - s.Require().NoError(err) -} - -func (s *ProvideRunnerTestSuite) TestValidRequestReturnsRunner() { - s.runnerManager. - On("Claim", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("int")). - Return(s.runner, nil) - recorder := httptest.NewRecorder() - - s.router.ServeHTTP(recorder, s.defaultRequest) - s.Equal(http.StatusOK, recorder.Code) - - s.Run("response contains runnerId", func() { - var runnerResponse dto.RunnerResponse - err := json.NewDecoder(recorder.Result().Body).Decode(&runnerResponse) - s.Require().NoError(err) - _ = recorder.Result().Body.Close() - s.Equal(s.runner.ID(), runnerResponse.ID) - }) -} - -func (s *ProvideRunnerTestSuite) TestInvalidRequestReturnsBadRequest() { - badRequest, err := http.NewRequest(http.MethodPost, s.path, strings.NewReader("")) - s.Require().NoError(err) - recorder := httptest.NewRecorder() - - s.router.ServeHTTP(recorder, badRequest) - s.Equal(http.StatusBadRequest, recorder.Code) -} - -func (s *ProvideRunnerTestSuite) TestWhenExecutionEnvironmentDoesNotExistReturnsNotFound() { - s.runnerManager. - On("Claim", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("int")). - Return(nil, runner.ErrUnknownExecutionEnvironment) - recorder := httptest.NewRecorder() - - s.router.ServeHTTP(recorder, s.defaultRequest) - s.Equal(http.StatusNotFound, recorder.Code) -} - -func (s *ProvideRunnerTestSuite) TestWhenNoRunnerAvailableReturnsNomadOverload() { - s.runnerManager. - On("Claim", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("int")). - Return(nil, runner.ErrNoRunnersAvailable) - recorder := httptest.NewRecorder() - - s.router.ServeHTTP(recorder, s.defaultRequest) - s.Equal(http.StatusInternalServerError, recorder.Code) - - var internalServerError dto.InternalServerError - err := json.NewDecoder(recorder.Result().Body).Decode(&internalServerError) - s.Require().NoError(err) - _ = recorder.Result().Body.Close() - s.Equal(dto.ErrorNomadOverload, internalServerError.ErrorCode) -} - -func (s *RunnerRouteTestSuite) TestExecuteRoute() { - path, err := s.router.Get(ExecutePath).URL(RunnerIDKey, s.runner.ID()) - s.Require().NoError(err) - - s.Run("valid request", func() { - recorder := httptest.NewRecorder() - executionRequest := dto.ExecutionRequest{ - Command: "command", - TimeLimit: 10, - Environment: nil, - } - body, err := json.Marshal(executionRequest) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPost, path.String(), bytes.NewReader(body)) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - - var webSocketResponse dto.ExecutionResponse - err = json.NewDecoder(recorder.Result().Body).Decode(&webSocketResponse) - s.Require().NoError(err) - - s.Equal(http.StatusOK, recorder.Code) - - s.Run("creates an execution request for the runner", func() { - webSocketURL, err := url.Parse(webSocketResponse.WebSocketURL) - s.Require().NoError(err) - executionID := webSocketURL.Query().Get(ExecutionIDKey) - ok := s.runner.ExecutionExists(executionID) - - s.True(ok, "No execution request with this id: ", executionID) - }) - }) - - s.Run("invalid request", func() { - recorder := httptest.NewRecorder() - body := "" - request, err := http.NewRequest(http.MethodPost, path.String(), strings.NewReader(body)) - s.Require().NoError(err) - s.router.ServeHTTP(recorder, request) - - s.Equal(http.StatusBadRequest, recorder.Code) - }) - - s.Run("forbidden characters in command", func() { - recorder := httptest.NewRecorder() - executionRequest := dto.ExecutionRequest{ - Command: "echo 'forbidden'", - TimeLimit: 10, - } - body, err := json.Marshal(executionRequest) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPost, path.String(), bytes.NewReader(body)) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - s.Equal(http.StatusBadRequest, recorder.Code) - }) -} - -func TestUpdateFileSystemRouteTestSuite(t *testing.T) { - suite.Run(t, new(UpdateFileSystemRouteTestSuite)) -} - -type UpdateFileSystemRouteTestSuite struct { - RunnerRouteTestSuite - path string - recorder *httptest.ResponseRecorder - runnerMock *runner.RunnerMock -} - -func (s *UpdateFileSystemRouteTestSuite) SetupTest() { - s.RunnerRouteTestSuite.SetupTest() - routeURL, err := s.router.Get(UpdateFileSystemPath).URL(RunnerIDKey, tests.DefaultMockID) - s.Require().NoError(err) - s.path = routeURL.String() - s.runnerMock = &runner.RunnerMock{} - s.runnerMock.On("ID").Return(tests.DefaultMockID) - s.runnerMock.On("Environment").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - s.runnerManager.On("Get", tests.DefaultMockID).Return(s.runnerMock, nil) - s.recorder = httptest.NewRecorder() -} - -func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsNoContentOnValidRequest() { - s.runnerMock.On("UpdateFileSystem", mock.AnythingOfType("*dto.UpdateFileSystemRequest"), mock.Anything). - Return(nil) - - copyRequest := dto.UpdateFileSystemRequest{} - body, err := json.Marshal(copyRequest) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPatch, s.path, bytes.NewReader(body)) - s.Require().NoError(err) - - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusNoContent, s.recorder.Code) - s.runnerMock.AssertCalled(s.T(), "UpdateFileSystem", - mock.AnythingOfType("*dto.UpdateFileSystemRequest"), mock.Anything) -} - -func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsBadRequestOnInvalidRequestBody() { - request, err := http.NewRequest(http.MethodPatch, s.path, strings.NewReader("")) - s.Require().NoError(err) - - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusBadRequest, s.recorder.Code) -} - -func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemToNonExistingRunnerReturnsGone() { - s.runnerManager.On("Get", invalidID).Return(nil, runner.ErrRunnerNotFound) - path, err := s.router.Get(UpdateFileSystemPath).URL(RunnerIDKey, invalidID) - s.Require().NoError(err) - copyRequest := dto.UpdateFileSystemRequest{} - body, err := json.Marshal(copyRequest) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPatch, path.String(), bytes.NewReader(body)) - s.Require().NoError(err) - - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusGone, s.recorder.Code) -} - -func (s *UpdateFileSystemRouteTestSuite) TestUpdateFileSystemReturnsInternalServerErrorWhenCopyFailed() { - s.runnerMock. - On("UpdateFileSystem", mock.AnythingOfType("*dto.UpdateFileSystemRequest"), mock.Anything). - Return(runner.ErrorFileCopyFailed) - - copyRequest := dto.UpdateFileSystemRequest{} - body, err := json.Marshal(copyRequest) - s.Require().NoError(err) - request, err := http.NewRequest(http.MethodPatch, s.path, bytes.NewReader(body)) - s.Require().NoError(err) - - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusInternalServerError, s.recorder.Code) -} - -func (s *UpdateFileSystemRouteTestSuite) TestListFileSystem() { - routeURL, err := s.router.Get(UpdateFileSystemPath).URL(RunnerIDKey, tests.DefaultMockID) - s.Require().NoError(err) - mockCall := s.runnerMock.On("ListFileSystem", mock.AnythingOfType("string"), - mock.AnythingOfType("bool"), mock.Anything, mock.AnythingOfType("bool"), mock.Anything) - - s.Run("default parameters", func() { - mockCall.Run(func(args mock.Arguments) { - path, ok := args.Get(0).(string) - s.True(ok) - s.Equal("./", path) - recursive, ok := args.Get(1).(bool) - s.True(ok) - s.True(recursive) - mockCall.ReturnArguments = mock.Arguments{nil} - }) - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusOK, s.recorder.Code) - }) - - s.recorder = httptest.NewRecorder() - s.Run("passed parameters", func() { - expectedPath := "/flag" - - mockCall.Run(func(args mock.Arguments) { - path, ok := args.Get(0).(string) - s.True(ok) - s.Equal(expectedPath, path) - recursive, ok := args.Get(1).(bool) - s.True(ok) - s.False(recursive) - mockCall.ReturnArguments = mock.Arguments{nil} - }) - - query := routeURL.Query() - query.Set(PathKey, expectedPath) - query.Set(RecursiveKey, strconv.FormatBool(false)) - routeURL.RawQuery = query.Encode() - - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusOK, s.recorder.Code) - }) - - s.recorder = httptest.NewRecorder() - s.Run("Internal Server Error on failure", func() { - mockCall.Run(func(args mock.Arguments) { - mockCall.ReturnArguments = mock.Arguments{runner.ErrRunnerNotFound} - }) - - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusInternalServerError, s.recorder.Code) - }) -} - -func (s *UpdateFileSystemRouteTestSuite) TestFileContent() { - routeURL, err := s.router.Get(FileContentRawPath).URL(RunnerIDKey, tests.DefaultMockID) - s.Require().NoError(err) - mockCall := s.runnerMock.On("GetFileContent", - mock.AnythingOfType("string"), mock.Anything, mock.AnythingOfType("bool"), mock.Anything) - - s.Run("Not Found", func() { - mockCall.Return(runner.ErrFileNotFound) - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusFailedDependency, s.recorder.Code) - }) - - s.recorder = httptest.NewRecorder() - s.Run("Unknown Error", func() { - mockCall.Return(nomad.ErrorExecutorCommunicationFailed) - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusInternalServerError, s.recorder.Code) - }) - - s.recorder = httptest.NewRecorder() - s.Run("No Error", func() { - mockCall.Return(nil) - request, err := http.NewRequest(http.MethodGet, routeURL.String(), strings.NewReader("")) - s.Require().NoError(err) - s.router.ServeHTTP(s.recorder, request) - s.Equal(http.StatusOK, s.recorder.Code) - }) -} - -func TestDeleteRunnerRouteTestSuite(t *testing.T) { - suite.Run(t, new(DeleteRunnerRouteTestSuite)) -} - -type DeleteRunnerRouteTestSuite struct { - RunnerRouteTestSuite - path string -} - -func (s *DeleteRunnerRouteTestSuite) SetupTest() { - s.RunnerRouteTestSuite.SetupTest() - deleteURL, err := s.router.Get(DeleteRoute).URL(RunnerIDKey, s.runner.ID()) - s.Require().NoError(err) - s.path = deleteURL.String() -} - -func (s *DeleteRunnerRouteTestSuite) TestValidRequestReturnsNoContent() { - s.runnerManager.On("Return", s.runner).Return(nil) - - recorder := httptest.NewRecorder() - request, err := http.NewRequest(http.MethodDelete, s.path, http.NoBody) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - - s.Equal(http.StatusNoContent, recorder.Code) - - s.Run("runner was returned to runner manager", func() { - s.runnerManager.AssertCalled(s.T(), "Return", s.runner) - }) -} - -func (s *DeleteRunnerRouteTestSuite) TestReturnInternalServerErrorWhenApiCallToNomadFailed() { - s.runnerManager.On("Return", s.runner).Return(tests.ErrDefault) - - recorder := httptest.NewRecorder() - request, err := http.NewRequest(http.MethodDelete, s.path, http.NoBody) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - - s.Equal(http.StatusInternalServerError, recorder.Code) -} - -func (s *DeleteRunnerRouteTestSuite) TestDeleteInvalidRunnerIdReturnsGone() { - s.runnerManager.On("Get", mock.AnythingOfType("string")).Return(nil, tests.ErrDefault) - deleteURL, err := s.router.Get(DeleteRoute).URL(RunnerIDKey, "1nv4l1dID") - s.Require().NoError(err) - deletePath := deleteURL.String() - - recorder := httptest.NewRecorder() - request, err := http.NewRequest(http.MethodDelete, deletePath, http.NoBody) - s.Require().NoError(err) - - s.router.ServeHTTP(recorder, request) - - s.Equal(http.StatusGone, recorder.Code) -} diff --git a/internal/api/websocket_test.go b/internal/api/websocket_test.go deleted file mode 100644 index 7627497..0000000 --- a/internal/api/websocket_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package api - -import ( - "bufio" - "context" - "crypto/tls" - "fmt" - "github.com/gorilla/mux" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/environment" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "io" - "net/http" - "net/http/httptest" - "net/url" - "testing" - "time" -) - -func TestWebSocketTestSuite(t *testing.T) { - suite.Run(t, new(WebSocketTestSuite)) -} - -type WebSocketTestSuite struct { - tests.MemoryLeakTestSuite - router *mux.Router - executionID string - runner runner.Runner - apiMock *nomad.ExecutorAPIMock - server *httptest.Server -} - -func (s *WebSocketTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - runnerID := "runner-id" - s.runner, s.apiMock = newNomadAllocationWithMockedAPIClient(runnerID) - - // default execution - s.executionID = tests.DefaultExecutionID - s.runner.StoreExecution(s.executionID, &executionRequestLs) - mockAPIExecuteLs(s.apiMock) - - runnerManager := &runner.ManagerMock{} - runnerManager.On("Get", s.runner.ID()).Return(s.runner, nil) - s.router = NewRouter(runnerManager, nil) - s.server = httptest.NewServer(s.router) -} - -func (s *WebSocketTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - s.server.Close() - err := s.runner.Destroy(nil) - s.Require().NoError(err) -} - -func (s *WebSocketTestSuite) TestWebsocketConnectionCanBeEstablished() { - wsURL, err := s.webSocketURL("ws", s.runner.ID(), s.executionID) - s.Require().NoError(err) - conn, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - <-time.After(tests.ShortTimeout) - err = conn.Close() - s.NoError(err) -} - -func (s *WebSocketTestSuite) TestWebsocketReturns404IfExecutionDoesNotExist() { - wsURL, err := s.webSocketURL("ws", s.runner.ID(), "invalid-execution-id") - s.Require().NoError(err) - _, response, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().ErrorIs(err, websocket.ErrBadHandshake) - s.Equal(http.StatusNotFound, response.StatusCode) -} - -func (s *WebSocketTestSuite) TestWebsocketReturns400IfRequestedViaHttp() { - wsURL, err := s.webSocketURL("http", s.runner.ID(), s.executionID) - s.Require().NoError(err) - response, err := http.Get(wsURL.String()) - s.Require().NoError(err) - // This functionality is implemented by the WebSocket library. - s.Equal(http.StatusBadRequest, response.StatusCode) - _, err = io.ReadAll(response.Body) - s.NoError(err) -} - -func (s *WebSocketTestSuite) TestWebsocketConnection() { - s.runner.StoreExecution(s.executionID, &executionRequestHead) - mockAPIExecuteHead(s.apiMock) - wsURL, err := s.webSocketURL("ws", s.runner.ID(), s.executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - err = connection.SetReadDeadline(time.Now().Add(5 * time.Second)) - s.Require().NoError(err) - - s.Run("Receives start message", func() { - message, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, message.Type) - }) - - s.Run("Executes the request in the runner", func() { - <-time.After(tests.ShortTimeout) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.AnythingOfType("bool"), - mock.Anything, mock.Anything, mock.Anything) - }) - - s.Run("Can send input", func() { - err = connection.WriteMessage(websocket.TextMessage, []byte("Hello World\n")) - s.Require().NoError(err) - }) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - - s.Run("Receives output message", func() { - stdout, _, _ := helpers.WebSocketOutputMessages(messages) - s.Equal("Hello World", stdout) - }) - - s.Run("Receives exit message", func() { - controlMessages := helpers.WebSocketControlMessages(messages) - s.Require().Equal(1, len(controlMessages)) - s.Equal(dto.WebSocketExit, controlMessages[0].Type) - }) -} - -func (s *WebSocketTestSuite) TestCancelWebSocketConnection() { - executionID := "sleeping-execution" - s.runner.StoreExecution(executionID, &executionRequestSleep) - canceled := mockAPIExecuteSleep(s.apiMock) - - wsURL, err := webSocketURL("ws", s.server, s.router, s.runner.ID(), executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - message, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, message.Type) - - select { - case <-canceled: - s.Fail("ExecuteInteractively canceled unexpected") - default: - } - - err = connection.WriteControl(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) - s.Require().NoError(err) - - select { - case <-canceled: - case <-time.After(time.Second): - s.Fail("ExecuteInteractively not canceled") - } -} - -func (s *WebSocketTestSuite) TestWebSocketConnectionTimeout() { - executionID := "time-out-execution" - limitExecution := executionRequestSleep - limitExecution.TimeLimit = 2 - s.runner.StoreExecution(executionID, &limitExecution) - canceled := mockAPIExecuteSleep(s.apiMock) - - wsURL, err := webSocketURL("ws", s.server, s.router, s.runner.ID(), executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - message, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, message.Type) - - select { - case <-canceled: - s.Fail("ExecuteInteractively canceled unexpected") - case <-time.After(time.Duration(limitExecution.TimeLimit-1) * time.Second): - <-time.After(time.Second) - } - - select { - case <-canceled: - case <-time.After(time.Second): - s.Fail("ExecuteInteractively not canceled") - } - - message, err = helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaTimeout, message.Type) -} - -func (s *WebSocketTestSuite) TestWebsocketStdoutAndStderr() { - executionID := "ls-execution" - s.runner.StoreExecution(executionID, &executionRequestLs) - mockAPIExecuteLs(s.apiMock) - - wsURL, err := webSocketURL("ws", s.server, s.router, s.runner.ID(), executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) - - s.Contains(stdout, "existing-file") - - s.Contains(stderr, "non-existing-file") -} - -func (s *WebSocketTestSuite) TestWebsocketError() { - executionID := "error-execution" - s.runner.StoreExecution(executionID, &executionRequestError) - mockAPIExecuteError(s.apiMock) - - wsURL, err := webSocketURL("ws", s.server, s.router, s.runner.ID(), executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - - _, _, errMessages := helpers.WebSocketOutputMessages(messages) - s.Require().Equal(1, len(errMessages)) - s.Equal("Error executing the request", errMessages[0]) -} - -func (s *WebSocketTestSuite) TestWebsocketNonZeroExit() { - executionID := "exit-execution" - s.runner.StoreExecution(executionID, &executionRequestExitNonZero) - mockAPIExecuteExitNonZero(s.apiMock) - - wsURL, err := webSocketURL("ws", s.server, s.router, s.runner.ID(), executionID) - s.Require().NoError(err) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - - controlMessages := helpers.WebSocketControlMessages(messages) - s.Equal(2, len(controlMessages)) - s.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit, ExitCode: 42}, controlMessages[1]) -} - -func (s *MainTestSuite) TestWebsocketTLS() { - runnerID := "runner-id" - r, apiMock := newNomadAllocationWithMockedAPIClient(runnerID) - - executionID := tests.DefaultExecutionID - r.StoreExecution(executionID, &executionRequestLs) - mockAPIExecuteLs(apiMock) - - runnerManager := &runner.ManagerMock{} - runnerManager.On("Get", r.ID()).Return(r, nil) - router := NewRouter(runnerManager, nil) - - server, err := helpers.StartTLSServer(s.T(), router) - s.Require().NoError(err) - defer server.Close() - - wsURL, err := webSocketURL("wss", server, router, runnerID, executionID) - s.Require().NoError(err) - - config := &tls.Config{RootCAs: nil, InsecureSkipVerify: true} //nolint:gosec // test needs self-signed cert - d := websocket.Dialer{TLSClientConfig: config} - connection, _, err := d.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - message, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, message.Type) - _, err = helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - s.NoError(r.Destroy(nil)) -} - -func (s *MainTestSuite) TestWebSocketProxyStopsReadingTheWebSocketAfterClosingIt() { - apiMock := &nomad.ExecutorAPIMock{} - executionID := tests.DefaultExecutionID - r, wsURL, cleanup := newRunnerWithNotMockedRunnerManager(s, apiMock, executionID) - defer cleanup() - - logger, hook := test.NewNullLogger() - log = logger.WithField("pkg", "api") - - r.StoreExecution(executionID, &executionRequestHead) - mockAPIExecute(apiMock, &executionRequestHead, - func(_ string, ctx context.Context, _ string, _ bool, _ io.Reader, _, _ io.Writer) (int, error) { - return 0, nil - }) - connection, _, err := websocket.DefaultDialer.Dial(wsURL.String(), nil) - s.Require().NoError(err) - - _, err = helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - for _, logMsg := range hook.Entries { - if logMsg.Level < logrus.InfoLevel { - s.Fail(logMsg.Message) - } - } -} - -// --- Test suite specific test helpers --- - -func newNomadAllocationWithMockedAPIClient(runnerID string) (runner.Runner, *nomad.ExecutorAPIMock) { - executorAPIMock := &nomad.ExecutorAPIMock{} - executorAPIMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - manager := &runner.ManagerMock{} - manager.On("Return", mock.Anything).Return(nil) - r := runner.NewNomadJob(runnerID, nil, executorAPIMock, nil) - return r, executorAPIMock -} - -func newRunnerWithNotMockedRunnerManager(s *MainTestSuite, apiMock *nomad.ExecutorAPIMock, executionID string) ( - r runner.Runner, wsURL *url.URL, cleanup func()) { - s.T().Helper() - apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string"), mock.AnythingOfType("int")).Return(nil) - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - apiMock.On("RegisterRunnerJob", mock.AnythingOfType("*api.Job")).Return(nil) - call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) - call.Run(func(args mock.Arguments) { - <-s.TestCtx.Done() - call.ReturnArguments = mock.Arguments{nil} - }) - - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - router := NewRouter(runnerManager, nil) - s.ExpectedGoroutineIncrease++ // The server is not closing properly. Therefore, we don't even try. - server := httptest.NewServer(router) - - runnerID := tests.DefaultRunnerID - runnerJob := runner.NewNomadJob(runnerID, nil, apiMock, nil) - e, err := environment.NewNomadEnvironment(0, apiMock, "job \"template-0\" {}") - s.Require().NoError(err) - eID, err := nomad.EnvironmentIDFromRunnerID(runnerID) - s.Require().NoError(err) - e.SetID(eID) - e.SetPrewarmingPoolSize(0) - runnerManager.StoreEnvironment(e) - e.AddRunner(runnerJob) - - r, err = runnerManager.Claim(e.ID(), int(tests.DefaultTestTimeout.Seconds())) - s.Require().NoError(err) - wsURL, err = webSocketURL("ws", server, router, r.ID(), executionID) - s.Require().NoError(err) - - return r, wsURL, func() { - err = r.Destroy(tests.ErrCleanupDestroyReason) - s.NoError(err) - err = e.Delete(tests.ErrCleanupDestroyReason) - s.NoError(err) - } -} - -func webSocketURL(scheme string, server *httptest.Server, router *mux.Router, - runnerID string, executionID string, -) (*url.URL, error) { - websocketURL, err := url.Parse(server.URL) - if err != nil { - return nil, err - } - path, err := router.Get(WebsocketPath).URL(RunnerIDKey, runnerID) - if err != nil { - return nil, err - } - websocketURL.Scheme = scheme - websocketURL.Path = path.Path - websocketURL.RawQuery = fmt.Sprintf("executionID=%s", executionID) - return websocketURL, nil -} - -func (s *WebSocketTestSuite) webSocketURL(scheme, runnerID, executionID string) (*url.URL, error) { - return webSocketURL(scheme, s.server, s.router, runnerID, executionID) -} - -var executionRequestLs = dto.ExecutionRequest{Command: "ls"} - -// mockAPIExecuteLs mocks the ExecuteCommand of an ExecutorApi to act as if -// 'ls existing-file non-existing-file' was executed. -func mockAPIExecuteLs(api *nomad.ExecutorAPIMock) { - mockAPIExecute(api, &executionRequestLs, - func(_ string, _ context.Context, _ string, _ bool, _ io.Reader, stdout, stderr io.Writer) (int, error) { - _, _ = stdout.Write([]byte("existing-file\n")) - _, _ = stderr.Write([]byte("ls: cannot access 'non-existing-file': No such file or directory\n")) - return 0, nil - }) -} - -var executionRequestHead = dto.ExecutionRequest{Command: "head -n 1"} - -// mockAPIExecuteHead mocks the ExecuteCommand of an ExecutorApi to act as if 'head -n 1' was executed. -func mockAPIExecuteHead(api *nomad.ExecutorAPIMock) { - mockAPIExecute(api, &executionRequestHead, - func(_ string, _ context.Context, _ string, _ bool, - stdin io.Reader, stdout io.Writer, stderr io.Writer, - ) (int, error) { - scanner := bufio.NewScanner(stdin) - for !scanner.Scan() { - scanner = bufio.NewScanner(stdin) - } - _, _ = stdout.Write(scanner.Bytes()) - return 0, nil - }) -} - -var executionRequestSleep = dto.ExecutionRequest{Command: "sleep infinity"} - -// mockAPIExecuteSleep mocks the ExecuteCommand method of an ExecutorAPI to sleep -// until the execution receives a SIGQUIT. -func mockAPIExecuteSleep(api *nomad.ExecutorAPIMock) <-chan bool { - canceled := make(chan bool, 1) - - mockAPIExecute(api, &executionRequestSleep, - func(_ string, ctx context.Context, _ string, _ bool, - stdin io.Reader, stdout io.Writer, stderr io.Writer, - ) (int, error) { - var err error - buffer := make([]byte, 1) //nolint:makezero // if the length is zero, the Read call never reads anything - for n := 0; !(n == 1 && buffer[0] == runner.SIGQUIT); n, err = stdin.Read(buffer) { - if err != nil { - return 0, fmt.Errorf("error while reading stdin: %w", err) - } - } - close(canceled) - return 0, ctx.Err() - }) - return canceled -} - -var executionRequestError = dto.ExecutionRequest{Command: "error"} - -// mockAPIExecuteError mocks the ExecuteCommand method of an ExecutorApi to return an error. -func mockAPIExecuteError(api *nomad.ExecutorAPIMock) { - mockAPIExecute(api, &executionRequestError, - func(_ string, _ context.Context, _ string, _ bool, _ io.Reader, _, _ io.Writer) (int, error) { - return 0, tests.ErrDefault - }) -} - -var executionRequestExitNonZero = dto.ExecutionRequest{Command: "exit 42"} - -// mockAPIExecuteExitNonZero mocks the ExecuteCommand method of an ExecutorApi to exit with exit status 42. -func mockAPIExecuteExitNonZero(api *nomad.ExecutorAPIMock) { - mockAPIExecute(api, &executionRequestExitNonZero, - func(_ string, _ context.Context, _ string, _ bool, _ io.Reader, _, _ io.Writer) (int, error) { - return 42, nil - }) -} - -// mockAPIExecute mocks the ExecuteCommand method of an ExecutorApi to call the given method run when the command -// corresponding to the given ExecutionRequest is called. -func mockAPIExecute(api *nomad.ExecutorAPIMock, request *dto.ExecutionRequest, - run func(runnerId string, ctx context.Context, command string, tty bool, - stdin io.Reader, stdout, stderr io.Writer) (int, error)) { - tests.RemoveMethodFromMock(&api.Mock, "ExecuteCommand") - call := api.On("ExecuteCommand", - mock.AnythingOfType("string"), - mock.Anything, - request.FullCommand(), - mock.AnythingOfType("bool"), - mock.AnythingOfType("bool"), - mock.Anything, - mock.Anything, - mock.Anything) - call.Run(func(args mock.Arguments) { - exit, err := run(args.Get(0).(string), - args.Get(1).(context.Context), - args.Get(2).(string), - args.Get(3).(bool), - args.Get(5).(io.Reader), - args.Get(6).(io.Writer), - args.Get(7).(io.Writer)) - call.ReturnArguments = mock.Arguments{exit, err} - }) -} diff --git a/internal/api/ws/codeocean_reader_test.go b/internal/api/ws/codeocean_reader_test.go deleted file mode 100644 index 0639b16..0000000 --- a/internal/api/ws/codeocean_reader_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package ws - -import ( - "context" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "io" - "strings" - "testing" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestCodeOceanToRawReaderReturnsOnlyAfterOneByteWasRead() { - readingCtx, cancel := context.WithCancel(context.Background()) - forwardingCtx := readingCtx - defer cancel() - reader := NewCodeOceanToRawReader(nil, readingCtx, forwardingCtx) - - read := make(chan bool) - go func() { - //nolint:makezero // we can't make zero initial length here as the reader otherwise doesn't block - p := make([]byte, 10) - _, err := reader.Read(p) - s.Require().NoError(err) - read <- true - }() - - s.Run("Does not return immediately when there is no data", func() { - s.False(tests.ChannelReceivesSomething(read, tests.ShortTimeout)) - }) - - s.Run("Returns when there is data available", func() { - reader.buffer <- byte(42) - s.True(tests.ChannelReceivesSomething(read, tests.ShortTimeout)) - }) -} - -func (s *MainTestSuite) TestCodeOceanToRawReaderReturnsOnlyAfterOneByteWasReadFromConnection() { - messages := make(chan io.Reader) - defer close(messages) - - connection := &ConnectionMock{} - connection.On("WriteMessage", mock.AnythingOfType("int"), mock.AnythingOfType("[]uint8")).Return(nil) - connection.On("CloseHandler").Return(nil) - connection.On("SetCloseHandler", mock.Anything).Return() - call := connection.On("NextReader") - call.Run(func(_ mock.Arguments) { - call.Return(websocket.TextMessage, <-messages, nil) - }) - - readingCtx, cancel := context.WithCancel(context.Background()) - forwardingCtx := readingCtx - defer cancel() - reader := NewCodeOceanToRawReader(connection, readingCtx, forwardingCtx) - reader.Start() - - read := make(chan bool) - //nolint:makezero // this is required here to make the Read call blocking - message := make([]byte, 10) - go func() { - _, err := reader.Read(message) - s.Require().NoError(err) - read <- true - }() - - s.Run("Does not return immediately when there is no data", func() { - s.False(tests.ChannelReceivesSomething(read, tests.ShortTimeout)) - }) - - s.Run("Returns when there is data available", func() { - messages <- strings.NewReader("Hello") - s.True(tests.ChannelReceivesSomething(read, tests.ShortTimeout)) - }) -} diff --git a/internal/api/ws/codeocean_writer_test.go b/internal/api/ws/codeocean_writer_test.go deleted file mode 100644 index f7cf562..0000000 --- a/internal/api/ws/codeocean_writer_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package ws - -import ( - "context" - "encoding/json" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" -) - -func (s *MainTestSuite) TestRawToCodeOceanWriter() { - connectionMock, messages := buildConnectionMock(&s.MemoryLeakTestSuite) - proxyCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - output := NewCodeOceanOutputWriter(connectionMock, proxyCtx, cancel) - defer output.Close(nil) - <-messages // start messages - - s.Run("StdOut", func() { - testMessage := "testStdOut" - _, err := output.StdOut().Write([]byte(testMessage)) - s.Require().NoError(err) - - expected, err := json.Marshal(struct { - Type string `json:"type"` - Data string `json:"data"` - }{string(dto.WebSocketOutputStdout), testMessage}) - s.Require().NoError(err) - - s.Equal(expected, <-messages) - }) - - s.Run("StdErr", func() { - testMessage := "testStdErr" - _, err := output.StdErr().Write([]byte(testMessage)) - s.Require().NoError(err) - - expected, err := json.Marshal(struct { - Type string `json:"type"` - Data string `json:"data"` - }{string(dto.WebSocketOutputStderr), testMessage}) - s.Require().NoError(err) - - s.Equal(expected, <-messages) - }) -} - -type sendExitInfoTestCase struct { - name string - info *runner.ExitInfo - message dto.WebSocketMessage -} - -func (s *MainTestSuite) TestCodeOceanOutputWriter_SendExitInfo() { - testCases := []sendExitInfoTestCase{ - {"Timeout", &runner.ExitInfo{Err: runner.ErrorRunnerInactivityTimeout}, - dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout}}, - {"Error", &runner.ExitInfo{Err: websocket.ErrCloseSent}, - dto.WebSocketMessage{Type: dto.WebSocketOutputError, Data: "Error executing the request"}}, - // CodeOcean expects this exact string in case of a OOM Killed runner. - {"Specific data for OOM Killed runner", &runner.ExitInfo{Err: runner.ErrOOMKilled}, - dto.WebSocketMessage{Type: dto.WebSocketOutputError, Data: "the allocation was OOM Killed"}}, - {"Exit", &runner.ExitInfo{Code: 21}, - dto.WebSocketMessage{Type: dto.WebSocketExit, ExitCode: 21}}, - } - - for _, test := range testCases { - s.Run(test.name, func() { - connectionMock, messages := buildConnectionMock(&s.MemoryLeakTestSuite) - proxyCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - output := NewCodeOceanOutputWriter(connectionMock, proxyCtx, cancel) - <-messages // start messages - - output.Close(test.info) - expected, err := json.Marshal(test.message) - s.Require().NoError(err) - - msg := <-messages - s.Equal(expected, msg) - - <-messages // close message - }) - } -} - -func buildConnectionMock(s *tests.MemoryLeakTestSuite) (conn *ConnectionMock, messages <-chan []byte) { - s.T().Helper() - message := make(chan []byte) - connectionMock := &ConnectionMock{} - connectionMock.On("WriteMessage", mock.AnythingOfType("int"), mock.AnythingOfType("[]uint8")). - Run(func(args mock.Arguments) { - m, ok := args.Get(1).([]byte) - s.Require().True(ok) - select { - case <-s.TestCtx.Done(): - case message <- m: - } - }). - Return(nil) - connectionMock.On("CloseHandler").Return(nil) - connectionMock.On("SetCloseHandler", mock.Anything).Return() - connectionMock.On("Close").Return(nil) - return connectionMock, message -} diff --git a/internal/api/ws/connection_mock.go b/internal/api/ws/connection_mock.go deleted file mode 100644 index c7eb31c..0000000 --- a/internal/api/ws/connection_mock.go +++ /dev/null @@ -1,108 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package ws - -import ( - io "io" - - mock "github.com/stretchr/testify/mock" -) - -// ConnectionMock is an autogenerated mock type for the Connection type -type ConnectionMock struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *ConnectionMock) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// CloseHandler provides a mock function with given fields: -func (_m *ConnectionMock) CloseHandler() func(int, string) error { - ret := _m.Called() - - var r0 func(int, string) error - if rf, ok := ret.Get(0).(func() func(int, string) error); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(func(int, string) error) - } - } - - return r0 -} - -// NextReader provides a mock function with given fields: -func (_m *ConnectionMock) NextReader() (int, io.Reader, error) { - ret := _m.Called() - - var r0 int - if rf, ok := ret.Get(0).(func() int); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int) - } - - var r1 io.Reader - if rf, ok := ret.Get(1).(func() io.Reader); ok { - r1 = rf() - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(io.Reader) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func() error); ok { - r2 = rf() - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// SetCloseHandler provides a mock function with given fields: handler -func (_m *ConnectionMock) SetCloseHandler(handler func(int, string) error) { - _m.Called(handler) -} - -// WriteMessage provides a mock function with given fields: messageType, data -func (_m *ConnectionMock) WriteMessage(messageType int, data []byte) error { - ret := _m.Called(messageType, data) - - var r0 error - if rf, ok := ret.Get(0).(func(int, []byte) error); ok { - r0 = rf(messageType, data) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewConnectionMock interface { - mock.TestingT - Cleanup(func()) -} - -// NewConnectionMock creates a new instance of ConnectionMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnectionMock(t mockConstructorTestingTNewConnectionMock) *ConnectionMock { - mock := &ConnectionMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/config/config.go b/internal/config/config.go index 00c6f41..a5a61e3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,6 +13,7 @@ import ( "github.com/openHPI/poseidon/pkg/logging" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" + "k8s.io/client-go/rest" "net/url" "os" "reflect" @@ -66,6 +67,17 @@ var ( DNS: nil, }, }, + Kubernetes: Kubernetes{ + Enabled: false, + KubeConfig: rest.Config{ + Host: "", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: false, + ServerName: "", + }, + BearerToken: "", + }, + }, AWS: AWS{ Enabled: false, Endpoint: "", @@ -134,6 +146,11 @@ type Nomad struct { Network nomadApi.NetworkResource } +type Kubernetes struct { + Enabled bool + KubeConfig rest.Config +} + // URL returns the URL for the configured Nomad cluster. func (n *Nomad) URL() *url.URL { return parseURL(n.Address, n.Port, n.TLS.Active) @@ -179,13 +196,14 @@ type InfluxDB struct { // configuration contains the complete configuration of Poseidon. type configuration struct { - Server server - Nomad Nomad - AWS AWS - Logger Logger - Profiling Profiling - Sentry sentry.ClientOptions - InfluxDB InfluxDB + Server server + Nomad Nomad + Kubernetes Kubernetes + AWS AWS + Logger Logger + Profiling Profiling + Sentry sentry.ClientOptions + InfluxDB InfluxDB } // InitConfig merges configuration options from environment variables and diff --git a/internal/config/config_test.go b/internal/config/config_test.go deleted file mode 100644 index e16cf92..0000000 --- a/internal/config/config_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package config - -import ( - "fmt" - "github.com/openHPI/poseidon/tests" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "os" - "path/filepath" - "reflect" - "testing" -) - -var ( - getServerPort = func(c *configuration) interface{} { return c.Server.Port } - getNomadToken = func(c *configuration) interface{} { return c.Nomad.Token } - getNomadTLSActive = func(c *configuration) interface{} { return c.Nomad.TLS.Active } - getAWSFunctions = func(c *configuration) interface{} { return c.AWS.Functions } -) - -func newTestConfiguration() *configuration { - return &configuration{ - Server: server{ - Address: "127.0.0.1", - Port: 3000, - }, - Nomad: Nomad{ - Address: "127.0.0.2", - Port: 4646, - Token: "SECRET", - TLS: TLS{ - Active: false, - }, - }, - Logger: Logger{ - Level: "INFO", - }, - } -} - -func (c *configuration) getReflectValue() reflect.Value { - return reflect.ValueOf(c).Elem() -} - -// writeConfigurationFile creates a file on disk and returns the path to it. -func writeConfigurationFile(t *testing.T, name string, content []byte) string { - t.Helper() - directory := t.TempDir() - filePath := filepath.Join(directory, name) - file, err := os.Create(filePath) - if err != nil { - t.Fatal("Could not create config file") - } - defer file.Close() - _, err = file.Write(content) - require.NoError(t, err) - return filePath -} - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestCallingInitConfigTwiceReturnsError() { - configurationInitialized = false - err := InitConfig() - s.NoError(err) - err = InitConfig() - s.Error(err) -} - -func (s *MainTestSuite) TestCallingInitConfigTwiceDoesNotChangeConfig() { - configurationInitialized = false - err := InitConfig() - s.Require().NoError(err) - Config = newTestConfiguration() - filePath := writeConfigurationFile(s.T(), "test.yaml", []byte("server:\n port: 5000\n")) - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = append(os.Args, "-config", filePath) - err = InitConfig() - s.Require().Error(err) - s.Equal(3000, Config.Server.Port) -} - -func (s *MainTestSuite) TestReadEnvironmentVariables() { - var environmentTests = []struct { - variableSuffix string - valueToSet string - expectedValue interface{} - getTargetField func(*configuration) interface{} - }{ - {"SERVER_PORT", "4000", 4000, getServerPort}, - {"SERVER_PORT", "hello", 3000, getServerPort}, - {"NOMAD_TOKEN", "ACCESS", "ACCESS", getNomadToken}, - {"NOMAD_TLS_ACTIVE", "true", true, getNomadTLSActive}, - {"NOMAD_TLS_ACTIVE", "hello", false, getNomadTLSActive}, - {"AWS_FUNCTIONS", "java11Exec go118Exec", []string{"java11Exec", "go118Exec"}, getAWSFunctions}, - } - prefix := "POSEIDON_TEST" - for _, testCase := range environmentTests { - config := newTestConfiguration() - environmentVariable := fmt.Sprintf("%s_%s", prefix, testCase.variableSuffix) - _ = os.Setenv(environmentVariable, testCase.valueToSet) - readFromEnvironment(prefix, config.getReflectValue()) - _ = os.Unsetenv(environmentVariable) - s.Equal(testCase.expectedValue, testCase.getTargetField(config)) - } -} - -func (s *MainTestSuite) TestReadEnvironmentIgnoresNonPointerValue() { - config := newTestConfiguration() - _ = os.Setenv("POSEIDON_TEST_SERVER_PORT", "4000") - readFromEnvironment("POSEIDON_TEST", reflect.ValueOf(config)) - _ = os.Unsetenv("POSEIDON_TEST_SERVER_PORT") - s.Equal(3000, config.Server.Port) -} - -func (s *MainTestSuite) TestReadEnvironmentIgnoresNotSupportedType() { - config := &struct{ Timeout float64 }{1.0} - _ = os.Setenv("POSEIDON_TEST_TIMEOUT", "2.5") - readFromEnvironment("POSEIDON_TEST", reflect.ValueOf(config).Elem()) - _ = os.Unsetenv("POSEIDON_TEST_TIMEOUT") - s.Equal(1.0, config.Timeout) -} - -func (s *MainTestSuite) TestUnsetEnvironmentVariableDoesNotChangeConfig() { - config := newTestConfiguration() - readFromEnvironment("POSEIDON_TEST", config.getReflectValue()) - s.Equal("INFO", config.Logger.Level) -} - -func (s *MainTestSuite) TestReadYamlConfigFile() { - var yamlTests = []struct { - content []byte - expectedValue interface{} - getTargetField func(*configuration) interface{} - }{ - {[]byte("server:\n port: 5000\n"), 5000, getServerPort}, - {[]byte("nomad:\n token: ACCESS\n"), "ACCESS", getNomadToken}, - {[]byte("nomad:\n tls:\n active: true\n"), true, getNomadTLSActive}, - {[]byte(""), false, getNomadTLSActive}, - {[]byte("nomad:\n token:\n"), "SECRET", getNomadToken}, - {[]byte("aws:\n functions:\n - java11Exec\n - go118Exec\n"), - []string{"java11Exec", "go118Exec"}, getAWSFunctions}, - } - for _, testCase := range yamlTests { - config := newTestConfiguration() - config.mergeYaml(testCase.content) - s.Equal(testCase.expectedValue, testCase.getTargetField(config)) - } -} - -func (s *MainTestSuite) TestInvalidYamlExitsProgram() { - logger, hook := test.NewNullLogger() - // this function is used when calling log.Fatal() and - // prevents the program from exiting during this test - logger.ExitFunc = func(code int) {} - log = logger.WithField("package", "config_test") - config := newTestConfiguration() - config.mergeYaml([]byte("logger: level: DEBUG")) - s.Equal(1, len(hook.Entries)) - s.Equal(logrus.FatalLevel, hook.LastEntry().Level) -} - -func (s *MainTestSuite) TestReadConfigFileOverwritesConfig() { - Config = newTestConfiguration() - filePath := writeConfigurationFile(s.T(), "test.yaml", []byte("server:\n port: 5000\n")) - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = append(os.Args, "-config", filePath) - configurationInitialized = false - err := InitConfig() - s.Require().NoError(err) - s.Equal(5000, Config.Server.Port) -} - -func (s *MainTestSuite) TestReadNonExistingConfigFileDoesNotOverwriteConfig() { - Config = newTestConfiguration() - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = append(os.Args, "-config", "file_does_not_exist.yaml") - configurationInitialized = false - err := InitConfig() - s.Require().NoError(err) - s.Equal(3000, Config.Server.Port) -} - -func (s *MainTestSuite) TestURLParsing() { - var urlTests = []struct { - address string - port int - tls bool - expectedScheme string - expectedHost string - }{ - {"localhost", 3000, false, "http", "localhost:3000"}, - {"127.0.0.1", 4000, true, "https", "127.0.0.1:4000"}, - } - for _, testCase := range urlTests { - url := parseURL(testCase.address, testCase.port, testCase.tls) - s.Equal(testCase.expectedScheme, url.Scheme) - s.Equal(testCase.expectedHost, url.Host) - } -} - -func (s *MainTestSuite) TestNomadAPIURL() { - config := newTestConfiguration() - s.Equal("http", config.Nomad.URL().Scheme) - s.Equal("127.0.0.2:4646", config.Nomad.URL().Host) -} - -func (s *MainTestSuite) TestPoseidonAPIURL() { - config := newTestConfiguration() - s.Equal("http", config.Server.URL().Scheme) - s.Equal("127.0.0.1:3000", config.Server.URL().Host) -} diff --git a/internal/environment/aws_environment.go b/internal/environment/aws_environment.go deleted file mode 100644 index e6fcbb8..0000000 --- a/internal/environment/aws_environment.go +++ /dev/null @@ -1,121 +0,0 @@ -package environment - -import ( - "encoding/json" - "fmt" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" -) - -type AWSEnvironment struct { - id dto.EnvironmentID - awsEndpoint string - onDestroyRunner runner.DestroyRunnerHandler -} - -func NewAWSEnvironment(onDestroyRunner runner.DestroyRunnerHandler) *AWSEnvironment { - return &AWSEnvironment{onDestroyRunner: onDestroyRunner} -} - -func (a *AWSEnvironment) MarshalJSON() ([]byte, error) { - res, err := json.Marshal(dto.ExecutionEnvironmentData{ - ID: int(a.ID()), - ExecutionEnvironmentRequest: dto.ExecutionEnvironmentRequest{Image: a.Image()}, - }) - if err != nil { - return res, fmt.Errorf("couldn't marshal aws execution environment: %w", err) - } - return res, nil -} - -func (a *AWSEnvironment) ID() dto.EnvironmentID { - return a.id -} - -func (a *AWSEnvironment) SetID(id dto.EnvironmentID) { - a.id = id -} - -// Image is used to specify the AWS Endpoint Poseidon is connecting to. -func (a *AWSEnvironment) Image() string { - return a.awsEndpoint -} - -func (a *AWSEnvironment) SetImage(awsEndpoint string) { - a.awsEndpoint = awsEndpoint -} - -func (a *AWSEnvironment) Delete(_ runner.DestroyReason) error { - return nil -} - -func (a *AWSEnvironment) Sample() (r runner.Runner, ok bool) { - workload, err := runner.NewAWSFunctionWorkload(a, a.onDestroyRunner) - if err != nil { - return nil, false - } - return workload, true -} - -// The following methods are not supported at this moment. - -// IdleRunnerCount is not supported as we have no information about the AWS managed prewarming pool. -// For the Poseidon Health check we default to 1. -func (a *AWSEnvironment) IdleRunnerCount() uint { - return 1 -} - -// PrewarmingPoolSize is neither supported nor required. It is handled transparently by AWS. -// For easy compatibility with CodeOcean, 1 is the static value. -func (a *AWSEnvironment) PrewarmingPoolSize() uint { - return 1 -} - -// SetPrewarmingPoolSize is neither supported nor required. It is handled transparently by AWS. -func (a *AWSEnvironment) SetPrewarmingPoolSize(_ uint) {} - -// ApplyPrewarmingPoolSize is neither supported nor required. It is handled transparently by AWS. -func (a *AWSEnvironment) ApplyPrewarmingPoolSize() error { - return nil -} - -// CPULimit is disabled as one can only set the memory limit with AWS Lambda. -func (a *AWSEnvironment) CPULimit() uint { - return 0 -} - -// SetCPULimit is disabled as one can only set the memory limit with AWS Lambda. -func (a *AWSEnvironment) SetCPULimit(_ uint) {} - -func (a *AWSEnvironment) MemoryLimit() uint { - const memorySizeOfDeployedLambdaFunction = 2048 // configured /deploy/aws/template.yaml - return memorySizeOfDeployedLambdaFunction -} - -func (a *AWSEnvironment) SetMemoryLimit(_ uint) { - panic("not supported") -} - -func (a *AWSEnvironment) NetworkAccess() (enabled bool, mappedPorts []uint16) { - return true, nil -} - -func (a *AWSEnvironment) SetNetworkAccess(_ bool, _ []uint16) { - panic("not supported") -} - -func (a *AWSEnvironment) SetConfigFrom(_ runner.ExecutionEnvironment) { - panic("not supported") -} - -func (a *AWSEnvironment) Register() error { - panic("not supported") -} - -func (a *AWSEnvironment) AddRunner(_ runner.Runner) { - panic("not supported") -} - -func (a *AWSEnvironment) DeleteRunner(_ string) (r runner.Runner, ok bool) { - panic("not supported") -} diff --git a/internal/environment/aws_manager.go b/internal/environment/aws_manager.go deleted file mode 100644 index 92e9a0a..0000000 --- a/internal/environment/aws_manager.go +++ /dev/null @@ -1,67 +0,0 @@ -package environment - -import ( - "context" - "fmt" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" -) - -// AWSEnvironmentManager contains no functionality at the moment. -// IMPROVE: Create Lambda functions dynamically. -type AWSEnvironmentManager struct { - *AbstractManager -} - -func NewAWSEnvironmentManager(runnerManager runner.Manager) *AWSEnvironmentManager { - return &AWSEnvironmentManager{&AbstractManager{nil, runnerManager}} -} - -func (a *AWSEnvironmentManager) List(fetch bool) ([]runner.ExecutionEnvironment, error) { - list, err := a.NextHandler().List(fetch) - if err != nil { - return nil, fmt.Errorf("aws wrapped: %w", err) - } - return append(list, a.runnerManager.ListEnvironments()...), nil -} - -func (a *AWSEnvironmentManager) Get(id dto.EnvironmentID, fetch bool) (runner.ExecutionEnvironment, error) { - e, ok := a.runnerManager.GetEnvironment(id) - if ok { - return e, nil - } else { - e, err := a.NextHandler().Get(id, fetch) - if err != nil { - return nil, fmt.Errorf("aws wrapped: %w", err) - } - return e, nil - } -} - -func (a *AWSEnvironmentManager) CreateOrUpdate( - id dto.EnvironmentID, request dto.ExecutionEnvironmentRequest, ctx context.Context) (bool, error) { - if !isAWSEnvironment(request) { - isCreated, err := a.NextHandler().CreateOrUpdate(id, request, ctx) - if err != nil { - return false, fmt.Errorf("aws wrapped: %w", err) - } - return isCreated, nil - } - - _, ok := a.runnerManager.GetEnvironment(id) - e := NewAWSEnvironment(a.runnerManager.Return) - e.SetID(id) - e.SetImage(request.Image) - a.runnerManager.StoreEnvironment(e) - return !ok, nil -} - -func isAWSEnvironment(request dto.ExecutionEnvironmentRequest) bool { - for _, function := range config.Config.AWS.Functions { - if request.Image == function { - return true - } - } - return false -} diff --git a/internal/environment/aws_manager_test.go b/internal/environment/aws_manager_test.go deleted file mode 100644 index 63b0e87..0000000 --- a/internal/environment/aws_manager_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package environment - -import ( - "context" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "testing" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestAWSEnvironmentManager_CreateOrUpdate() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - runnerManager := runner.NewAWSRunnerManager(ctx) - m := NewAWSEnvironmentManager(runnerManager) - uniqueImage := "java11Exec" - - s.Run("can create default Java environment", func() { - config.Config.AWS.Functions = []string{uniqueImage} - _, err := m.CreateOrUpdate( - tests.AnotherEnvironmentIDAsInteger, dto.ExecutionEnvironmentRequest{Image: uniqueImage}, context.Background()) - s.NoError(err) - }) - - s.Run("can retrieve added environment", func() { - environment, err := m.Get(tests.AnotherEnvironmentIDAsInteger, false) - s.NoError(err) - s.Equal(environment.Image(), uniqueImage) - }) - - s.Run("non-handleable requests are forwarded to the next manager", func() { - nextHandler := &ManagerHandlerMock{} - nextHandler.On("CreateOrUpdate", mock.AnythingOfType("dto.EnvironmentID"), - mock.AnythingOfType("dto.ExecutionEnvironmentRequest"), mock.Anything).Return(true, nil) - m.SetNextHandler(nextHandler) - - request := dto.ExecutionEnvironmentRequest{} - _, err := m.CreateOrUpdate(tests.DefaultEnvironmentIDAsInteger, request, context.Background()) - s.NoError(err) - nextHandler.AssertCalled(s.T(), "CreateOrUpdate", - dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger), request, mock.Anything) - }) -} - -func (s *MainTestSuite) TestAWSEnvironmentManager_Get() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - runnerManager := runner.NewAWSRunnerManager(ctx) - m := NewAWSEnvironmentManager(runnerManager) - - s.Run("Calls next handler when not found", func() { - nextHandler := &ManagerHandlerMock{} - nextHandler.On("Get", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("bool")). - Return(nil, nil) - m.SetNextHandler(nextHandler) - - _, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.NoError(err) - nextHandler.AssertCalled(s.T(), "Get", dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger), false) - }) - - s.Run("Returns error when not found", func() { - nextHandler := &AbstractManager{nil, nil} - m.SetNextHandler(nextHandler) - - _, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.ErrorIs(err, runner.ErrRunnerNotFound) - }) - - s.Run("Returns environment when it was added before", func() { - expectedEnvironment := NewAWSEnvironment(nil) - expectedEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - runnerManager.StoreEnvironment(expectedEnvironment) - - environment, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.NoError(err) - s.Equal(expectedEnvironment, environment) - }) -} - -func (s *MainTestSuite) TestAWSEnvironmentManager_List() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - runnerManager := runner.NewAWSRunnerManager(ctx) - m := NewAWSEnvironmentManager(runnerManager) - - s.Run("also returns environments of the rest of the manager chain", func() { - nextHandler := &ManagerHandlerMock{} - existingEnvironment := NewAWSEnvironment(nil) - nextHandler.On("List", mock.AnythingOfType("bool")). - Return([]runner.ExecutionEnvironment{existingEnvironment}, nil) - m.SetNextHandler(nextHandler) - - environments, err := m.List(false) - s.NoError(err) - s.Require().Len(environments, 1) - s.Contains(environments, existingEnvironment) - }) - m.SetNextHandler(nil) - - s.Run("Returns added environment", func() { - localEnvironment := NewAWSEnvironment(nil) - localEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - runnerManager.StoreEnvironment(localEnvironment) - - environments, err := m.List(false) - s.NoError(err) - s.Len(environments, 1) - s.Contains(environments, localEnvironment) - }) -} diff --git a/internal/environment/kubernetes_environment.go b/internal/environment/kubernetes_environment.go new file mode 100644 index 0000000..4a3aaed --- /dev/null +++ b/internal/environment/kubernetes_environment.go @@ -0,0 +1,175 @@ +package environment + +import ( + "context" + "fmt" + poseidonK8s "github.com/openHPI/poseidon/internal/kubernetes" + "github.com/openHPI/poseidon/internal/runner" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/monitoring" + "github.com/openHPI/poseidon/pkg/storage" + appsv1 "k8s.io/api/apps/v1" + "time" +) + +type KubernetesEnvironment struct { + apiClient *poseidonK8s.ExecutorAPI + jobHCL string + deployment *appsv1.Deployment + idleRunners storage.Storage[runner.Runner] + ctx context.Context + cancel context.CancelFunc +} + +func (k KubernetesEnvironment) MarshalJSON() ([]byte, error) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) ID() dto.EnvironmentID { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetID(id dto.EnvironmentID) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) PrewarmingPoolSize() uint { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetPrewarmingPoolSize(count uint) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) ApplyPrewarmingPoolSize() error { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) CPULimit() uint { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetCPULimit(limit uint) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) MemoryLimit() uint { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetMemoryLimit(limit uint) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) Image() string { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetImage(image string) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) NetworkAccess() (bool, []uint16) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetNetworkAccess(allow bool, ports []uint16) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) SetConfigFrom(environment runner.ExecutionEnvironment) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) Register() error { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) Delete(reason runner.DestroyReason) error { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) Sample() (r runner.Runner, ok bool) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) AddRunner(r runner.Runner) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) DeleteRunner(id string) (r runner.Runner, ok bool) { + //TODO implement me + panic("implement me") +} + +func (k KubernetesEnvironment) IdleRunnerCount() uint { + //TODO implement me + panic("implement me") +} + +func NewKubernetesEnvironmentFromRequest( + apiClient poseidonK8s.ExecutorAPI, jobHCL string, id dto.EnvironmentID, request dto.ExecutionEnvironmentRequest) ( + *KubernetesEnvironment, error) { + environment, err := NewKubernetesEnvironment(id, apiClient, jobHCL) + if err != nil { + return nil, err + } + environment.SetID(id) + + // Set options according to request + environment.SetPrewarmingPoolSize(request.PrewarmingPoolSize) + environment.SetCPULimit(request.CPULimit) + environment.SetMemoryLimit(request.MemoryLimit) + environment.SetImage(request.Image) + environment.SetNetworkAccess(request.NetworkAccess, request.ExposedPorts) + return environment, nil +} + +func NewKubernetesEnvironment(id dto.EnvironmentID, apiClient poseidonK8s.ExecutorAPI, jobHCL string) (*KubernetesEnvironment, error) { + job, err := parseDeployment(jobHCL) + if err != nil { + return nil, fmt.Errorf("error parsing Nomad job: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + e := &KubernetesEnvironment{&apiClient, jobHCL, job, nil, ctx, cancel} + e.idleRunners = storage.NewMonitoredLocalStorage[runner.Runner](monitoring.MeasurementIdleRunnerNomad, + runner.MonitorEnvironmentID[runner.Runner](id), time.Minute, ctx) + return e, nil +} + +// TODO MISSING IMPLEMENTATION +func parseDeployment(jobHCL string) (*appsv1.Deployment, error) { + + deployment := appsv1.Deployment{} + + // jobConfig := jobspec2.ParseConfig{ + // Body: []byte(jobHCL), + // AllowFS: false, + // Strict: true, + // } + // job, err := jobspec2.ParseWithConfig(&jobConfig) + // if err != nil { + // return job, fmt.Errorf("couldn't parse job HCL: %w", err) + // } + return &deployment, nil +} diff --git a/internal/environment/kubernetes_manager.go b/internal/environment/kubernetes_manager.go new file mode 100644 index 0000000..d65e80f --- /dev/null +++ b/internal/environment/kubernetes_manager.go @@ -0,0 +1,297 @@ +package environment + +import ( + "context" + "fmt" + poseidonK8s "github.com/openHPI/poseidon/internal/kubernetes" + "github.com/openHPI/poseidon/internal/nomad" + "github.com/openHPI/poseidon/internal/runner" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" + "github.com/openHPI/poseidon/pkg/monitoring" + "github.com/openHPI/poseidon/pkg/storage" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strconv" + "time" +) + +type KubernetesEnvironmentManager struct { + *AbstractManager + api poseidonK8s.ExecutorAPI + templateEnvironmentHCL string +} + +func NewKubernetesEnvironmentManager( + runnerManager runner.Manager, + apiClient *poseidonK8s.ExecutorAPI, + templateJobFile string, +) (*KubernetesEnvironmentManager, error) { + if err := loadTemplateEnvironmentJobHCL(templateJobFile); err != nil { + return nil, err + } + + m := &KubernetesEnvironmentManager{ + AbstractManager: &AbstractManager{nil, runnerManager}, + api: *apiClient, + templateEnvironmentHCL: templateEnvironmentJobHCL, + } + return m, nil +} + +func (k *KubernetesEnvironmentManager) SetNextHandler(next ManagerHandler) { + k.nextHandler = next +} + +func (k *KubernetesEnvironmentManager) NextHandler() ManagerHandler { + if k.HasNextHandler() { + return k.nextHandler + } else { + return &AbstractManager{} + } +} + +func (k *KubernetesEnvironmentManager) HasNextHandler() bool { + return k.nextHandler != nil +} + +// List all Kubernetes-based environments +func (k *KubernetesEnvironmentManager) List(fetch bool) ([]runner.ExecutionEnvironment, error) { + if fetch { + if err := k.fetchEnvironments(); err != nil { + return nil, err + } + } + return k.runnerManager.ListEnvironments(), nil +} + +func (k *KubernetesEnvironmentManager) fetchEnvironments() error { + remoteDeploymentResponse, err := k.api.LoadEnvironmentJobs() + if err != nil { + return fmt.Errorf("failed fetching environments: %w", err) + } + + remoteDeployments := make(map[string]appsv1.Deployment) + + // Update local environments from remote environments. + for _, deployment := range remoteDeploymentResponse { + remoteDeployments[deployment.Name] = *deployment + + // Job Id to Environment Id Integer + intIdentifier, err := strconv.Atoi(deployment.Name) + if err != nil { + log.WithError(err).Warn("Failed to convert job name to int") + continue + } + id := dto.EnvironmentID(intIdentifier) + + if localEnvironment, ok := k.runnerManager.GetEnvironment(id); ok { + fetchedEnvironment := newKubernetesEnvironmentFromJob(deployment, &k.api) + localEnvironment.SetConfigFrom(fetchedEnvironment) + // We destroy only this (second) local reference to the environment. + if err = fetchedEnvironment.Delete(runner.ErrDestroyedAndReplaced); err != nil { + log.WithError(err).Warn("Failed to remove environment locally") + } + } else { + k.runnerManager.StoreEnvironment(newKubernetesEnvironmentFromJob(deployment, &k.api)) + } + } + + // Remove local environments that are not remote environments. + for _, localEnvironment := range k.runnerManager.ListEnvironments() { + if _, ok := remoteDeployments[localEnvironment.ID().ToString()]; !ok { + err := localEnvironment.Delete(runner.ErrLocalDestruction) + log.WithError(err).Warn("Failed to remove environment locally") + } + } + return nil +} + +// newNomadEnvironmentFromJob creates a Nomad environment from the passed Nomad job definition. +func newKubernetesEnvironmentFromJob(deployment *appsv1.Deployment, apiClient *poseidonK8s.ExecutorAPI) *KubernetesEnvironment { + ctx, cancel := context.WithCancel(context.Background()) + e := &KubernetesEnvironment{ + apiClient: apiClient, + jobHCL: templateEnvironmentJobHCL, + deployment: deployment, + ctx: ctx, + cancel: cancel, + } + e.idleRunners = storage.NewMonitoredLocalStorage[runner.Runner](monitoring.MeasurementIdleRunnerNomad, + runner.MonitorEnvironmentID[runner.Runner](e.ID()), time.Minute, ctx) + return e +} + +// Get retrieves a specific Kubernetes environment +func (k *KubernetesEnvironmentManager) Get(id dto.EnvironmentID, fetch bool) (executionEnvironment runner.ExecutionEnvironment, err error) { + executionEnvironment, ok := k.runnerManager.GetEnvironment(id) + + if fetch { + fetchedEnvironment, err := fetchK8sEnvironment(id, k.api) + switch { + case err != nil: + return nil, err + case fetchedEnvironment == nil: + _, err = k.Delete(id) + if err != nil { + return nil, err + } + ok = false + case !ok: + k.runnerManager.StoreEnvironment(fetchedEnvironment) + executionEnvironment = fetchedEnvironment + ok = true + default: + executionEnvironment.SetConfigFrom(fetchedEnvironment) + // We destroy only this (second) local reference to the environment. + err = fetchedEnvironment.Delete(runner.ErrDestroyedAndReplaced) + if err != nil { + log.WithError(err).Warn("Failed to remove environment locally") + } + } + } + + if !ok { + err = runner.ErrUnknownExecutionEnvironment + } + return executionEnvironment, err +} + +// CreateOrUpdate creates or updates an environment in Kubernetes +func (k *KubernetesEnvironmentManager) CreateOrUpdate( + id dto.EnvironmentID, request dto.ExecutionEnvironmentRequest, ctx context.Context) (created bool, err error) { + + // Check if execution environment is already existing (in the local memory). + environment, isExistingEnvironment := k.runnerManager.GetEnvironment(id) + if isExistingEnvironment { + // Remove existing environment to force downloading the newest Docker image. + // See https://github.com/openHPI/poseidon/issues/69 + err = environment.Delete(runner.ErrEnvironmentUpdated) + if err != nil { + return false, fmt.Errorf("failed to remove the environment: %w", err) + } + } + + // Create a new environment with the given request options. + environment, err = NewKubernetesEnvironmentFromRequest(k.api, k.templateEnvironmentHCL, id, request) + if err != nil { + return false, fmt.Errorf("error creating Nomad environment: %w", err) + } + + // Keep a copy of environment specification in memory. + k.runnerManager.StoreEnvironment(environment) + + // Register template Job with Nomad. + logging.StartSpan("env.update.register", "Register Environment", ctx, func(_ context.Context) { + err = environment.Register() + }) + if err != nil { + return false, fmt.Errorf("error registering template job in API: %w", err) + } + + // Launch idle runners based on the template job. + logging.StartSpan("env.update.poolsize", "Apply Prewarming Pool Size", ctx, func(_ context.Context) { + err = environment.ApplyPrewarmingPoolSize() + }) + if err != nil { + return false, fmt.Errorf("error scaling template job in API: %w", err) + } + + return !isExistingEnvironment, nil + +} + +// Statistics fetches statistics from Kubernetes +func (k *KubernetesEnvironmentManager) Statistics() map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData { + // Collect and return statistics for Kubernetes environments + return map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData{} +} + +// MapExecutionEnvironmentRequestToDeployment maps ExecutionEnvironmentRequest to a Kubernetes Deployment +func MapExecutionEnvironmentRequestToDeployment(req dto.ExecutionEnvironmentRequest, environmentID string) *appsv1.Deployment { + // Create the Deployment object + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: environmentID, // Set the environment ID as the name of the deployment + Labels: map[string]string{ + "environment-id": environmentID, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(int32(req.PrewarmingPoolSize)), // Use PrewarmingPoolSize to set the number of replicas + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment-id": environmentID, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "environment-id": environmentID, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "runner-container", + Image: req.Image, // Map the image to the container + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse(strconv.Itoa(int(req.CPULimit))), // Map CPU request + "memory": resource.MustParse(strconv.Itoa(int(req.MemoryLimit)) + "Mi"), // Map Memory request + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse(strconv.Itoa(int(req.CPULimit))), // Map CPU limit + "memory": resource.MustParse(strconv.Itoa(int(req.MemoryLimit)) + "Mi"), // Map Memory limit + }, + }, + }, + }, + }, + }, + }, + } + + // Handle network access and exposed ports + if req.NetworkAccess { + var containerPorts []v1.ContainerPort + for _, port := range req.ExposedPorts { + containerPorts = append(containerPorts, v1.ContainerPort{ + ContainerPort: int32(port), + }) + } + deployment.Spec.Template.Spec.Containers[0].Ports = containerPorts + } + + return deployment +} + +// Helper function to return a pointer to an int32 +func int32Ptr(i int32) *int32 { + return &i +} + +func fetchK8sEnvironment(id dto.EnvironmentID, apiClient poseidonK8s.ExecutorAPI) (runner.ExecutionEnvironment, error) { + environments, err := apiClient.LoadEnvironmentJobs() + if err != nil { + return nil, fmt.Errorf("error fetching the environment jobs: %w", err) + } + var fetchedEnvironment runner.ExecutionEnvironment + for _, deployment := range environments { + environmentID, err := nomad.EnvironmentIDFromTemplateJobID(deployment.Name) + if err != nil { + log.WithError(err).Warn("Cannot parse environment id of loaded environment") + continue + } + if id == environmentID { + fetchedEnvironment = newKubernetesEnvironmentFromJob(deployment, &apiClient) + } + } + return fetchedEnvironment, nil +} diff --git a/internal/environment/manager_handler_mock.go b/internal/environment/manager_handler_mock.go deleted file mode 100644 index 455aa06..0000000 --- a/internal/environment/manager_handler_mock.go +++ /dev/null @@ -1,171 +0,0 @@ -// Code generated by mockery v2.16.0. DO NOT EDIT. - -package environment - -import ( - context "context" - - dto "github.com/openHPI/poseidon/pkg/dto" - mock "github.com/stretchr/testify/mock" - - runner "github.com/openHPI/poseidon/internal/runner" -) - -// ManagerHandlerMock is an autogenerated mock type for the ManagerHandler type -type ManagerHandlerMock struct { - mock.Mock -} - -// CreateOrUpdate provides a mock function with given fields: id, request, ctx -func (_m *ManagerHandlerMock) CreateOrUpdate(id dto.EnvironmentID, request dto.ExecutionEnvironmentRequest, ctx context.Context) (bool, error) { - ret := _m.Called(id, request, ctx) - - var r0 bool - if rf, ok := ret.Get(0).(func(dto.EnvironmentID, dto.ExecutionEnvironmentRequest, context.Context) bool); ok { - r0 = rf(id, request, ctx) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(dto.EnvironmentID, dto.ExecutionEnvironmentRequest, context.Context) error); ok { - r1 = rf(id, request, ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Delete provides a mock function with given fields: id -func (_m *ManagerHandlerMock) Delete(id dto.EnvironmentID) (bool, error) { - ret := _m.Called(id) - - var r0 bool - if rf, ok := ret.Get(0).(func(dto.EnvironmentID) bool); ok { - r0 = rf(id) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(dto.EnvironmentID) error); ok { - r1 = rf(id) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Get provides a mock function with given fields: id, fetch -func (_m *ManagerHandlerMock) Get(id dto.EnvironmentID, fetch bool) (runner.ExecutionEnvironment, error) { - ret := _m.Called(id, fetch) - - var r0 runner.ExecutionEnvironment - if rf, ok := ret.Get(0).(func(dto.EnvironmentID, bool) runner.ExecutionEnvironment); ok { - r0 = rf(id, fetch) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(runner.ExecutionEnvironment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(dto.EnvironmentID, bool) error); ok { - r1 = rf(id, fetch) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// HasNextHandler provides a mock function with given fields: -func (_m *ManagerHandlerMock) HasNextHandler() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// List provides a mock function with given fields: fetch -func (_m *ManagerHandlerMock) List(fetch bool) ([]runner.ExecutionEnvironment, error) { - ret := _m.Called(fetch) - - var r0 []runner.ExecutionEnvironment - if rf, ok := ret.Get(0).(func(bool) []runner.ExecutionEnvironment); ok { - r0 = rf(fetch) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]runner.ExecutionEnvironment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(bool) error); ok { - r1 = rf(fetch) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NextHandler provides a mock function with given fields: -func (_m *ManagerHandlerMock) NextHandler() ManagerHandler { - ret := _m.Called() - - var r0 ManagerHandler - if rf, ok := ret.Get(0).(func() ManagerHandler); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(ManagerHandler) - } - } - - return r0 -} - -// SetNextHandler provides a mock function with given fields: next -func (_m *ManagerHandlerMock) SetNextHandler(next ManagerHandler) { - _m.Called(next) -} - -// Statistics provides a mock function with given fields: -func (_m *ManagerHandlerMock) Statistics() map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData { - ret := _m.Called() - - var r0 map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData - if rf, ok := ret.Get(0).(func() map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData) - } - } - - return r0 -} - -type mockConstructorTestingTNewManagerHandlerMock interface { - mock.TestingT - Cleanup(func()) -} - -// NewManagerHandlerMock creates a new instance of ManagerHandlerMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewManagerHandlerMock(t mockConstructorTestingTNewManagerHandlerMock) *ManagerHandlerMock { - mock := &ManagerHandlerMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/environment/nomad_environment_test.go b/internal/environment/nomad_environment_test.go deleted file mode 100644 index dd71319..0000000 --- a/internal/environment/nomad_environment_test.go +++ /dev/null @@ -1,265 +0,0 @@ -package environment - -import ( - "context" - "fmt" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/storage" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "testing" - "time" -) - -func (s *MainTestSuite) TestConfigureNetworkCreatesNewNetworkWhenNoNetworkExists() { - _, job := helpers.CreateTemplateJob() - defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) - environment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil} - - if s.Equal(0, len(defaultTaskGroup.Networks)) { - environment.SetNetworkAccess(true, []uint16{}) - - s.Equal(1, len(defaultTaskGroup.Networks)) - } -} - -func (s *MainTestSuite) TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkExists() { - _, job := helpers.CreateTemplateJob() - defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) - environment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil} - - networkResource := config.Config.Nomad.Network - defaultTaskGroup.Networks = []*nomadApi.NetworkResource{&networkResource} - - if s.Equal(1, len(defaultTaskGroup.Networks)) { - environment.SetNetworkAccess(true, []uint16{}) - - s.Equal(1, len(defaultTaskGroup.Networks)) - s.Equal(&networkResource, defaultTaskGroup.Networks[0]) - } -} - -func (s *MainTestSuite) TestConfigureNetworkSetsCorrectValues() { - _, job := helpers.CreateTemplateJob() - defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) - defaultTask := nomad.FindAndValidateDefaultTask(defaultTaskGroup) - - mode, ok := defaultTask.Config["network_mode"] - s.True(ok) - s.Equal("none", mode) - s.Equal(0, len(defaultTaskGroup.Networks)) - - exposedPortsTests := [][]uint16{{}, {1337}, {42, 1337}} - s.Run("with no network access", func() { - for _, ports := range exposedPortsTests { - _, testJob := helpers.CreateTemplateJob() - testTaskGroup := nomad.FindAndValidateDefaultTaskGroup(testJob) - testTask := nomad.FindAndValidateDefaultTask(testTaskGroup) - testEnvironment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil} - - testEnvironment.SetNetworkAccess(false, ports) - mode, ok := testTask.Config["network_mode"] - s.True(ok) - s.Equal("none", mode) - s.Equal(0, len(testTaskGroup.Networks)) - } - }) - - s.Run("with network access", func() { - for _, ports := range exposedPortsTests { - _, testJob := helpers.CreateTemplateJob() - testTaskGroup := nomad.FindAndValidateDefaultTaskGroup(testJob) - testTask := nomad.FindAndValidateDefaultTask(testTaskGroup) - testEnvironment := &NomadEnvironment{nil, "", testJob, nil, context.Background(), nil} - - testEnvironment.SetNetworkAccess(true, ports) - s.Require().Equal(1, len(testTaskGroup.Networks)) - - networkResource := testTaskGroup.Networks[0] - s.Equal(config.Config.Nomad.Network.Mode, networkResource.Mode) - s.Require().Equal(len(ports), len(networkResource.DynamicPorts)) - - assertExpectedPorts(s.T(), ports, networkResource) - - mode, ok := testTask.Config["network_mode"] - s.True(ok) - s.Equal(mode, "") - } - }) -} - -func assertExpectedPorts(t *testing.T, expectedPorts []uint16, networkResource *nomadApi.NetworkResource) { - t.Helper() - for _, expectedPort := range expectedPorts { - found := false - for _, actualPort := range networkResource.DynamicPorts { - if actualPort.To == int(expectedPort) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("port list should contain %v", expectedPort)) - } -} - -func (s *MainTestSuite) TestRegisterFailsWhenNomadJobRegistrationFails() { - apiClientMock := &nomad.ExecutorAPIMock{} - expectedErr := tests.ErrDefault - - apiClientMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return("", expectedErr) - apiClientMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiClientMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - environment := &NomadEnvironment{apiClientMock, "", &nomadApi.Job{}, - storage.NewLocalStorage[runner.Runner](), nil, nil} - environment.SetID(tests.DefaultEnvironmentIDAsInteger) - err := environment.Register() - - s.ErrorIs(err, expectedErr) - apiClientMock.AssertNotCalled(s.T(), "MonitorEvaluation") -} - -func (s *MainTestSuite) TestRegisterTemplateJobSucceedsWhenMonitoringEvaluationSucceeds() { - apiClientMock := &nomad.ExecutorAPIMock{} - evaluationID := "id" - - apiClientMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return(evaluationID, nil) - apiClientMock.On("MonitorEvaluation", mock.AnythingOfType("string"), mock.Anything).Return(nil) - apiClientMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiClientMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - environment := &NomadEnvironment{apiClientMock, "", &nomadApi.Job{}, - storage.NewLocalStorage[runner.Runner](), context.Background(), nil} - environment.SetID(tests.DefaultEnvironmentIDAsInteger) - err := environment.Register() - - s.NoError(err) -} - -func (s *MainTestSuite) TestRegisterTemplateJobReturnsErrorWhenMonitoringEvaluationFails() { - apiClientMock := &nomad.ExecutorAPIMock{} - evaluationID := "id" - - apiClientMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return(evaluationID, nil) - apiClientMock.On("MonitorEvaluation", mock.AnythingOfType("string"), mock.Anything).Return(tests.ErrDefault) - apiClientMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiClientMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - environment := &NomadEnvironment{apiClientMock, "", &nomadApi.Job{}, - storage.NewLocalStorage[runner.Runner](), context.Background(), nil} - environment.SetID(tests.DefaultEnvironmentIDAsInteger) - err := environment.Register() - - s.ErrorIs(err, tests.ErrDefault) -} - -func (s *MainTestSuite) TestParseJob() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.Run("parses the given default job", func() { - environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.NoError(err) - s.NotNil(environment.job) - s.NoError(environment.Delete(tests.ErrCleanupDestroyReason)) - }) - - s.Run("returns error when given wrong job", func() { - environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, nil, "") - s.Error(err) - s.Nil(environment) - }) -} - -func (s *MainTestSuite) TestTwoSampleAddExactlyTwoRunners() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("RegisterRunnerJob", mock.AnythingOfType("*api.Job")).Return(nil) - - _, job := helpers.CreateTemplateJob() - environment := &NomadEnvironment{apiMock, templateEnvironmentJobHCL, job, - storage.NewLocalStorage[runner.Runner](), context.Background(), nil} - environment.SetPrewarmingPoolSize(2) - runner1 := &runner.RunnerMock{} - runner1.On("ID").Return(tests.DefaultRunnerID) - runner2 := &runner.RunnerMock{} - runner2.On("ID").Return(tests.AnotherRunnerID) - - environment.AddRunner(runner1) - environment.AddRunner(runner2) - - _, ok := environment.Sample() - s.Require().True(ok) - _, ok = environment.Sample() - s.Require().True(ok) - - <-time.After(tests.ShortTimeout) // New Runners are requested asynchronously - apiMock.AssertNumberOfCalls(s.T(), "RegisterRunnerJob", 2) -} - -func (s *MainTestSuite) TestSampleDoesNotSetForcePullFlag() { - apiMock := &nomad.ExecutorAPIMock{} - call := apiMock.On("RegisterRunnerJob", mock.AnythingOfType("*api.Job")) - call.Run(func(args mock.Arguments) { - job, ok := args.Get(0).(*nomadApi.Job) - s.True(ok) - - taskGroup := nomad.FindAndValidateDefaultTaskGroup(job) - task := nomad.FindAndValidateDefaultTask(taskGroup) - s.False(task.Config["force_pull"].(bool)) - - call.ReturnArguments = mock.Arguments{nil} - }) - - _, job := helpers.CreateTemplateJob() - environment := &NomadEnvironment{apiMock, templateEnvironmentJobHCL, job, - storage.NewLocalStorage[runner.Runner](), s.TestCtx, nil} - runner1 := &runner.RunnerMock{} - runner1.On("ID").Return(tests.DefaultRunnerID) - environment.AddRunner(runner1) - - _, ok := environment.Sample() - s.Require().True(ok) - <-time.After(tests.ShortTimeout) // New Runners are requested asynchronously -} - -func (s *MainTestSuite) TestNomadEnvironment_DeleteLocally() { - apiMock := &nomad.ExecutorAPIMock{} - environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - - err = environment.Delete(runner.ErrLocalDestruction) - s.NoError(err) - apiMock.AssertExpectations(s.T()) -} - -func (s *MainTestSuite) TestNomadEnvironment_AddRunner() { - s.Run("Destroys runner before replacing it", func() { - apiMock := &nomad.ExecutorAPIMock{} - environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - r := &runner.RunnerMock{} - r.On("ID").Return(tests.DefaultRunnerID) - r.On("Destroy", mock.Anything).Run(func(args mock.Arguments) { - err, ok := args[0].(error) - s.Require().True(ok) - s.ErrorIs(err, runner.ErrLocalDestruction) - }).Return(nil).Once() - r2 := &runner.RunnerMock{} - r2.On("ID").Return(tests.DefaultRunnerID) - - environment.AddRunner(r) - environment.AddRunner(r2) - r.AssertExpectations(s.T()) - - // Teardown test case - r2.On("Destroy", mock.Anything).Return(nil) - apiMock.On("LoadRunnerIDs", mock.Anything).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.Anything).Return(nil) - s.NoError(environment.Delete(tests.ErrCleanupDestroyReason)) - }) -} diff --git a/internal/environment/nomad_manager_test.go b/internal/environment/nomad_manager_test.go deleted file mode 100644 index 040f73a..0000000 --- a/internal/environment/nomad_manager_test.go +++ /dev/null @@ -1,455 +0,0 @@ -package environment - -import ( - "context" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/internal/runner" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "os" - "testing" - "time" -) - -type CreateOrUpdateTestSuite struct { - tests.MemoryLeakTestSuite - runnerManagerMock runner.ManagerMock - apiMock nomad.ExecutorAPIMock - request dto.ExecutionEnvironmentRequest - manager *NomadEnvironmentManager - environmentID dto.EnvironmentID -} - -func TestCreateOrUpdateTestSuite(t *testing.T) { - suite.Run(t, new(CreateOrUpdateTestSuite)) -} - -func (s *CreateOrUpdateTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.runnerManagerMock = runner.ManagerMock{} - - s.apiMock = nomad.ExecutorAPIMock{} - s.request = dto.ExecutionEnvironmentRequest{ - PrewarmingPoolSize: 10, - CPULimit: 20, - MemoryLimit: 30, - Image: "my-image", - NetworkAccess: false, - ExposedPorts: nil, - } - - s.manager = &NomadEnvironmentManager{ - AbstractManager: &AbstractManager{runnerManager: &s.runnerManagerMock}, - api: &s.apiMock, - templateEnvironmentHCL: templateEnvironmentJobHCL, - } - - s.environmentID = dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger) -} - -func (s *CreateOrUpdateTestSuite) TestReturnsErrorIfCreatesOrUpdateEnvironmentReturnsError() { - s.apiMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return("", tests.ErrDefault) - s.apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.runnerManagerMock.On("GetEnvironment", mock.AnythingOfType("dto.EnvironmentID")).Return(nil, false) - s.runnerManagerMock.On("StoreEnvironment", mock.AnythingOfType("*environment.NomadEnvironment")).Return(true) - s.ExpectedGoroutineIncrease++ // We don't care about removing the created environment. - _, err := s.manager.CreateOrUpdate( - dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger), s.request, context.Background()) - s.ErrorIs(err, tests.ErrDefault) -} - -func (s *CreateOrUpdateTestSuite) TestCreateOrUpdatesSetsForcePullFlag() { - s.apiMock.On("RegisterNomadJob", mock.AnythingOfType("*api.Job")).Return("", nil) - s.apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.runnerManagerMock.On("GetEnvironment", mock.AnythingOfType("dto.EnvironmentID")).Return(nil, false) - s.runnerManagerMock.On("StoreEnvironment", mock.AnythingOfType("*environment.NomadEnvironment")).Return(true) - s.apiMock.On("MonitorEvaluation", mock.AnythingOfType("string"), mock.Anything).Return(nil) - s.apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - call := s.apiMock.On("RegisterRunnerJob", mock.AnythingOfType("*api.Job")) - count := 0 - call.Run(func(args mock.Arguments) { - count++ - job, ok := args.Get(0).(*nomadApi.Job) - s.True(ok) - - // The environment job itself has not the force_pull flag - if count > 1 { - taskGroup := nomad.FindAndValidateDefaultTaskGroup(job) - task := nomad.FindAndValidateDefaultTask(taskGroup) - s.True(task.Config["force_pull"].(bool)) - } - - call.ReturnArguments = mock.Arguments{nil} - }) - s.ExpectedGoroutineIncrease++ // We dont care about removing the created environment at this point. - _, err := s.manager.CreateOrUpdate( - dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger), s.request, context.Background()) - s.NoError(err) - s.True(count > 1) -} - -func (s *MainTestSuite) TestNewNomadEnvironmentManager() { - executorAPIMock := &nomad.ExecutorAPIMock{} - executorAPIMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil) - executorAPIMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - executorAPIMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - runnerManagerMock := &runner.ManagerMock{} - runnerManagerMock.On("Load").Return() - - previousTemplateEnvironmentJobHCL := templateEnvironmentJobHCL - - s.Run("returns error if template file does not exist", func() { - _, err := NewNomadEnvironmentManager(runnerManagerMock, executorAPIMock, "/non-existent/file") - s.Error(err) - }) - - s.Run("loads template environment job from file", func() { - templateJobHCL := "job \"" + tests.DefaultTemplateJobID + "\" {}" - - environment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, executorAPIMock, templateJobHCL) - s.Require().NoError(err) - f := createTempFile(s.T(), templateJobHCL) - defer os.Remove(f.Name()) - - m, err := NewNomadEnvironmentManager(runnerManagerMock, executorAPIMock, f.Name()) - s.NoError(err) - s.NotNil(m) - s.Equal(templateJobHCL, m.templateEnvironmentHCL) - - s.NoError(environment.Delete(tests.ErrCleanupDestroyReason)) - }) - - s.Run("returns error if template file is invalid", func() { - templateJobHCL := "invalid hcl file" - f := createTempFile(s.T(), templateJobHCL) - defer os.Remove(f.Name()) - - m, err := NewNomadEnvironmentManager(runnerManagerMock, executorAPIMock, f.Name()) - s.Require().NoError(err) - _, err = NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, nil, m.templateEnvironmentHCL) - s.Error(err) - }) - - templateEnvironmentJobHCL = previousTemplateEnvironmentJobHCL -} - -func (s *MainTestSuite) TestNomadEnvironmentManager_Get() { - apiMock := &nomad.ExecutorAPIMock{} - mockWatchAllocations(s.TestCtx, apiMock) - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - call := apiMock.On("LoadEnvironmentJobs") - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]*nomadApi.Job{}, nil} - }) - - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - s.Run("Returns error when not found", func() { - _, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.Error(err) - }) - - s.Run("Returns environment when it was added before", func() { - expectedEnvironment, err := - NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - expectedEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - s.Require().NoError(err) - runnerManager.StoreEnvironment(expectedEnvironment) - - environment, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.NoError(err) - s.Equal(expectedEnvironment, environment) - - err = environment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) - - s.Run("Fetch", func() { - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.Run("Returns error when not found", func() { - _, err := m.Get(tests.DefaultEnvironmentIDAsInteger, true) - s.Error(err) - }) - - s.Run("Updates values when environment already known by Poseidon", func() { - fetchedEnvironment, err := NewNomadEnvironment( - tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - fetchedEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - fetchedEnvironment.SetImage("random docker image") - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]*nomadApi.Job{fetchedEnvironment.job}, nil} - }) - - localEnvironment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - localEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - runnerManager.StoreEnvironment(localEnvironment) - - environment, err := m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.NoError(err) - s.NotEqual(fetchedEnvironment.Image(), environment.Image()) - - environment, err = m.Get(tests.DefaultEnvironmentIDAsInteger, true) - s.NoError(err) - s.Equal(fetchedEnvironment.Image(), environment.Image()) - - err = fetchedEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - err = environment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - err = localEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) - runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) - - s.Run("Adds environment when not already known by Poseidon", func() { - fetchedEnvironment, err := NewNomadEnvironment( - tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - fetchedEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - fetchedEnvironment.SetImage("random docker image") - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]*nomadApi.Job{fetchedEnvironment.job}, nil} - }) - - _, err = m.Get(tests.DefaultEnvironmentIDAsInteger, false) - s.Error(err) - - environment, err := m.Get(tests.DefaultEnvironmentIDAsInteger, true) - s.NoError(err) - s.Equal(fetchedEnvironment.Image(), environment.Image()) - - err = fetchedEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - err = environment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) - }) -} - -func (s *MainTestSuite) TestNomadEnvironmentManager_List() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - mockWatchAllocations(s.TestCtx, apiMock) - call := apiMock.On("LoadEnvironmentJobs") - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]*nomadApi.Job{}, nil} - }) - - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - s.Run("with no environments", func() { - environments, err := m.List(true) - s.NoError(err) - s.Empty(environments) - }) - - s.Run("Returns added environment", func() { - localEnvironment, err := NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - localEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - runnerManager.StoreEnvironment(localEnvironment) - - environments, err := m.List(false) - s.NoError(err) - s.Equal(1, len(environments)) - s.Equal(localEnvironment, environments[0]) - - err = localEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) - runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) - - s.Run("Fetches new Runners via the api client", func() { - fetchedEnvironment, err := - NewNomadEnvironment(tests.DefaultEnvironmentIDAsInteger, apiMock, templateEnvironmentJobHCL) - s.Require().NoError(err) - fetchedEnvironment.SetID(tests.DefaultEnvironmentIDAsInteger) - status := structs.JobStatusRunning - fetchedEnvironment.job.Status = &status - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{[]*nomadApi.Job{fetchedEnvironment.job}, nil} - }) - - environments, err := m.List(false) - s.NoError(err) - s.Empty(environments) - - environments, err = m.List(true) - s.NoError(err) - s.Equal(1, len(environments)) - nomadEnvironment, ok := environments[0].(*NomadEnvironment) - s.True(ok) - s.Equal(fetchedEnvironment.job, nomadEnvironment.job) - - err = fetchedEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - err = nomadEnvironment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) -} - -func (s *MainTestSuite) TestNomadEnvironmentManager_Load() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("LoadRunnerIDs", mock.AnythingOfType("string")).Return([]string{}, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - mockWatchAllocations(s.TestCtx, apiMock) - call := apiMock.On("LoadEnvironmentJobs") - apiMock.On("LoadRunnerJobs", mock.AnythingOfType("dto.EnvironmentID")). - Return([]*nomadApi.Job{}, nil) - - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - - s.Run("deletes local environments before loading Nomad environments", func() { - call.Return([]*nomadApi.Job{}, nil) - environment := &runner.ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - environment.On("Image").Return("") - environment.On("CPULimit").Return(uint(0)) - environment.On("MemoryLimit").Return(uint(0)) - environment.On("NetworkAccess").Return(false, nil) - environment.On("Delete", mock.Anything).Return(nil) - runnerManager.StoreEnvironment(environment) - - m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - err = m.load() - s.Require().NoError(err) - environment.AssertExpectations(s.T()) - }) - runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) - - s.Run("Stores fetched environments", func() { - _, job := helpers.CreateTemplateJob() - call.Return([]*nomadApi.Job{job}, nil) - - _, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.Require().False(ok) - - m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - err = m.load() - s.Require().NoError(err) - - environment, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.Require().True(ok) - s.Equal("python:latest", environment.Image()) - - err = environment.Delete(tests.ErrCleanupDestroyReason) - s.Require().NoError(err) - }) - - runnerManager.DeleteEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.Run("Processes only running environments", func() { - _, job := helpers.CreateTemplateJob() - jobStatus := structs.JobStatusDead - job.Status = &jobStatus - call.Return([]*nomadApi.Job{job}, nil) - - _, ok := runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.Require().False(ok) - - _, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - _, ok = runnerManager.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.Require().False(ok) - }) -} - -func (s *MainTestSuite) TestNomadEnvironmentManager_KeepEnvironmentsSynced() { - apiMock := &nomad.ExecutorAPIMock{} - runnerManager := runner.NewNomadRunnerManager(apiMock, s.TestCtx) - m, err := NewNomadEnvironmentManager(runnerManager, apiMock, "") - s.Require().NoError(err) - - s.Run("stops when context is done", func() { - apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, context.DeadlineExceeded) - ctx, cancel := context.WithCancel(s.TestCtx) - cancel() - - var done bool - go func() { - <-time.After(tests.ShortTimeout) - if !done { - s.FailNow("KeepEnvironmentsSynced is ignoring the context") - } - }() - - m.KeepEnvironmentsSynced(func(_ context.Context) error { return nil }, ctx) - done = true - }) - apiMock.ExpectedCalls = []*mock.Call{} - apiMock.Calls = []mock.Call{} - - s.Run("retries loading environments", func() { - ctx, cancel := context.WithCancel(s.TestCtx) - - apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, context.DeadlineExceeded).Once() - apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil).Run(func(_ mock.Arguments) { - cancel() - }).Once() - - m.KeepEnvironmentsSynced(func(_ context.Context) error { return nil }, ctx) - apiMock.AssertExpectations(s.T()) - }) - apiMock.ExpectedCalls = []*mock.Call{} - apiMock.Calls = []mock.Call{} - - s.Run("retries synchronizing runners", func() { - apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil) - ctx, cancel := context.WithCancel(s.TestCtx) - - count := 0 - synchronizeRunners := func(ctx context.Context) error { - count++ - if count >= 2 { - cancel() - return nil - } - return context.DeadlineExceeded - } - m.KeepEnvironmentsSynced(synchronizeRunners, ctx) - - if count < 2 { - s.Fail("KeepEnvironmentsSynced is not retrying to synchronize the runners") - } - }) -} - -func mockWatchAllocations(ctx context.Context, apiMock *nomad.ExecutorAPIMock) { - call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) - call.Run(func(args mock.Arguments) { - <-ctx.Done() - call.ReturnArguments = mock.Arguments{nil} - }) -} - -func createTempFile(t *testing.T, content string) *os.File { - t.Helper() - f, err := os.CreateTemp("", "test") - require.NoError(t, err) - n, err := f.WriteString(content) - require.NoError(t, err) - require.Equal(t, len(content), n) - - return f -} diff --git a/internal/kubernetes/api_querier.go b/internal/kubernetes/api_querier.go new file mode 100644 index 0000000..7ba1370 --- /dev/null +++ b/internal/kubernetes/api_querier.go @@ -0,0 +1,122 @@ +package kubernetes + +import ( + "context" + "errors" + nomadApi "github.com/hashicorp/nomad/api" + "io" + appv1 "k8s.io/api/apps/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + ErrorNoAllocationFound = errors.New("no allocation found") +) + +// apiQuerier provides access to the Nomad functionality. +type apiQuerier interface { + // init prepares an apiClient to be able to communicate to a provided Nomad API. + init(nomadConfig *rest.Config) (err error) + + // LoadJobList loads the list of jobs from the Nomad API. + LoadJobList() (list []*appv1.DeploymentList, err error) + + // JobScale returns the scale of the passed job. + JobScale(jobID string) (jobScale uint, err error) + + // SetJobScale sets the scaling count of the passed job to Nomad. + SetJobScale(jobID string, count uint, reason string) (err error) + + // DeleteJob deletes the Job with the given ID. + DeleteDeployment(name string) (err error) + + // Execute runs a command in the passed job. + Execute(jobID string, ctx context.Context, command string, tty bool, + stdin io.Reader, stdout, stderr io.Writer) (int, error) + + // listJobs loads all jobs with the specified prefix. + listDeployments(namespace string) (jobListStub []*appv1.DeploymentList, err error) + + // job returns the job of the given jobID. + deployment(name string) (deployment appv1.Deployment, err error) + + // listAllocations loads all allocations. + listAllocations() (allocationListStub []*nomadApi.AllocationListStub, err error) + + // allocation returns the first allocation of the given job. + allocation(jobID string) (*nomadApi.Allocation, error) + + // RegisterKubernetesDeployment registers a deployment with Kubernetes. + // It returns the deployment ID that can be used when listening to the Kubernetes event stream. + RegisterKubernetesDeployment(deployment appv1.Deployment) (string, error) + + // EventStream returns a Nomad event stream filtered to return only allocation and evaluation events. + EventStream(ctx context.Context) (<-chan *nomadApi.Events, error) +} + +// nomadAPIClient implements the nomadApiQuerier interface and provides access to a real Nomad API. +type kubernetesAPIClient struct { + client *kubernetes.Clientset + namespace string +} + +func (k kubernetesAPIClient) init(nomadConfig *rest.Config) (err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) LoadJobList() (list []*appv1.DeploymentList, err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) JobScale(jobID string) (jobScale uint, err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) SetJobScale(jobID string, count uint, reason string) (err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) DeleteDeployment(name string) (err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) Execute(jobID string, ctx context.Context, command string, tty bool, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) listDeployments(namespace string) (jobListStub []*appv1.DeploymentList, err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) deployment(name string) (deployment appv1.Deployment, err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) listAllocations() (allocationListStub []*nomadApi.AllocationListStub, err error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) allocation(jobID string) (*nomadApi.Allocation, error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) RegisterKubernetesDeployment(deployment appv1.Deployment) (string, error) { + //TODO implement me + panic("implement me") +} + +func (k kubernetesAPIClient) EventStream(ctx context.Context) (<-chan *nomadApi.Events, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/kubernetes/deployment.go b/internal/kubernetes/deployment.go new file mode 100644 index 0000000..c2ae28c --- /dev/null +++ b/internal/kubernetes/deployment.go @@ -0,0 +1,125 @@ +package kubernetes + +import ( + "context" + "errors" + "fmt" + nomadApi "github.com/hashicorp/nomad/api" + "github.com/openHPI/poseidon/pkg/dto" + appv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "strconv" + "strings" + "time" +) + +const ( + TemplateJobPrefix = "template" + TaskGroupName = "default-group" + TaskName = "default-task" + TaskCount = 1 + TaskDriver = "docker" + TaskCommand = "sleep" + ConfigTaskGroupName = "config" + ConfigTaskName = "config" + ConfigTaskDriver = "exec" + ConfigTaskCommand = "true" + ConfigMetaUsedKey = "used" + ConfigMetaUsedValue = "true" + ConfigMetaUnusedValue = "false" + ConfigMetaTimeoutKey = "timeout" + ConfigMetaPoolSizeKey = "prewarmingPoolSize" + TemplateJobNameParts = 2 + RegisterTimeout = 10 * time.Second + RunnerTimeoutFallback = 60 * time.Second +) + +var ( + ErrorInvalidJobID = errors.New("invalid job id") + ErrorMissingTaskGroup = errors.New("couldn't find config task group in job") + TaskArgs = []string{"infinity"} +) + +func (a *APIClient) RegisterRunnerJob(template *appv1.Deployment) error { + evalID, err := a.apiQuerier.RegisterKubernetesDeployment(*template) + if err != nil { + return fmt.Errorf("couldn't register runner job: %w", err) + } + + registerTimeout, cancel := context.WithTimeout(context.Background(), RegisterTimeout) + defer cancel() + return a.MonitorEvaluation(evalID, registerTimeout) +} + +// SetForcePullFlag sets the flag of a job if the image should be pulled again. +func SetForcePullFlag(deployment *appv1.Deployment, value bool) { + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == TaskName { + if value { + container.ImagePullPolicy = v1.PullAlways + } else { + container.ImagePullPolicy = v1.PullIfNotPresent + } + } + } +} + +// IsEnvironmentTemplateID checks if the passed job id belongs to a template job. +func IsEnvironmentTemplateID(jobID string) bool { + parts := strings.Split(jobID, "-") + if len(parts) != TemplateJobNameParts || parts[0] != TemplateJobPrefix { + return false + } + + _, err := EnvironmentIDFromTemplateJobID(jobID) + return err == nil +} + +// RunnerJobID returns the nomad job id of the runner with the given environmentID and id. +func RunnerJobID(environmentID dto.EnvironmentID, id string) string { + return fmt.Sprintf("%d-%s", environmentID, id) +} + +// TemplateJobID returns the id of the nomad job for the environment with the given id. +func TemplateJobID(id dto.EnvironmentID) string { + return fmt.Sprintf("%s-%d", TemplateJobPrefix, id) +} + +// EnvironmentIDFromRunnerID returns the environment id that is part of the passed runner job id. +func EnvironmentIDFromRunnerID(jobID string) (dto.EnvironmentID, error) { + return partOfJobID(jobID, 0) +} + +// EnvironmentIDFromTemplateJobID returns the environment id that is part of the passed environment job id. +func EnvironmentIDFromTemplateJobID(id string) (dto.EnvironmentID, error) { + return partOfJobID(id, 1) +} + +func partOfJobID(id string, part uint) (dto.EnvironmentID, error) { + parts := strings.Split(id, "-") + if len(parts) == 0 { + return 0, fmt.Errorf("empty job id: %w", ErrorInvalidJobID) + } + environmentID, err := strconv.Atoi(parts[part]) + if err != nil { + return 0, fmt.Errorf("invalid environment id par %v: %w", err, ErrorInvalidJobID) + } + return dto.EnvironmentID(environmentID), nil +} + +func isOOMKilled(alloc *nomadApi.Allocation) bool { + state, ok := alloc.TaskStates[TaskName] + if !ok { + return false + } + + var oomKilledCount uint64 + for _, event := range state.Events { + if oomString, ok := event.Details["oom_killed"]; ok { + if oom, err := strconv.ParseBool(oomString); err == nil && oom { + oomKilledCount++ + } + } + } + return oomKilledCount >= state.Restarts +} diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go new file mode 100644 index 0000000..f7795f8 --- /dev/null +++ b/internal/kubernetes/kubernetes.go @@ -0,0 +1,221 @@ +package kubernetes + +import ( + "context" + "errors" + "fmt" + nomadApi "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/influxdata/influxdb-client-go/v2/api/write" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/logging" + "github.com/openHPI/poseidon/pkg/monitoring" + "github.com/openHPI/poseidon/pkg/storage" + "io" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" + "strings" + "time" +) + +var ( + log = logging.GetLogger("kubernetes") + ErrorExecutorCommunicationFailed = errors.New("communication with executor failed") + ErrorEvaluation = errors.New("evaluation could not complete") + ErrorPlacingAllocations = errors.New("failed to place all allocations") + ErrorLoadingJob = errors.New("failed to load job") + ErrorNoAllocatedResourcesFound = errors.New("no allocated resources found") + ErrorLocalDestruction RunnerDeletedReason = errors.New("the destruction should not cause external changes") + ErrorOOMKilled RunnerDeletedReason = fmt.Errorf("%s: %w", dto.ErrOOMKilled.Error(), ErrorLocalDestruction) + ErrorAllocationRescheduled RunnerDeletedReason = fmt.Errorf("the allocation was rescheduled: %w", ErrorLocalDestruction) + ErrorAllocationStopped RunnerDeletedReason = errors.New("the allocation was stopped") + ErrorAllocationStoppedUnexpectedly RunnerDeletedReason = fmt.Errorf("%w unexpectedly", ErrorAllocationStopped) + ErrorAllocationRescheduledUnexpectedly RunnerDeletedReason = fmt.Errorf( + "%w correctly but rescheduled", ErrorAllocationStopped) + // ErrorAllocationCompleted is for reporting the reason for the stopped allocation. + // We do not consider it as an error but add it anyway for a complete reporting. + ErrorAllocationCompleted RunnerDeletedReason = errors.New("the allocation completed") +) + +type allocationData struct { + // allocClientStatus defines the state defined by Nomad. + allocClientStatus string + // allocDesiredStatus defines if the allocation wants to be running or being stopped. + allocDesiredStatus string + jobID string + start time.Time + // stopExpected is used to suppress warnings that could be triggered by a race condition + // between the Inactivity timer and an external event leadng to allocation rescheduling. + stopExpected bool + // Just debugging information + allocNomadNode string +} + +// resultChannelWriteTimeout is to detect the error when more element are written into a channel than expected. +const resultChannelWriteTimeout = 10 * time.Millisecond + +type DeletedAllocationProcessor func(jobID string, RunnerDeletedReason error) (removedByPoseidon bool) +type NewAllocationProcessor func(*nomadApi.Allocation, time.Duration) + +// AllocationProcessing includes the callbacks to interact with allocation events. +type AllocationProcessing struct { + OnNew NewAllocationProcessor + OnDeleted DeletedAllocationProcessor +} +type RunnerDeletedReason error + +// ExecutorAPI provides access to a container orchestration solution. +type ExecutorAPI interface { + apiQuerier + + // LoadEnvironmentJobs loads all environment jobs. + LoadEnvironmentJobs() ([]*appsv1.Deployment, error) + + // LoadRunnerJobs loads all runner jobs specific for the environment. + LoadRunnerJobs(environmentID dto.EnvironmentID) ([]*appsv1.Deployment, error) + + // LoadRunnerIDs returns the IDs of all runners with the specified id prefix which are not about to + // get stopped. + LoadRunnerIDs(prefix string) (runnerIds []string, err error) + + // LoadRunnerPortMappings returns the mapped ports of the runner. + LoadRunnerPortMappings(runnerID string) ([]v1.ContainerPort, error) + + // RegisterRunnerJob creates a runner job based on the template job. + // It registers the job and waits until the registration completes. + RegisterRunnerJob(template *appsv1.Deployment) error + + // MonitorEvaluation monitors the given evaluation ID. + // It waits until the evaluation reaches one of the states complete, canceled or failed. + // If the evaluation was not successful, an error containing the failures is returned. + // See also https://github.com/hashicorp/nomad/blob/7d5a9ecde95c18da94c9b6ace2565afbfdd6a40d/command/monitor.go#L175 + MonitorEvaluation(evaluationID string, ctx context.Context) error + + // WatchEventStream listens on the Nomad event stream for allocation and evaluation events. + // Depending on the incoming event, any of the given function is executed. + // Do not run multiple times simultaneously. + WatchEventStream(ctx context.Context, callbacks *AllocationProcessing) error + + // ExecuteCommand executes the given command in the job/runner with the given id. + // It writes the output of the command to stdout/stderr and reads input from stdin. + // If tty is true, the command will run with a tty. + // Iff privilegedExecution is true, the command will be executed privileged. + // The command is passed in the shell form (not the exec array form) and will be executed in a shell. + ExecuteCommand(jobID string, ctx context.Context, command string, tty bool, privilegedExecution bool, + stdin io.Reader, stdout, stderr io.Writer) (int, error) + + // MarkRunnerAsUsed marks the runner with the given ID as used. It also stores the timeout duration in the metadata. + MarkRunnerAsUsed(runnerID string, duration int) error +} + +// APIClient implements the ExecutorAPI interface and can be used to perform different operations on the real +// Executor API and its return values. +type APIClient struct { + apiQuerier + evaluations storage.Storage[chan error] + // allocations contain management data for all pending and running allocations. + allocations storage.Storage[*allocationData] + isListening bool +} + +func (A APIClient) LoadEnvironmentJobs() ([]*appsv1.Deployment, error) { + //TODO implement me + panic("implement me") +} + +func (a *APIClient) LoadRunnerJobs(environmentID dto.EnvironmentID) ([]*appsv1.Deployment, error) { + go a.initializeAllocations(environmentID) + + runnerIDs, err := a.LoadRunnerIDs(RunnerJobID(environmentID, "")) + if err != nil { + return []*appsv1.Deployment{}, fmt.Errorf("couldn't load jobs: %w", err) + } + + var occurredError error + jobs := make([]*appsv1.Deployment, 0, len(runnerIDs)) + for _, id := range runnerIDs { + job, err := a.apiQuerier.deployment(id) + if err != nil { + if occurredError == nil { + occurredError = ErrorLoadingJob + } + occurredError = fmt.Errorf("%w: couldn't load job info for runner %s - %v", occurredError, id, err) + continue + } + jobs = append(jobs, &job) + } + return jobs, occurredError +} + +func (A APIClient) LoadRunnerIDs(prefix string) (runnerIds []string, err error) { + //TODO implement me + panic("implement me") +} + +func (A APIClient) LoadRunnerPortMappings(runnerID string) ([]v1.ContainerPort, error) { + //TODO implement me + panic("implement me") +} + +func (A APIClient) MonitorEvaluation(evaluationID string, ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (A APIClient) WatchEventStream(ctx context.Context, callbacks *AllocationProcessing) error { + //TODO implement me + panic("implement me") +} + +func (A APIClient) ExecuteCommand(jobID string, ctx context.Context, command string, tty bool, privilegedExecution bool, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + //TODO implement me + panic("implement me") +} + +func (A APIClient) MarkRunnerAsUsed(runnerID string, duration int) error { + //TODO implement me + panic("implement me") +} + +// NewExecutorAPI creates a new api client. +// One client is usually sufficient for the complete runtime of the API. +func NewExecutorAPI(kubernetesConfig *rest.Config) (ExecutorAPI, error) { + client := &APIClient{ + apiQuerier: &kubernetesAPIClient{}, + evaluations: storage.NewLocalStorage[chan error](), + allocations: storage.NewMonitoredLocalStorage[*allocationData](monitoring.MeasurementNomadAllocations, + func(p *write.Point, object *allocationData, _ storage.EventType) { + p.AddTag(monitoring.InfluxKeyJobID, object.jobID) + p.AddTag(monitoring.InfluxKeyClientStatus, object.allocClientStatus) + p.AddTag(monitoring.InfluxKeyNomadNode, object.allocNomadNode) + }, 0, nil), + } + err := client.init(kubernetesConfig) + return client, err +} + +func (a *APIClient) initializeAllocations(environmentID dto.EnvironmentID) { + allocationStubs, err := a.listAllocations() + if err != nil { + log.WithError(err).Warn("Could not initialize allocations") + } else { + for _, stub := range allocationStubs { + switch { + case IsEnvironmentTemplateID(stub.JobID): + continue + case !strings.HasPrefix(stub.JobID, RunnerJobID(environmentID, "")): + continue + case stub.ClientStatus == structs.AllocClientStatusPending || stub.ClientStatus == structs.AllocClientStatusRunning: + log.WithField("jobID", stub.JobID).WithField("status", stub.ClientStatus).Debug("Recovered Allocation") + a.allocations.Add(stub.ID, &allocationData{ + allocClientStatus: stub.ClientStatus, + allocDesiredStatus: stub.DesiredStatus, + jobID: stub.JobID, + start: time.Unix(0, stub.CreateTime), + allocNomadNode: stub.NodeName, + }) + } + } + } +} diff --git a/internal/nomad/api_querier_mock.go b/internal/nomad/api_querier_mock.go deleted file mode 100644 index 728e842..0000000 --- a/internal/nomad/api_querier_mock.go +++ /dev/null @@ -1,304 +0,0 @@ -// Code generated by mockery v2.23.1. DO NOT EDIT. - -package nomad - -import ( - context "context" - - api "github.com/hashicorp/nomad/api" - config "github.com/openHPI/poseidon/internal/config" - - io "io" - - mock "github.com/stretchr/testify/mock" -) - -// apiQuerierMock is an autogenerated mock type for the apiQuerier type -type apiQuerierMock struct { - mock.Mock -} - -// DeleteJob provides a mock function with given fields: jobID -func (_m *apiQuerierMock) DeleteJob(jobID string) error { - ret := _m.Called(jobID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(jobID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EventStream provides a mock function with given fields: ctx -func (_m *apiQuerierMock) EventStream(ctx context.Context) (<-chan *api.Events, error) { - ret := _m.Called(ctx) - - var r0 <-chan *api.Events - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (<-chan *api.Events, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) <-chan *api.Events); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan *api.Events) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Execute provides a mock function with given fields: jobID, ctx, command, tty, stdin, stdout, stderr -func (_m *apiQuerierMock) Execute(jobID string, ctx context.Context, command string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { - ret := _m.Called(jobID, ctx, command, tty, stdin, stdout, stderr) - - var r0 int - var r1 error - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) (int, error)); ok { - return rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) int); ok { - r0 = rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } else { - r0 = ret.Get(0).(int) - } - - if rf, ok := ret.Get(1).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) error); ok { - r1 = rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// JobScale provides a mock function with given fields: jobID -func (_m *apiQuerierMock) JobScale(jobID string) (uint, error) { - ret := _m.Called(jobID) - - var r0 uint - var r1 error - if rf, ok := ret.Get(0).(func(string) (uint, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) uint); ok { - r0 = rf(jobID) - } else { - r0 = ret.Get(0).(uint) - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadJobList provides a mock function with given fields: -func (_m *apiQuerierMock) LoadJobList() ([]*api.JobListStub, error) { - ret := _m.Called() - - var r0 []*api.JobListStub - var r1 error - if rf, ok := ret.Get(0).(func() ([]*api.JobListStub, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []*api.JobListStub); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.JobListStub) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RegisterNomadJob provides a mock function with given fields: job -func (_m *apiQuerierMock) RegisterNomadJob(job *api.Job) (string, error) { - ret := _m.Called(job) - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(*api.Job) (string, error)); ok { - return rf(job) - } - if rf, ok := ret.Get(0).(func(*api.Job) string); ok { - r0 = rf(job) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(*api.Job) error); ok { - r1 = rf(job) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// SetJobScale provides a mock function with given fields: jobID, count, reason -func (_m *apiQuerierMock) SetJobScale(jobID string, count uint, reason string) error { - ret := _m.Called(jobID, count, reason) - - var r0 error - if rf, ok := ret.Get(0).(func(string, uint, string) error); ok { - r0 = rf(jobID, count, reason) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// allocation provides a mock function with given fields: jobID -func (_m *apiQuerierMock) allocation(jobID string) (*api.Allocation, error) { - ret := _m.Called(jobID) - - var r0 *api.Allocation - var r1 error - if rf, ok := ret.Get(0).(func(string) (*api.Allocation, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) *api.Allocation); ok { - r0 = rf(jobID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*api.Allocation) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// init provides a mock function with given fields: nomadConfig -func (_m *apiQuerierMock) init(nomadConfig *config.Nomad) error { - ret := _m.Called(nomadConfig) - - var r0 error - if rf, ok := ret.Get(0).(func(*config.Nomad) error); ok { - r0 = rf(nomadConfig) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// job provides a mock function with given fields: jobID -func (_m *apiQuerierMock) job(jobID string) (*api.Job, error) { - ret := _m.Called(jobID) - - var r0 *api.Job - var r1 error - if rf, ok := ret.Get(0).(func(string) (*api.Job, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) *api.Job); ok { - r0 = rf(jobID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*api.Job) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// listAllocations provides a mock function with given fields: -func (_m *apiQuerierMock) listAllocations() ([]*api.AllocationListStub, error) { - ret := _m.Called() - - var r0 []*api.AllocationListStub - var r1 error - if rf, ok := ret.Get(0).(func() ([]*api.AllocationListStub, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []*api.AllocationListStub); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.AllocationListStub) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// listJobs provides a mock function with given fields: prefix -func (_m *apiQuerierMock) listJobs(prefix string) ([]*api.JobListStub, error) { - ret := _m.Called(prefix) - - var r0 []*api.JobListStub - var r1 error - if rf, ok := ret.Get(0).(func(string) ([]*api.JobListStub, error)); ok { - return rf(prefix) - } - if rf, ok := ret.Get(0).(func(string) []*api.JobListStub); ok { - r0 = rf(prefix) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.JobListStub) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(prefix) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTnewApiQuerierMock interface { - mock.TestingT - Cleanup(func()) -} - -// newApiQuerierMock creates a new instance of apiQuerierMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func newApiQuerierMock(t mockConstructorTestingTnewApiQuerierMock) *apiQuerierMock { - mock := &apiQuerierMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/nomad/api_querier_test.go b/internal/nomad/api_querier_test.go deleted file mode 100644 index 755202a..0000000 --- a/internal/nomad/api_querier_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package nomad - -import ( - "errors" - "fmt" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "testing" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestWebsocketErrorNeedsToBeUnwrapped() { - rawError := &websocket.CloseError{Code: websocket.CloseNormalClosure} - err := fmt.Errorf("websocket closed before receiving exit code: %w", rawError) - - s.False(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - rootCause := errors.Unwrap(err) - s.True(websocket.IsCloseError(rootCause, websocket.CloseNormalClosure)) -} diff --git a/internal/nomad/executor_api_mock.go b/internal/nomad/executor_api_mock.go deleted file mode 100644 index bc63be1..0000000 --- a/internal/nomad/executor_api_mock.go +++ /dev/null @@ -1,490 +0,0 @@ -// Code generated by mockery v2.23.1. DO NOT EDIT. - -package nomad - -import ( - context "context" - - api "github.com/hashicorp/nomad/api" - config "github.com/openHPI/poseidon/internal/config" - - dto "github.com/openHPI/poseidon/pkg/dto" - - io "io" - - mock "github.com/stretchr/testify/mock" -) - -// ExecutorAPIMock is an autogenerated mock type for the ExecutorAPI type -type ExecutorAPIMock struct { - mock.Mock -} - -// DeleteJob provides a mock function with given fields: jobID -func (_m *ExecutorAPIMock) DeleteJob(jobID string) error { - ret := _m.Called(jobID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(jobID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EventStream provides a mock function with given fields: ctx -func (_m *ExecutorAPIMock) EventStream(ctx context.Context) (<-chan *api.Events, error) { - ret := _m.Called(ctx) - - var r0 <-chan *api.Events - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (<-chan *api.Events, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) <-chan *api.Events); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan *api.Events) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Execute provides a mock function with given fields: jobID, ctx, command, tty, stdin, stdout, stderr -func (_m *ExecutorAPIMock) Execute(jobID string, ctx context.Context, command string, tty bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { - ret := _m.Called(jobID, ctx, command, tty, stdin, stdout, stderr) - - var r0 int - var r1 error - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) (int, error)); ok { - return rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) int); ok { - r0 = rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } else { - r0 = ret.Get(0).(int) - } - - if rf, ok := ret.Get(1).(func(string, context.Context, string, bool, io.Reader, io.Writer, io.Writer) error); ok { - r1 = rf(jobID, ctx, command, tty, stdin, stdout, stderr) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ExecuteCommand provides a mock function with given fields: jobID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr -func (_m *ExecutorAPIMock) ExecuteCommand(jobID string, ctx context.Context, command string, tty bool, privilegedExecution bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) { - ret := _m.Called(jobID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr) - - var r0 int - var r1 error - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, bool, io.Reader, io.Writer, io.Writer) (int, error)); ok { - return rf(jobID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr) - } - if rf, ok := ret.Get(0).(func(string, context.Context, string, bool, bool, io.Reader, io.Writer, io.Writer) int); ok { - r0 = rf(jobID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr) - } else { - r0 = ret.Get(0).(int) - } - - if rf, ok := ret.Get(1).(func(string, context.Context, string, bool, bool, io.Reader, io.Writer, io.Writer) error); ok { - r1 = rf(jobID, ctx, command, tty, privilegedExecution, stdin, stdout, stderr) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// JobScale provides a mock function with given fields: jobID -func (_m *ExecutorAPIMock) JobScale(jobID string) (uint, error) { - ret := _m.Called(jobID) - - var r0 uint - var r1 error - if rf, ok := ret.Get(0).(func(string) (uint, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) uint); ok { - r0 = rf(jobID) - } else { - r0 = ret.Get(0).(uint) - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadEnvironmentJobs provides a mock function with given fields: -func (_m *ExecutorAPIMock) LoadEnvironmentJobs() ([]*api.Job, error) { - ret := _m.Called() - - var r0 []*api.Job - var r1 error - if rf, ok := ret.Get(0).(func() ([]*api.Job, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []*api.Job); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.Job) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadJobList provides a mock function with given fields: -func (_m *ExecutorAPIMock) LoadJobList() ([]*api.JobListStub, error) { - ret := _m.Called() - - var r0 []*api.JobListStub - var r1 error - if rf, ok := ret.Get(0).(func() ([]*api.JobListStub, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []*api.JobListStub); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.JobListStub) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadRunnerIDs provides a mock function with given fields: prefix -func (_m *ExecutorAPIMock) LoadRunnerIDs(prefix string) ([]string, error) { - ret := _m.Called(prefix) - - var r0 []string - var r1 error - if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok { - return rf(prefix) - } - if rf, ok := ret.Get(0).(func(string) []string); ok { - r0 = rf(prefix) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(prefix) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadRunnerJobs provides a mock function with given fields: environmentID -func (_m *ExecutorAPIMock) LoadRunnerJobs(environmentID dto.EnvironmentID) ([]*api.Job, error) { - ret := _m.Called(environmentID) - - var r0 []*api.Job - var r1 error - if rf, ok := ret.Get(0).(func(dto.EnvironmentID) ([]*api.Job, error)); ok { - return rf(environmentID) - } - if rf, ok := ret.Get(0).(func(dto.EnvironmentID) []*api.Job); ok { - r0 = rf(environmentID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.Job) - } - } - - if rf, ok := ret.Get(1).(func(dto.EnvironmentID) error); ok { - r1 = rf(environmentID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoadRunnerPortMappings provides a mock function with given fields: runnerID -func (_m *ExecutorAPIMock) LoadRunnerPortMappings(runnerID string) ([]api.PortMapping, error) { - ret := _m.Called(runnerID) - - var r0 []api.PortMapping - var r1 error - if rf, ok := ret.Get(0).(func(string) ([]api.PortMapping, error)); ok { - return rf(runnerID) - } - if rf, ok := ret.Get(0).(func(string) []api.PortMapping); ok { - r0 = rf(runnerID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]api.PortMapping) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(runnerID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MarkRunnerAsUsed provides a mock function with given fields: runnerID, duration -func (_m *ExecutorAPIMock) MarkRunnerAsUsed(runnerID string, duration int) error { - ret := _m.Called(runnerID, duration) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int) error); ok { - r0 = rf(runnerID, duration) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MonitorEvaluation provides a mock function with given fields: evaluationID, ctx -func (_m *ExecutorAPIMock) MonitorEvaluation(evaluationID string, ctx context.Context) error { - ret := _m.Called(evaluationID, ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(string, context.Context) error); ok { - r0 = rf(evaluationID, ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// RegisterNomadJob provides a mock function with given fields: job -func (_m *ExecutorAPIMock) RegisterNomadJob(job *api.Job) (string, error) { - ret := _m.Called(job) - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(*api.Job) (string, error)); ok { - return rf(job) - } - if rf, ok := ret.Get(0).(func(*api.Job) string); ok { - r0 = rf(job) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(*api.Job) error); ok { - r1 = rf(job) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RegisterRunnerJob provides a mock function with given fields: template -func (_m *ExecutorAPIMock) RegisterRunnerJob(template *api.Job) error { - ret := _m.Called(template) - - var r0 error - if rf, ok := ret.Get(0).(func(*api.Job) error); ok { - r0 = rf(template) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetJobScale provides a mock function with given fields: jobID, count, reason -func (_m *ExecutorAPIMock) SetJobScale(jobID string, count uint, reason string) error { - ret := _m.Called(jobID, count, reason) - - var r0 error - if rf, ok := ret.Get(0).(func(string, uint, string) error); ok { - r0 = rf(jobID, count, reason) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// WatchEventStream provides a mock function with given fields: ctx, callbacks -func (_m *ExecutorAPIMock) WatchEventStream(ctx context.Context, callbacks *AllocationProcessing) error { - ret := _m.Called(ctx, callbacks) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *AllocationProcessing) error); ok { - r0 = rf(ctx, callbacks) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// allocation provides a mock function with given fields: jobID -func (_m *ExecutorAPIMock) allocation(jobID string) (*api.Allocation, error) { - ret := _m.Called(jobID) - - var r0 *api.Allocation - var r1 error - if rf, ok := ret.Get(0).(func(string) (*api.Allocation, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) *api.Allocation); ok { - r0 = rf(jobID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*api.Allocation) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// init provides a mock function with given fields: nomadConfig -func (_m *ExecutorAPIMock) init(nomadConfig *config.Nomad) error { - ret := _m.Called(nomadConfig) - - var r0 error - if rf, ok := ret.Get(0).(func(*config.Nomad) error); ok { - r0 = rf(nomadConfig) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// job provides a mock function with given fields: jobID -func (_m *ExecutorAPIMock) job(jobID string) (*api.Job, error) { - ret := _m.Called(jobID) - - var r0 *api.Job - var r1 error - if rf, ok := ret.Get(0).(func(string) (*api.Job, error)); ok { - return rf(jobID) - } - if rf, ok := ret.Get(0).(func(string) *api.Job); ok { - r0 = rf(jobID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*api.Job) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(jobID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// listAllocations provides a mock function with given fields: -func (_m *ExecutorAPIMock) listAllocations() ([]*api.AllocationListStub, error) { - ret := _m.Called() - - var r0 []*api.AllocationListStub - var r1 error - if rf, ok := ret.Get(0).(func() ([]*api.AllocationListStub, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []*api.AllocationListStub); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.AllocationListStub) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// listJobs provides a mock function with given fields: prefix -func (_m *ExecutorAPIMock) listJobs(prefix string) ([]*api.JobListStub, error) { - ret := _m.Called(prefix) - - var r0 []*api.JobListStub - var r1 error - if rf, ok := ret.Get(0).(func(string) ([]*api.JobListStub, error)); ok { - return rf(prefix) - } - if rf, ok := ret.Get(0).(func(string) []*api.JobListStub); ok { - r0 = rf(prefix) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*api.JobListStub) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(prefix) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewExecutorAPIMock interface { - mock.TestingT - Cleanup(func()) -} - -// NewExecutorAPIMock creates a new instance of ExecutorAPIMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewExecutorAPIMock(t mockConstructorTestingTNewExecutorAPIMock) *ExecutorAPIMock { - mock := &ExecutorAPIMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/nomad/job_test.go b/internal/nomad/job_test.go deleted file mode 100644 index fb0543e..0000000 --- a/internal/nomad/job_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package nomad - -import ( - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests/helpers" -) - -func (s *MainTestSuite) TestFindTaskGroup() { - s.Run("Returns nil if task group not found", func() { - group := FindTaskGroup(&nomadApi.Job{}, TaskGroupName) - s.Nil(group) - }) - - s.Run("Finds task group when existent", func() { - _, job := helpers.CreateTemplateJob() - group := FindTaskGroup(job, TaskGroupName) - s.NotNil(group) - }) -} - -func (s *MainTestSuite) TestFindOrCreateDefaultTask() { - s.Run("Adds default task group when not set", func() { - job := &nomadApi.Job{} - group := FindAndValidateDefaultTaskGroup(job) - s.NotNil(group) - s.Equal(TaskGroupName, *group.Name) - s.Equal(1, len(job.TaskGroups)) - s.Equal(group, job.TaskGroups[0]) - s.Equal(TaskCount, *group.Count) - }) - - s.Run("Does not modify task group when already set", func() { - job := &nomadApi.Job{} - groupName := TaskGroupName - expectedGroup := &nomadApi.TaskGroup{Name: &groupName} - job.TaskGroups = []*nomadApi.TaskGroup{expectedGroup} - - group := FindAndValidateDefaultTaskGroup(job) - s.NotNil(group) - s.Equal(1, len(job.TaskGroups)) - s.Equal(expectedGroup, group) - }) -} - -func (s *MainTestSuite) TestFindOrCreateConfigTaskGroup() { - s.Run("Adds config task group when not set", func() { - job := &nomadApi.Job{} - group := FindAndValidateConfigTaskGroup(job) - s.NotNil(group) - s.Equal(group, job.TaskGroups[0]) - s.Equal(1, len(job.TaskGroups)) - - s.Equal(ConfigTaskGroupName, *group.Name) - s.Equal(0, *group.Count) - }) - - s.Run("Does not modify task group when already set", func() { - job := &nomadApi.Job{} - groupName := ConfigTaskGroupName - expectedGroup := &nomadApi.TaskGroup{Name: &groupName} - job.TaskGroups = []*nomadApi.TaskGroup{expectedGroup} - - group := FindAndValidateConfigTaskGroup(job) - s.NotNil(group) - s.Equal(1, len(job.TaskGroups)) - s.Equal(expectedGroup, group) - }) -} - -func (s *MainTestSuite) TestFindOrCreateTask() { - s.Run("Does not modify default task when already set", func() { - groupName := TaskGroupName - group := &nomadApi.TaskGroup{Name: &groupName} - expectedTask := &nomadApi.Task{Name: TaskName} - group.Tasks = []*nomadApi.Task{expectedTask} - - task := FindAndValidateDefaultTask(group) - s.NotNil(task) - s.Equal(1, len(group.Tasks)) - s.Equal(expectedTask, task) - }) - - s.Run("Does not modify config task when already set", func() { - groupName := ConfigTaskGroupName - group := &nomadApi.TaskGroup{Name: &groupName} - expectedTask := &nomadApi.Task{Name: ConfigTaskName} - group.Tasks = []*nomadApi.Task{expectedTask} - - task := FindAndValidateConfigTask(group) - s.NotNil(task) - s.Equal(1, len(group.Tasks)) - s.Equal(expectedTask, task) - }) -} - -func (s *MainTestSuite) TestSetForcePullFlag() { - _, job := helpers.CreateTemplateJob() - taskGroup := FindAndValidateDefaultTaskGroup(job) - task := FindAndValidateDefaultTask(taskGroup) - - s.Run("Ignoring passed value if DisableForcePull", func() { - config.Config.Nomad.DisableForcePull = true - SetForcePullFlag(job, true) - s.Equal(false, task.Config["force_pull"]) - }) - - s.Run("Using passed value if not DisableForcePull", func() { - config.Config.Nomad.DisableForcePull = false - SetForcePullFlag(job, true) - s.Equal(true, task.Config["force_pull"]) - - SetForcePullFlag(job, false) - s.Equal(false, task.Config["force_pull"]) - }) -} - -func (s *MainTestSuite) TestIsEnvironmentTemplateID() { - s.True(IsEnvironmentTemplateID("template-42")) - s.False(IsEnvironmentTemplateID("template-42-100")) - s.False(IsEnvironmentTemplateID("job-42")) - s.False(IsEnvironmentTemplateID("template-top")) -} - -func (s *MainTestSuite) TestRunnerJobID() { - s.Equal("0-RANDOM-UUID", RunnerJobID(0, "RANDOM-UUID")) -} - -func (s *MainTestSuite) TestTemplateJobID() { - s.Equal("template-42", TemplateJobID(42)) -} - -func (s *MainTestSuite) TestEnvironmentIDFromRunnerID() { - id, err := EnvironmentIDFromRunnerID("42-RANDOM-UUID") - s.NoError(err) - s.Equal(dto.EnvironmentID(42), id) - - _, err = EnvironmentIDFromRunnerID("") - s.Error(err) -} - -func (s *MainTestSuite) TestOOMKilledAllocation() { - event := nomadApi.TaskEvent{Details: map[string]string{"oom_killed": "true"}} - state := nomadApi.TaskState{Restarts: 2, Events: []*nomadApi.TaskEvent{&event}} - alloc := nomadApi.Allocation{TaskStates: map[string]*nomadApi.TaskState{TaskName: &state}} - s.False(isOOMKilled(&alloc)) - - event2 := nomadApi.TaskEvent{Details: map[string]string{"oom_killed": "false"}} - alloc.TaskStates[TaskName].Events = []*nomadApi.TaskEvent{&event, &event2} - s.False(isOOMKilled(&alloc)) - - event3 := nomadApi.TaskEvent{Details: map[string]string{"oom_killed": "true"}} - alloc.TaskStates[TaskName].Events = []*nomadApi.TaskEvent{&event, &event2, &event3} - s.True(isOOMKilled(&alloc)) -} diff --git a/internal/nomad/nomad_test.go b/internal/nomad/nomad_test.go deleted file mode 100644 index c2e0e12..0000000 --- a/internal/nomad/nomad_test.go +++ /dev/null @@ -1,983 +0,0 @@ -package nomad - -import ( - "bytes" - "context" - "errors" - "fmt" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/mitchellh/mapstructure" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/nullio" - "github.com/openHPI/poseidon/pkg/storage" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "io" - "regexp" - "strings" - "testing" - "time" -) - -var ( - noopAllocationProcessing = &AllocationProcessing{ - OnNew: func(_ *nomadApi.Allocation, _ time.Duration) {}, - OnDeleted: func(_ string, _ error) bool { return false }, - } - ErrUnexpectedEOF = errors.New("unexpected EOF") -) - -func TestLoadRunnersTestSuite(t *testing.T) { - suite.Run(t, new(LoadRunnersTestSuite)) -} - -type LoadRunnersTestSuite struct { - tests.MemoryLeakTestSuite - jobID string - mock *apiQuerierMock - nomadAPIClient APIClient - availableRunner *nomadApi.JobListStub - anotherAvailableRunner *nomadApi.JobListStub - pendingRunner *nomadApi.JobListStub - deadRunner *nomadApi.JobListStub -} - -func (s *LoadRunnersTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.jobID = tests.DefaultRunnerID - - s.mock = &apiQuerierMock{} - s.nomadAPIClient = APIClient{apiQuerier: s.mock} - - s.availableRunner = newJobListStub(tests.DefaultRunnerID, structs.JobStatusRunning, 1) - s.anotherAvailableRunner = newJobListStub(tests.AnotherRunnerID, structs.JobStatusRunning, 1) - s.pendingRunner = newJobListStub(tests.DefaultRunnerID+"-1", structs.JobStatusPending, 0) - s.deadRunner = newJobListStub(tests.AnotherRunnerID+"-1", structs.JobStatusDead, 0) -} - -func newJobListStub(id, status string, amountRunning int) *nomadApi.JobListStub { - return &nomadApi.JobListStub{ - ID: id, - Status: status, - JobSummary: &nomadApi.JobSummary{ - JobID: id, - Summary: map[string]nomadApi.TaskGroupSummary{TaskGroupName: {Running: amountRunning}}, - }, - } -} - -func (s *LoadRunnersTestSuite) TestErrorOfUnderlyingApiCallIsPropagated() { - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return(nil, tests.ErrDefault) - - returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.Nil(returnedIds) - s.Equal(tests.ErrDefault, err) -} - -func (s *LoadRunnersTestSuite) TestReturnsNoErrorWhenUnderlyingApiCallDoesNot() { - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return([]*nomadApi.JobListStub{}, nil) - - _, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.NoError(err) -} - -func (s *LoadRunnersTestSuite) TestAvailableRunnerIsReturned() { - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return([]*nomadApi.JobListStub{s.availableRunner}, nil) - - returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.Require().NoError(err) - s.Len(returnedIds, 1) - s.Equal(s.availableRunner.ID, returnedIds[0]) -} - -func (s *LoadRunnersTestSuite) TestPendingRunnerIsReturned() { - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return([]*nomadApi.JobListStub{s.pendingRunner}, nil) - - returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.Require().NoError(err) - s.Len(returnedIds, 1) - s.Equal(s.pendingRunner.ID, returnedIds[0]) -} - -func (s *LoadRunnersTestSuite) TestDeadRunnerIsNotReturned() { - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return([]*nomadApi.JobListStub{s.deadRunner}, nil) - - returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.Require().NoError(err) - s.Empty(returnedIds) -} - -func (s *LoadRunnersTestSuite) TestReturnsAllAvailableRunners() { - runnersList := []*nomadApi.JobListStub{ - s.availableRunner, - s.anotherAvailableRunner, - s.pendingRunner, - s.deadRunner, - } - s.mock.On("listJobs", mock.AnythingOfType("string")). - Return(runnersList, nil) - - returnedIds, err := s.nomadAPIClient.LoadRunnerIDs(s.jobID) - s.Require().NoError(err) - s.Len(returnedIds, 3) - s.Contains(returnedIds, s.availableRunner.ID) - s.Contains(returnedIds, s.anotherAvailableRunner.ID) - s.Contains(returnedIds, s.pendingRunner.ID) - s.NotContains(returnedIds, s.deadRunner.ID) -} - -const TestNamespace = "unit-tests" -const TestNomadToken = "n0m4d-t0k3n" -const TestDefaultAddress = "127.0.0.1" -const evaluationID = "evaluation-id" - -func NomadTestConfig(address string) *config.Nomad { - return &config.Nomad{ - Address: address, - Port: 4646, - Token: TestNomadToken, - TLS: config.TLS{ - Active: false, - }, - Namespace: TestNamespace, - } -} - -func (s *MainTestSuite) TestApiClient_init() { - client := &APIClient{apiQuerier: &nomadAPIClient{}} - err := client.init(NomadTestConfig(TestDefaultAddress)) - s.Require().Nil(err) -} - -func (s *MainTestSuite) TestApiClientCanNotBeInitializedWithInvalidUrl() { - client := &APIClient{apiQuerier: &nomadAPIClient{}} - err := client.init(NomadTestConfig("http://" + TestDefaultAddress)) - s.NotNil(err) -} - -func (s *MainTestSuite) TestNewExecutorApiCanBeCreatedWithoutError() { - expectedClient := &APIClient{apiQuerier: &nomadAPIClient{}} - err := expectedClient.init(NomadTestConfig(TestDefaultAddress)) - s.Require().Nil(err) - - _, err = NewExecutorAPI(NomadTestConfig(TestDefaultAddress)) - s.Require().Nil(err) -} - -// asynchronouslyMonitorEvaluation creates an APIClient with mocked Nomad API and -// runs the MonitorEvaluation method in a goroutine. The mock returns a read-only -// version of the given stream to simulate an event stream gotten from the real -// Nomad API. -func asynchronouslyMonitorEvaluation(stream <-chan *nomadApi.Events) chan error { - ctx := context.Background() - // We can only get a read-only channel once we return it from a function. - readOnlyStream := func() <-chan *nomadApi.Events { return stream }() - - apiMock := &apiQuerierMock{} - apiMock.On("EventStream", mock.AnythingOfType("*context.cancelCtx")). - Return(readOnlyStream, nil) - apiClient := &APIClient{apiMock, storage.NewLocalStorage[chan error](), storage.NewLocalStorage[*allocationData](), false} - - errChan := make(chan error) - go func() { - errChan <- apiClient.MonitorEvaluation(evaluationID, ctx) - }() - return errChan -} - -func (s *MainTestSuite) TestApiClient_MonitorEvaluationReturnsNilWhenStreamIsClosed() { - stream := make(chan *nomadApi.Events) - errChan := asynchronouslyMonitorEvaluation(stream) - - close(stream) - var err error - // If close doesn't terminate MonitorEvaluation, this test won't complete without a timeout. - select { - case err = <-errChan: - case <-time.After(time.Millisecond * 10): - s.T().Fatal("MonitorEvaluation didn't finish as expected") - } - s.Nil(err) -} - -func (s *MainTestSuite) TestApiClient_MonitorEvaluationReturnsErrorWhenStreamReturnsError() { - apiMock := &apiQuerierMock{} - apiMock.On("EventStream", mock.AnythingOfType("*context.cancelCtx")). - Return(nil, tests.ErrDefault) - apiClient := &APIClient{apiMock, storage.NewLocalStorage[chan error](), storage.NewLocalStorage[*allocationData](), false} - err := apiClient.MonitorEvaluation("id", context.Background()) - s.ErrorIs(err, tests.ErrDefault) -} - -type eventPayload struct { - Evaluation *nomadApi.Evaluation - Allocation *nomadApi.Allocation -} - -// eventForEvaluation takes an evaluation and creates an Event with the given evaluation -// as its payload. Nomad uses the mapstructure library to decode the payload, which we -// simply reverse here. -func eventForEvaluation(t *testing.T, eval *nomadApi.Evaluation) nomadApi.Event { - t.Helper() - payload := make(map[string]interface{}) - - err := mapstructure.Decode(eventPayload{Evaluation: eval}, &payload) - if err != nil { - t.Fatalf("Couldn't decode evaluation %v into payload map", eval) - return nomadApi.Event{} - } - event := nomadApi.Event{Topic: nomadApi.TopicEvaluation, Payload: payload} - return event -} - -// simulateNomadEventStream streams the given events sequentially to the stream channel. -// It returns how many events have been processed until an error occurred. -func simulateNomadEventStream( - ctx context.Context, - stream chan<- *nomadApi.Events, - errChan chan error, - events []*nomadApi.Events, -) (int, error) { - eventsProcessed := 0 - var e *nomadApi.Events - for _, e = range events { - select { - case err := <-errChan: - return eventsProcessed, err - case stream <- e: - eventsProcessed++ - } - } - close(stream) - // Wait for last event being processed - var err error - select { - case <-ctx.Done(): - case err = <-errChan: - } - return eventsProcessed, err -} - -// runEvaluationMonitoring simulates events streamed from the Nomad event stream -// to the MonitorEvaluation method. It starts the MonitorEvaluation function as a goroutine -// and sequentially transfers the events from the given array to a channel simulating the stream. -func runEvaluationMonitoring(ctx context.Context, events []*nomadApi.Events) (eventsProcessed int, err error) { - stream := make(chan *nomadApi.Events) - errChan := asynchronouslyMonitorEvaluation(stream) - return simulateNomadEventStream(ctx, stream, errChan, events) -} - -func (s *MainTestSuite) TestApiClient_MonitorEvaluationWithSuccessfulEvent() { - eval := nomadApi.Evaluation{Status: structs.EvalStatusComplete} - pendingEval := nomadApi.Evaluation{Status: structs.EvalStatusPending} - - // make sure that the tested function can complete - s.Require().Nil(checkEvaluation(&eval)) - - events := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(s.T(), &eval)}} - pendingEvaluationEvents := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(s.T(), &pendingEval)}} - multipleEventsWithPending := nomadApi.Events{Events: []nomadApi.Event{ - eventForEvaluation(s.T(), &pendingEval), eventForEvaluation(s.T(), &eval), - }} - - var cases = []struct { - streamedEvents []*nomadApi.Events - expectedEventsProcessed int - name string - }{ - {[]*nomadApi.Events{&events}, 1, - "it completes with successful event"}, - {[]*nomadApi.Events{&events, &events}, 2, - "it keeps listening after first successful event"}, - {[]*nomadApi.Events{{}, &events}, 2, - "it skips heartbeat and completes"}, - {[]*nomadApi.Events{&pendingEvaluationEvents, &events}, 2, - "it skips pending evaluation and completes"}, - {[]*nomadApi.Events{&multipleEventsWithPending}, 1, - "it handles multiple events per received event"}, - } - - for _, c := range cases { - s.Run(c.name, func() { - eventsProcessed, err := runEvaluationMonitoring(s.TestCtx, c.streamedEvents) - s.Nil(err) - s.Equal(c.expectedEventsProcessed, eventsProcessed) - }) - } -} - -func (s *MainTestSuite) TestApiClient_MonitorEvaluationWithFailingEvent() { - eval := nomadApi.Evaluation{ID: evaluationID, Status: structs.EvalStatusFailed} - evalErr := checkEvaluation(&eval) - s.Require().NotNil(evalErr) - - pendingEval := nomadApi.Evaluation{Status: structs.EvalStatusPending} - - events := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(s.T(), &eval)}} - pendingEvaluationEvents := nomadApi.Events{Events: []nomadApi.Event{eventForEvaluation(s.T(), &pendingEval)}} - multipleEventsWithPending := nomadApi.Events{Events: []nomadApi.Event{ - eventForEvaluation(s.T(), &pendingEval), eventForEvaluation(s.T(), &eval), - }} - eventsWithErr := nomadApi.Events{Err: tests.ErrDefault, Events: []nomadApi.Event{{}}} - - var cases = []struct { - streamedEvents []*nomadApi.Events - expectedEventsProcessed int - expectedError error - name string - }{ - {[]*nomadApi.Events{&events}, 1, evalErr, - "it fails with failing event"}, - {[]*nomadApi.Events{{}, &events}, 2, evalErr, - "it skips heartbeat and fail"}, - {[]*nomadApi.Events{&pendingEvaluationEvents, &events}, 2, evalErr, - "it skips pending evaluation and fail"}, - {[]*nomadApi.Events{&multipleEventsWithPending}, 1, evalErr, - "it handles multiple events per received event and fails"}, - {[]*nomadApi.Events{&eventsWithErr}, 1, tests.ErrDefault, - "it fails with event error when event has error"}, - } - - for _, c := range cases { - s.Run(c.name, func() { - eventsProcessed, err := runEvaluationMonitoring(s.TestCtx, c.streamedEvents) - s.Require().NotNil(err) - s.Contains(err.Error(), c.expectedError.Error()) - s.Equal(c.expectedEventsProcessed, eventsProcessed) - }) - } -} - -func (s *MainTestSuite) TestApiClient_MonitorEvaluationFailsWhenFailingToDecodeEvaluation() { - event := nomadApi.Event{ - Topic: nomadApi.TopicEvaluation, - // This should fail decoding, as Evaluation.Status is expected to be a string, not int - Payload: map[string]interface{}{"Evaluation": map[string]interface{}{"Status": 1}}, - } - _, err := event.Evaluation() - s.Require().NotNil(err) - eventsProcessed, err := runEvaluationMonitoring(s.TestCtx, []*nomadApi.Events{{Events: []nomadApi.Event{event}}}) - s.Error(err) - s.Equal(1, eventsProcessed) -} - -func (s *MainTestSuite) TestCheckEvaluationWithFailedAllocations() { - testKey := "test1" - failedAllocs := map[string]*nomadApi.AllocationMetric{ - testKey: {NodesExhausted: 1}, - } - evaluation := nomadApi.Evaluation{FailedTGAllocs: failedAllocs, Status: structs.EvalStatusFailed} - - assertMessageContainsCorrectStrings := func(msg string) { - s.Contains(msg, evaluation.Status, "error should contain the evaluation status") - s.Contains(msg, fmt.Sprintf("%s: %#v", testKey, failedAllocs[testKey]), - "error should contain the failed allocations metric") - } - - var msgWithoutBlockedEval, msgWithBlockedEval string - s.Run("without blocked eval", func() { - err := checkEvaluation(&evaluation) - s.Require().NotNil(err) - msgWithoutBlockedEval = err.Error() - assertMessageContainsCorrectStrings(msgWithoutBlockedEval) - }) - - s.Run("with blocked eval", func() { - evaluation.BlockedEval = "blocking-eval" - err := checkEvaluation(&evaluation) - s.Require().NotNil(err) - msgWithBlockedEval = err.Error() - assertMessageContainsCorrectStrings(msgWithBlockedEval) - }) - - s.NotEqual(msgWithBlockedEval, msgWithoutBlockedEval) -} - -func (s *MainTestSuite) TestCheckEvaluationWithoutFailedAllocations() { - evaluation := nomadApi.Evaluation{FailedTGAllocs: make(map[string]*nomadApi.AllocationMetric)} - - s.Run("when evaluation status complete", func() { - evaluation.Status = structs.EvalStatusComplete - err := checkEvaluation(&evaluation) - s.Nil(err) - }) - - s.Run("when evaluation status not complete", func() { - incompleteStates := []string{structs.EvalStatusFailed, structs.EvalStatusCancelled, - structs.EvalStatusBlocked, structs.EvalStatusPending} - for _, status := range incompleteStates { - evaluation.Status = status - err := checkEvaluation(&evaluation) - s.Require().NotNil(err) - s.Contains(err.Error(), status, "error should contain the evaluation status") - } - }) -} - -func (s *MainTestSuite) TestApiClient_WatchAllocationsIgnoresOldAllocations() { - oldStoppedAllocation := createOldAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusStop) - oldPendingAllocation := createOldAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - oldRunningAllocation := createOldAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun) - oldAllocationEvents := nomadApi.Events{Events: []nomadApi.Event{ - eventForAllocation(s.T(), oldStoppedAllocation), - eventForAllocation(s.T(), oldPendingAllocation), - eventForAllocation(s.T(), oldRunningAllocation), - }} - - assertWatchAllocation(s, []*nomadApi.Events{&oldAllocationEvents}, - []*nomadApi.Allocation(nil), []string(nil)) -} - -func createOldAllocation(clientStatus, desiredStatus string) *nomadApi.Allocation { - return createAllocation(time.Now().Add(-time.Minute).UnixNano(), clientStatus, desiredStatus) -} - -func (s *MainTestSuite) TestApiClient_WatchAllocationsIgnoresUnhandledEvents() { - nodeEvents := nomadApi.Events{Events: []nomadApi.Event{ - { - Topic: nomadApi.TopicNode, - Type: structs.TypeNodeEvent, - }, - }} - assertWatchAllocation(s, []*nomadApi.Events{&nodeEvents}, []*nomadApi.Allocation(nil), []string(nil)) -} - -func (s *MainTestSuite) TestApiClient_WatchAllocationsUsesCallbacksForEvents() { - pendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - pendingEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), pendingAllocation)}} - - s.Run("it does not add allocation when client status is pending", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingEvents}, []*nomadApi.Allocation(nil), []string(nil)) - }) - - startedAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun) - startedEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), startedAllocation)}} - pendingStartedEvents := nomadApi.Events{Events: []nomadApi.Event{ - eventForAllocation(s.T(), pendingAllocation), eventForAllocation(s.T(), startedAllocation)}} - - s.Run("it adds allocation with matching events", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents}, - []*nomadApi.Allocation{startedAllocation}, []string(nil)) - }) - - s.Run("it skips heartbeat and adds allocation with matching events", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents}, - []*nomadApi.Allocation{startedAllocation}, []string(nil)) - }) - - stoppedAllocation := createRecentAllocation(structs.AllocClientStatusComplete, structs.AllocDesiredStatusStop) - stoppedEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), stoppedAllocation)}} - pendingStartStopEvents := nomadApi.Events{Events: []nomadApi.Event{ - eventForAllocation(s.T(), pendingAllocation), - eventForAllocation(s.T(), startedAllocation), - eventForAllocation(s.T(), stoppedAllocation), - }} - - s.Run("it adds and deletes the allocation", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartStopEvents}, - []*nomadApi.Allocation{startedAllocation}, []string{stoppedAllocation.JobID}) - }) - - s.Run("it ignores duplicate events", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingEvents, &startedEvents, &startedEvents, - &stoppedEvents, &stoppedEvents, &stoppedEvents}, - []*nomadApi.Allocation{startedAllocation}, []string{startedAllocation.JobID}) - }) - - s.Run("it ignores events of unknown allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&startedEvents, &startedEvents, - &stoppedEvents, &stoppedEvents, &stoppedEvents}, []*nomadApi.Allocation(nil), []string(nil)) - }) - - s.Run("it removes restarted allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents, &pendingStartedEvents}, - []*nomadApi.Allocation{startedAllocation, startedAllocation}, []string{startedAllocation.JobID}) - }) - - rescheduleAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - rescheduleAllocation.ID = tests.AnotherUUID - rescheduleAllocation.PreviousAllocation = pendingAllocation.ID - rescheduleStartedAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun) - rescheduleStartedAllocation.ID = tests.AnotherUUID - rescheduleAllocation.PreviousAllocation = pendingAllocation.ID - rescheduleEvents := nomadApi.Events{Events: []nomadApi.Event{ - eventForAllocation(s.T(), rescheduleAllocation), eventForAllocation(s.T(), rescheduleStartedAllocation)}} - - s.Run("it removes rescheduled allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents, &rescheduleEvents}, - []*nomadApi.Allocation{startedAllocation, rescheduleStartedAllocation}, []string{startedAllocation.JobID}) - }) - - stoppedPendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusStop) - stoppedPendingEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), stoppedPendingAllocation)}} - - s.Run("it does not callback for stopped pending allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingEvents, &stoppedPendingEvents}, - []*nomadApi.Allocation(nil), []string(nil)) - }) - - failedAllocation := createRecentAllocation(structs.AllocClientStatusFailed, structs.AllocDesiredStatusStop) - failedEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), failedAllocation)}} - - s.Run("it removes stopped failed allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents, &failedEvents}, - []*nomadApi.Allocation{startedAllocation}, []string{failedAllocation.JobID}) - }) - - lostAllocation := createRecentAllocation(structs.AllocClientStatusLost, structs.AllocDesiredStatusStop) - lostEvents := nomadApi.Events{Events: []nomadApi.Event{eventForAllocation(s.T(), lostAllocation)}} - - s.Run("it removes stopped lost allocations", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents, &lostEvents}, - []*nomadApi.Allocation{startedAllocation}, []string{lostAllocation.JobID}) - }) - - rescheduledLostAllocation := createRecentAllocation(structs.AllocClientStatusLost, structs.AllocDesiredStatusStop) - rescheduledLostAllocation.NextAllocation = tests.AnotherUUID - rescheduledLostEvents := nomadApi.Events{Events: []nomadApi.Event{ - eventForAllocation(s.T(), rescheduledLostAllocation)}} - - s.Run("it removes lost allocations not before the last restart attempt", func() { - assertWatchAllocation(s, []*nomadApi.Events{&pendingStartedEvents, &rescheduledLostEvents}, - []*nomadApi.Allocation{startedAllocation}, []string(nil)) - }) -} - -func (s *MainTestSuite) TestHandleAllocationEventBuffersPendingAllocation() { - s.Run("AllocationUpdated", func() { - newPendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - newPendingEvent := eventForAllocation(s.T(), newPendingAllocation) - - allocations := storage.NewLocalStorage[*allocationData]() - err := handleAllocationEvent( - time.Now().UnixNano(), allocations, &newPendingEvent, noopAllocationProcessing) - s.Require().NoError(err) - - _, ok := allocations.Get(newPendingAllocation.ID) - s.True(ok) - }) - s.Run("PlanResult", func() { - newPendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - newPendingEvent := eventForAllocation(s.T(), newPendingAllocation) - newPendingEvent.Type = structs.TypePlanResult - - allocations := storage.NewLocalStorage[*allocationData]() - err := handleAllocationEvent( - time.Now().UnixNano(), allocations, &newPendingEvent, noopAllocationProcessing) - s.Require().NoError(err) - - _, ok := allocations.Get(newPendingAllocation.ID) - s.True(ok) - }) -} - -func (s *MainTestSuite) TestHandleAllocationEvent_RegressionTest_14_09_2023() { - jobID := "29-6f04b525-5315-11ee-af32-fa163e079f19" - a1ID := "04d86250-550c-62f9-9a21-ecdc3b38773e" - a1Starting := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - a1Starting.ID = a1ID - a1Starting.JobID = jobID - - // With this event the job is added to the idle runners - a1Running := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun) - a1Running.ID = a1ID - a1Running.JobID = jobID - - // With this event the job is removed from the idle runners - a2ID := "102f282f-376a-1453-4d3d-7d4e32046acd" - a2Starting := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - a2Starting.ID = a2ID - a2Starting.PreviousAllocation = a1ID - a2Starting.JobID = jobID - - // Because the runner is neither an idle runner nor an used runner, this event triggered the now removed - // race condition handling that led to neither removing a2 from the allocations nor adding a3 to the allocations. - a3ID := "0d8a8ece-cf52-2968-5a9f-e972a4150a6e" - a3Starting := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - a3Starting.ID = a3ID - a3Starting.PreviousAllocation = a2ID - a3Starting.JobID = jobID - - // a2Stopping was not ignored and led to an unexpected allocation stopping. - a2Stopping := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusStop) - a2Stopping.ID = a2ID - a2Stopping.PreviousAllocation = a1ID - a2Stopping.NextAllocation = a3ID - a2Stopping.JobID = jobID - - // a2Complete was not ignored (wrong behavior). - a2Complete := createRecentAllocation(structs.AllocClientStatusComplete, structs.AllocDesiredStatusStop) - a2Complete.ID = a2ID - a2Complete.PreviousAllocation = a1ID - a2Complete.NextAllocation = a3ID - a2Complete.JobID = jobID - - // a3Running was ignored because it was unknown (wrong behavior). - a3Running := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun) - a3Running.ID = a3ID - a3Running.PreviousAllocation = a2ID - a3Running.JobID = jobID - - events := []*nomadApi.Events{{Events: []nomadApi.Event{ - eventForAllocation(s.T(), a1Starting), - eventForAllocation(s.T(), a1Running), - eventForAllocation(s.T(), a2Starting), - eventForAllocation(s.T(), a3Starting), - eventForAllocation(s.T(), a2Stopping), - eventForAllocation(s.T(), a2Complete), - eventForAllocation(s.T(), a3Running), - }}} - - idleRunner := make(map[string]bool) - callbacks := &AllocationProcessing{ - OnNew: func(alloc *nomadApi.Allocation, _ time.Duration) { - idleRunner[alloc.JobID] = true - }, - OnDeleted: func(jobID string, _ error) bool { - _, ok := idleRunner[jobID] - delete(idleRunner, jobID) - return !ok - }, - } - - _, err := runAllocationWatching(s, events, callbacks) - s.NoError(err) - s.True(idleRunner[jobID]) -} - -func (s *MainTestSuite) TestHandleAllocationEvent_ReportsOOMKilledStatus() { - restartedAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun) - event := nomadApi.TaskEvent{Details: map[string]string{"oom_killed": "true"}} - state := nomadApi.TaskState{Restarts: 1, Events: []*nomadApi.TaskEvent{&event}} - restartedAllocation.TaskStates = map[string]*nomadApi.TaskState{TaskName: &state} - restartedEvent := eventForAllocation(s.T(), restartedAllocation) - - allocations := storage.NewLocalStorage[*allocationData]() - allocations.Add(restartedAllocation.ID, &allocationData{jobID: restartedAllocation.JobID}) - - var reason error - err := handleAllocationEvent(time.Now().UnixNano(), allocations, &restartedEvent, &AllocationProcessing{ - OnNew: func(_ *nomadApi.Allocation, _ time.Duration) {}, - OnDeleted: func(_ string, r error) bool { - reason = r - return true - }, - }) - s.Require().NoError(err) - s.ErrorIs(reason, ErrorOOMKilled) -} - -func (s *MainTestSuite) TestAPIClient_WatchAllocationsReturnsErrorWhenAllocationStreamCannotBeRetrieved() { - apiMock := &apiQuerierMock{} - apiMock.On("EventStream", mock.Anything).Return(nil, tests.ErrDefault) - apiClient := &APIClient{apiMock, storage.NewLocalStorage[chan error](), storage.NewLocalStorage[*allocationData](), false} - - err := apiClient.WatchEventStream(context.Background(), noopAllocationProcessing) - s.ErrorIs(err, tests.ErrDefault) -} - -// Test case: WatchAllocations returns an error when an allocation cannot be retrieved without receiving further events. -func (s *MainTestSuite) TestAPIClient_WatchAllocations() { - event := nomadApi.Event{ - Type: structs.TypeAllocationUpdated, - Topic: nomadApi.TopicAllocation, - // This should fail decoding, as Allocation.ID is expected to be a string, not int - Payload: map[string]interface{}{"Allocation": map[string]interface{}{"ID": 1}}, - } - _, err := event.Allocation() - s.Require().Error(err) - - events := []*nomadApi.Events{{Events: []nomadApi.Event{event}}, {}} - eventsProcessed, err := runAllocationWatching(s, events, noopAllocationProcessing) - s.Error(err) - s.Equal(1, eventsProcessed) -} - -func (s *MainTestSuite) TestAPIClient_WatchAllocationsReturnsErrorOnUnexpectedEOF() { - events := []*nomadApi.Events{{Err: ErrUnexpectedEOF}, {}} - eventsProcessed, err := runAllocationWatching(s, events, noopAllocationProcessing) - s.Error(err) - s.Equal(1, eventsProcessed) -} - -func assertWatchAllocation(s *MainTestSuite, events []*nomadApi.Events, - expectedNewAllocations []*nomadApi.Allocation, expectedDeletedAllocations []string) { - s.T().Helper() - var newAllocations []*nomadApi.Allocation - var deletedAllocations []string - callbacks := &AllocationProcessing{ - OnNew: func(alloc *nomadApi.Allocation, _ time.Duration) { - newAllocations = append(newAllocations, alloc) - }, - OnDeleted: func(jobID string, _ error) bool { - deletedAllocations = append(deletedAllocations, jobID) - return false - }, - } - - eventsProcessed, err := runAllocationWatching(s, events, callbacks) - s.NoError(err) - s.Equal(len(events), eventsProcessed) - - s.Equal(expectedNewAllocations, newAllocations) - s.Equal(expectedDeletedAllocations, deletedAllocations) -} - -// runAllocationWatching simulates events streamed from the Nomad event stream -// to the MonitorEvaluation method. It starts the MonitorEvaluation function as a goroutine -// and sequentially transfers the events from the given array to a channel simulating the stream. -func runAllocationWatching(s *MainTestSuite, events []*nomadApi.Events, callbacks *AllocationProcessing) ( - eventsProcessed int, err error) { - s.T().Helper() - stream := make(chan *nomadApi.Events) - errChan := asynchronouslyWatchAllocations(stream, callbacks) - return simulateNomadEventStream(s.TestCtx, stream, errChan, events) -} - -// asynchronouslyMonitorEvaluation creates an APIClient with mocked Nomad API and -// runs the MonitorEvaluation method in a goroutine. The mock returns a read-only -// version of the given stream to simulate an event stream gotten from the real -// Nomad API. -func asynchronouslyWatchAllocations(stream chan *nomadApi.Events, callbacks *AllocationProcessing) chan error { - ctx := context.Background() - // We can only get a read-only channel once we return it from a function. - readOnlyStream := func() <-chan *nomadApi.Events { return stream }() - - apiMock := &apiQuerierMock{} - apiMock.On("EventStream", ctx).Return(readOnlyStream, nil) - apiClient := &APIClient{apiMock, storage.NewLocalStorage[chan error](), storage.NewLocalStorage[*allocationData](), false} - - errChan := make(chan error) - go func() { - errChan <- apiClient.WatchEventStream(ctx, callbacks) - }() - return errChan -} - -// eventForEvaluation takes an evaluation and creates an Event with the given evaluation -// as its payload. Nomad uses the mapstructure library to decode the payload, which we -// simply reverse here. -func eventForAllocation(t *testing.T, alloc *nomadApi.Allocation) nomadApi.Event { - t.Helper() - payload := make(map[string]interface{}) - - err := mapstructure.Decode(eventPayload{Allocation: alloc}, &payload) - if err != nil { - t.Fatalf("Couldn't decode allocation %v into payload map", err) - return nomadApi.Event{} - } - event := nomadApi.Event{ - Topic: nomadApi.TopicAllocation, - Type: structs.TypeAllocationUpdated, - Payload: payload, - } - return event -} - -func createAllocation(modifyTime int64, clientStatus, desiredStatus string) *nomadApi.Allocation { - return &nomadApi.Allocation{ - ID: tests.DefaultUUID, - JobID: tests.DefaultRunnerID, - ModifyTime: modifyTime, - ClientStatus: clientStatus, - DesiredStatus: desiredStatus, - } -} - -func createRecentAllocation(clientStatus, desiredStatus string) *nomadApi.Allocation { - return createAllocation(time.Now().Add(time.Minute).UnixNano(), clientStatus, desiredStatus) -} - -func TestExecuteCommandTestSuite(t *testing.T) { - suite.Run(t, new(ExecuteCommandTestSuite)) -} - -type ExecuteCommandTestSuite struct { - tests.MemoryLeakTestSuite - allocationID string - ctx context.Context - testCommand string - expectedStdout string - expectedStderr string - apiMock *apiQuerierMock - nomadAPIClient APIClient -} - -func (s *ExecuteCommandTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.allocationID = "test-allocation-id" - s.ctx = context.Background() - s.testCommand = "echo \"do nothing\"" - s.expectedStdout = "stdout" - s.expectedStderr = "stderr" - s.apiMock = &apiQuerierMock{} - s.nomadAPIClient = APIClient{apiQuerier: s.apiMock} -} - -const withTTY = true - -func (s *ExecuteCommandTestSuite) TestWithSeparateStderr() { - config.Config.Server.InteractiveStderr = true - commandExitCode := 42 - stderrExitCode := 1 - - var stdout, stderr bytes.Buffer - var calledStdoutCommand, calledStderrCommand string - runFn := func(args mock.Arguments) { - var ok bool - calledCommand, ok := args.Get(2).(string) - s.Require().True(ok) - var out string - if isStderrCommand := strings.Contains(calledCommand, "mkfifo"); isStderrCommand { - calledStderrCommand = calledCommand - out = s.expectedStderr - } else { - calledStdoutCommand = calledCommand - out = s.expectedStdout - } - - writer, ok := args.Get(5).(io.Writer) - s.Require().True(ok) - _, err := writer.Write([]byte(out)) - s.Require().NoError(err) - } - - s.apiMock.On("Execute", s.allocationID, mock.Anything, mock.Anything, withTTY, - mock.AnythingOfType("nullio.Reader"), mock.Anything, mock.Anything).Run(runFn).Return(stderrExitCode, nil) - s.apiMock.On("Execute", s.allocationID, mock.Anything, mock.Anything, withTTY, - mock.AnythingOfType("*bytes.Buffer"), mock.Anything, mock.Anything).Run(runFn).Return(commandExitCode, nil) - - exitCode, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommand, withTTY, - UnprivilegedExecution, &bytes.Buffer{}, &stdout, &stderr) - s.Require().NoError(err) - - s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 2) - s.Equal(commandExitCode, exitCode) - - s.Run("should wrap command in stderr wrapper", func() { - s.Require().NotEmpty(calledStdoutCommand) - stderrWrapperCommand := fmt.Sprintf(stderrWrapperCommandFormat, stderrFifoFormat, s.testCommand, stderrFifoFormat) - stdoutFifoRegexp := strings.ReplaceAll(regexp.QuoteMeta(stderrWrapperCommand), "%d", "\\d*") - stdoutFifoRegexp = strings.Replace(stdoutFifoRegexp, s.testCommand, ".*", 1) - s.Regexp(stdoutFifoRegexp, calledStdoutCommand) - }) - - s.Run("should call correct stderr command", func() { - s.Require().NotEmpty(calledStderrCommand) - stderrFifoCommand := fmt.Sprintf(stderrFifoCommandFormat, stderrFifoFormat, stderrFifoFormat, stderrFifoFormat) - stderrFifoRegexp := strings.ReplaceAll(regexp.QuoteMeta(stderrFifoCommand), "%d", "\\d*") - s.Regexp(stderrFifoRegexp, calledStderrCommand) - }) - - s.Run("should return correct output", func() { - s.Equal(s.expectedStdout, stdout.String()) - s.Equal(s.expectedStderr, stderr.String()) - }) -} - -func (s *ExecuteCommandTestSuite) TestWithSeparateStderrReturnsCommandError() { - config.Config.Server.InteractiveStderr = true - - call := s.mockExecute(mock.AnythingOfType("string"), 0, nil, func(_ mock.Arguments) {}) - call.Run(func(args mock.Arguments) { - var ok bool - calledCommand, ok := args.Get(2).(string) - s.Require().True(ok) - - if isStderrCommand := strings.Contains(calledCommand, "mkfifo"); isStderrCommand { - // Here we defuse the data race condition of the ReturnArguments being set twice at the same time. - <-time.After(tests.ShortTimeout) - call.ReturnArguments = mock.Arguments{1, nil} - } else { - call.ReturnArguments = mock.Arguments{1, tests.ErrDefault} - } - }) - - _, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommand, withTTY, UnprivilegedExecution, - nullio.Reader{}, io.Discard, io.Discard) - s.Equal(tests.ErrDefault, err) -} - -func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderr() { - config.Config.Server.InteractiveStderr = false - var stdout, stderr bytes.Buffer - commandExitCode := 42 - - // mock regular call - expectedCommand := prepareCommandWithoutTTY(s.testCommand, UnprivilegedExecution) - s.mockExecute(expectedCommand, commandExitCode, nil, func(args mock.Arguments) { - stdout, ok := args.Get(5).(io.Writer) - s.Require().True(ok) - _, err := stdout.Write([]byte(s.expectedStdout)) - s.Require().NoError(err) - stderr, ok := args.Get(6).(io.Writer) - s.Require().True(ok) - _, err = stderr.Write([]byte(s.expectedStderr)) - s.Require().NoError(err) - }) - - exitCode, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommand, withTTY, - UnprivilegedExecution, nullio.Reader{}, &stdout, &stderr) - s.Require().NoError(err) - - s.apiMock.AssertNumberOfCalls(s.T(), "Execute", 1) - s.Equal(commandExitCode, exitCode) - s.Equal(s.expectedStdout, stdout.String()) - s.Equal(s.expectedStderr, stderr.String()) -} - -func (s *ExecuteCommandTestSuite) TestWithoutSeparateStderrReturnsCommandError() { - config.Config.Server.InteractiveStderr = false - expectedCommand := prepareCommandWithoutTTY(s.testCommand, UnprivilegedExecution) - s.mockExecute(expectedCommand, 1, tests.ErrDefault, func(args mock.Arguments) {}) - _, err := s.nomadAPIClient.ExecuteCommand(s.allocationID, s.ctx, s.testCommand, withTTY, UnprivilegedExecution, - nullio.Reader{}, io.Discard, io.Discard) - s.ErrorIs(err, tests.ErrDefault) -} - -func (s *ExecuteCommandTestSuite) mockExecute(command interface{}, exitCode int, - err error, runFunc func(arguments mock.Arguments)) *mock.Call { - return s.apiMock.On("Execute", s.allocationID, mock.Anything, command, withTTY, - mock.Anything, mock.Anything, mock.Anything). - Run(runFunc). - Return(exitCode, err) -} - -func (s *MainTestSuite) TestAPIClient_LoadRunnerPortMappings() { - apiMock := &apiQuerierMock{} - mockedCall := apiMock.On("allocation", tests.DefaultRunnerID) - nomadAPIClient := APIClient{apiQuerier: apiMock} - - s.Run("should return error when API query fails", func() { - mockedCall.Return(nil, tests.ErrDefault) - portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID) - s.Nil(portMappings) - s.ErrorIs(err, tests.ErrDefault) - }) - - s.Run("should return error when AllocatedResources is nil", func() { - mockedCall.Return(&nomadApi.Allocation{AllocatedResources: nil}, nil) - portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID) - s.ErrorIs(err, ErrorNoAllocatedResourcesFound) - s.Nil(portMappings) - }) - - s.Run("should correctly return ports", func() { - allocation := &nomadApi.Allocation{ - AllocatedResources: &nomadApi.AllocatedResources{ - Shared: nomadApi.AllocatedSharedResources{Ports: tests.DefaultPortMappings}, - }, - } - mockedCall.Return(allocation, nil) - portMappings, err := nomadAPIClient.LoadRunnerPortMappings(tests.DefaultRunnerID) - s.NoError(err) - s.Equal(tests.DefaultPortMappings, portMappings) - }) -} diff --git a/internal/nomad/sentry_debug_writer_test.go b/internal/nomad/sentry_debug_writer_test.go deleted file mode 100644 index cb47be0..0000000 --- a/internal/nomad/sentry_debug_writer_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package nomad - -import ( - "bytes" -) - -func (s *MainTestSuite) TestSentryDebugWriter_Write() { - buf := &bytes.Buffer{} - w := SentryDebugWriter{Target: buf, Ctx: s.TestCtx} - - description := "TestDebugMessageDescription" - data := "\x1EPoseidon " + description + " 1676646791482\x1E" - count, err := w.Write([]byte(data)) - - s.Require().NoError(err) - s.Equal(len(data), count) - s.NotContains(buf.String(), description) -} - -func (s *MainTestSuite) TestSentryDebugWriter_WriteComposed() { - buf := &bytes.Buffer{} - w := SentryDebugWriter{Target: buf, Ctx: s.TestCtx} - - data := "Hello World!\r\n\x1EPoseidon unset 1678540012404\x1E\x1EPoseidon /sbin/setuser user 1678540012408\x1E" - count, err := w.Write([]byte(data)) - - s.Require().NoError(err) - s.Equal(len(data), count) - s.Contains(buf.String(), "Hello World!") -} - -func (s *MainTestSuite) TestSentryDebugWriter_Close() { - buf := &bytes.Buffer{} - w := NewSentryDebugWriter(buf, s.TestCtx) - s.Require().Empty(w.lastSpan.Tags) - - w.Close(42) - s.Require().Contains(w.lastSpan.Tags, "exit_code") - s.Equal("42", w.lastSpan.Tags["exit_code"]) -} - -func (s *MainTestSuite) TestSentryDebugWriter_handleTimeDebugMessage() { - buf := &bytes.Buffer{} - w := NewSentryDebugWriter(buf, s.TestCtx) - s.Require().Equal("nomad.execute.connect", w.lastSpan.Op) - - description := "TestDebugMessageDescription" - match := map[string][]byte{"time": []byte("1676646791482"), "text": []byte(description)} - w.handleTimeDebugMessage(match) - s.Equal("nomad.execute.bash", w.lastSpan.Op) - s.Equal(description, w.lastSpan.Description) -} diff --git a/internal/runner/aws_manager.go b/internal/runner/aws_manager.go deleted file mode 100644 index 8c72991..0000000 --- a/internal/runner/aws_manager.go +++ /dev/null @@ -1,48 +0,0 @@ -package runner - -import ( - "context" - "fmt" - "github.com/openHPI/poseidon/pkg/dto" - "time" -) - -type AWSRunnerManager struct { - *AbstractManager -} - -// NewAWSRunnerManager creates a new runner manager that keeps track of all runners at AWS. -func NewAWSRunnerManager(ctx context.Context) *AWSRunnerManager { - return &AWSRunnerManager{NewAbstractManager(ctx)} -} - -func (a AWSRunnerManager) Claim(id dto.EnvironmentID, duration int) (Runner, error) { - environment, ok := a.GetEnvironment(id) - if !ok { - r, err := a.NextHandler().Claim(id, duration) - if err != nil { - return nil, fmt.Errorf("aws wrapped: %w", err) - } - return r, nil - } - - runner, ok := environment.Sample() - if !ok { - log.Warn("no aws runner available") - return nil, ErrNoRunnersAvailable - } - - a.usedRunners.Add(runner.ID(), runner) - runner.SetupTimeout(time.Duration(duration) * time.Second) - return runner, nil -} - -func (a AWSRunnerManager) Return(r Runner) error { - _, isAWSRunner := r.(*AWSFunctionWorkload) - if isAWSRunner { - a.usedRunners.Delete(r.ID()) - } else if err := a.NextHandler().Return(r); err != nil { - return fmt.Errorf("aws wrapped: %w", err) - } - return nil -} diff --git a/internal/runner/aws_manager_test.go b/internal/runner/aws_manager_test.go deleted file mode 100644 index 8aee34b..0000000 --- a/internal/runner/aws_manager_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package runner - -import ( - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "testing" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestAWSRunnerManager_EnvironmentAccessor() { - m := NewAWSRunnerManager(s.TestCtx) - - environments := m.ListEnvironments() - s.Empty(environments) - - environment := createBasicEnvironmentMock(defaultEnvironmentID) - m.StoreEnvironment(environment) - - environments = m.ListEnvironments() - s.Len(environments, 1) - s.Equal(environments[0].ID(), dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - - e, ok := m.GetEnvironment(tests.DefaultEnvironmentIDAsInteger) - s.True(ok) - s.Equal(environment, e) - - _, ok = m.GetEnvironment(tests.AnotherEnvironmentIDAsInteger) - s.False(ok) -} - -func (s *MainTestSuite) TestAWSRunnerManager_Claim() { - m := NewAWSRunnerManager(s.TestCtx) - environment := createBasicEnvironmentMock(defaultEnvironmentID) - r, err := NewAWSFunctionWorkload(environment, func(_ Runner) error { return nil }) - s.NoError(err) - environment.On("Sample").Return(r, true) - m.StoreEnvironment(environment) - - s.Run("returns runner for AWS environment", func() { - r, err := m.Claim(tests.DefaultEnvironmentIDAsInteger, 60) - s.NoError(err) - s.NotNil(r) - }) - - s.Run("forwards request for non-AWS environments", func() { - nextHandler := &ManagerMock{} - nextHandler.On("Claim", mock.AnythingOfType("dto.EnvironmentID"), mock.AnythingOfType("int")). - Return(nil, nil) - m.SetNextHandler(nextHandler) - - _, err := m.Claim(tests.AnotherEnvironmentIDAsInteger, 60) - s.Nil(err) - nextHandler.AssertCalled(s.T(), "Claim", dto.EnvironmentID(tests.AnotherEnvironmentIDAsInteger), 60) - }) - - err = r.Destroy(nil) - s.NoError(err) -} - -func (s *MainTestSuite) TestAWSRunnerManager_Return() { - m := NewAWSRunnerManager(s.TestCtx) - environment := createBasicEnvironmentMock(defaultEnvironmentID) - m.StoreEnvironment(environment) - r, err := NewAWSFunctionWorkload(environment, func(_ Runner) error { return nil }) - s.NoError(err) - - s.Run("removes usedRunner", func() { - m.usedRunners.Add(r.ID(), r) - s.Contains(m.usedRunners.List(), r) - - err := m.Return(r) - s.NoError(err) - s.NotContains(m.usedRunners.List(), r) - }) - - s.Run("calls nextHandler for non-AWS runner", func() { - nextHandler := &ManagerMock{} - nextHandler.On("Return", mock.AnythingOfType("*runner.NomadJob")).Return(nil) - m.SetNextHandler(nextHandler) - - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - nonAWSRunner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, nil) - err := m.Return(nonAWSRunner) - s.NoError(err) - nextHandler.AssertCalled(s.T(), "Return", nonAWSRunner) - - err = nonAWSRunner.Destroy(nil) - s.NoError(err) - }) - - err = r.Destroy(nil) - s.NoError(err) -} - -func createBasicEnvironmentMock(id dto.EnvironmentID) *ExecutionEnvironmentMock { - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(id) - environment.On("Image").Return("") - environment.On("CPULimit").Return(uint(0)) - environment.On("MemoryLimit").Return(uint(0)) - environment.On("NetworkAccess").Return(false, nil) - environment.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil, false) - environment.On("ApplyPrewarmingPoolSize").Return(nil) - environment.On("IdleRunnerCount").Return(uint(1)).Maybe() - environment.On("PrewarmingPoolSize").Return(uint(1)).Maybe() - return environment -} diff --git a/internal/runner/aws_runner.go b/internal/runner/aws_runner.go deleted file mode 100644 index c6cf4cd..0000000 --- a/internal/runner/aws_runner.go +++ /dev/null @@ -1,246 +0,0 @@ -package runner - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "github.com/google/uuid" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/monitoring" - "github.com/openHPI/poseidon/pkg/storage" - "io" - "net/http" - "time" -) - -var ErrWrongMessageType = errors.New("received message that is not a text message") - -type awsFunctionRequest struct { - Action string `json:"action"` - Cmd []string `json:"cmd"` - Files map[dto.FilePath][]byte `json:"files"` -} - -// AWSFunctionWorkload is an abstraction to build a request to an AWS Lambda Function. -// It is not persisted on a Poseidon restart. -// The InactivityTimer is used actively. It stops listening to the Lambda function. -// AWS terminates the Lambda Function after the [Globals.Function.Timeout](deploy/aws/template.yaml). -type AWSFunctionWorkload struct { - InactivityTimer - id string - fs map[dto.FilePath][]byte - executions storage.Storage[*dto.ExecutionRequest] - runningExecutions map[string]context.CancelFunc - onDestroy DestroyRunnerHandler - environment ExecutionEnvironment - ctx context.Context - cancel context.CancelFunc -} - -// NewAWSFunctionWorkload creates a new AWSFunctionWorkload with the provided id. -func NewAWSFunctionWorkload( - environment ExecutionEnvironment, onDestroy DestroyRunnerHandler) (*AWSFunctionWorkload, error) { - newUUID, err := uuid.NewUUID() - if err != nil { - return nil, fmt.Errorf("failed generating runner id: %w", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - workload := &AWSFunctionWorkload{ - id: newUUID.String(), - fs: make(map[dto.FilePath][]byte), - runningExecutions: make(map[string]context.CancelFunc), - onDestroy: onDestroy, - environment: environment, - ctx: ctx, - cancel: cancel, - } - workload.executions = storage.NewMonitoredLocalStorage[*dto.ExecutionRequest]( - monitoring.MeasurementExecutionsAWS, monitorExecutionsRunnerID(environment.ID(), workload.id), time.Minute, ctx) - workload.InactivityTimer = NewInactivityTimer(workload, func(_ Runner) error { - return workload.Destroy(nil) - }) - return workload, nil -} - -func (w *AWSFunctionWorkload) ID() string { - return w.id -} - -func (w *AWSFunctionWorkload) Environment() dto.EnvironmentID { - return w.environment.ID() -} - -func (w *AWSFunctionWorkload) MappedPorts() []*dto.MappedPort { - return []*dto.MappedPort{} -} - -func (w *AWSFunctionWorkload) StoreExecution(id string, request *dto.ExecutionRequest) { - w.executions.Add(id, request) -} - -func (w *AWSFunctionWorkload) ExecutionExists(id string) bool { - _, ok := w.executions.Get(id) - return ok -} - -// ExecuteInteractively runs the execution request in an AWS function. -// It should be further improved by using the passed context to handle lost connections. -func (w *AWSFunctionWorkload) ExecuteInteractively( - id string, _ io.ReadWriter, stdout, stderr io.Writer, _ context.Context) ( - <-chan ExitInfo, context.CancelFunc, error) { - w.ResetTimeout() - request, ok := w.executions.Pop(id) - if !ok { - return nil, nil, ErrorUnknownExecution - } - hideEnvironmentVariables(request, "AWS") - request.PrivilegedExecution = true // AWS does not support multiple users at this moment. - command, ctx, cancel := prepareExecution(request, w.ctx) - commands := []string{"/bin/bash", "-c", command} - exitInternal := make(chan ExitInfo) - exit := make(chan ExitInfo, 1) - - go w.executeCommand(ctx, commands, stdout, stderr, exitInternal) - go w.handleRunnerTimeout(ctx, exitInternal, exit, id) - - return exit, cancel, nil -} - -// ListFileSystem is currently not supported with this aws serverless function. -// This is because the function execution ends with the termination of the workload code. -// So an on-demand file system listing after the termination is not possible. Also, we do not want to copy all files. -func (w *AWSFunctionWorkload) ListFileSystem(_ string, _ bool, _ io.Writer, _ bool, _ context.Context) error { - return dto.ErrNotSupported -} - -// UpdateFileSystem copies Files into the executor. -// Current limitation: No files can be deleted apart from the previously added files. -// Future Work: Deduplication of the file systems, as the largest workload is likely to be used by additional -// CSV files or similar, which are the same for many executions. -func (w *AWSFunctionWorkload) UpdateFileSystem(request *dto.UpdateFileSystemRequest, _ context.Context) error { - for _, path := range request.Delete { - delete(w.fs, path) - } - for _, file := range request.Copy { - w.fs[file.Path] = file.Content - } - return nil -} - -// GetFileContent is currently not supported with this aws serverless function. -// This is because the function execution ends with the termination of the workload code. -// So an on-demand file streaming after the termination is not possible. Also, we do not want to copy all files. -func (w *AWSFunctionWorkload) GetFileContent(_ string, _ http.ResponseWriter, _ bool, _ context.Context) error { - return dto.ErrNotSupported -} - -func (w *AWSFunctionWorkload) Destroy(_ DestroyReason) error { - w.cancel() - if err := w.onDestroy(w); err != nil { - return fmt.Errorf("error while destroying aws runner: %w", err) - } - return nil -} - -func (w *AWSFunctionWorkload) executeCommand(ctx context.Context, command []string, - stdout, stderr io.Writer, exit chan<- ExitInfo, -) { - defer close(exit) - data := &awsFunctionRequest{ - Action: w.environment.Image(), - Cmd: command, - Files: w.fs, - } - log.WithContext(ctx).WithField("request", data).Trace("Sending request to AWS") - rawData, err := json.Marshal(data) - if err != nil { - exit <- ExitInfo{uint8(1), fmt.Errorf("cannot stingify aws function request: %w", err)} - return - } - - wsConn, response, err := websocket.DefaultDialer.Dial(config.Config.AWS.Endpoint, nil) - if err != nil { - exit <- ExitInfo{uint8(1), fmt.Errorf("failed to establish aws connection: %w", err)} - return - } - _ = response.Body.Close() - defer wsConn.Close() - err = wsConn.WriteMessage(websocket.TextMessage, rawData) - if err != nil { - exit <- ExitInfo{uint8(1), fmt.Errorf("cannot send aws request: %w", err)} - return - } - - // receiveOutput listens for the execution timeout (or the exit code). - exitCode, err := w.receiveOutput(wsConn, stdout, stderr, ctx) - // TimeoutPassed checks the runner timeout - if w.TimeoutPassed() { - err = ErrorRunnerInactivityTimeout - } - exit <- ExitInfo{exitCode, err} -} - -func (w *AWSFunctionWorkload) receiveOutput( - conn *websocket.Conn, stdout, stderr io.Writer, ctx context.Context) (uint8, error) { - for ctx.Err() == nil { - messageType, reader, err := conn.NextReader() - if err != nil { - return 1, fmt.Errorf("cannot read from aws connection: %w", err) - } - if messageType != websocket.TextMessage { - return 1, ErrWrongMessageType - } - var wsMessage dto.WebSocketMessage - err = json.NewDecoder(reader).Decode(&wsMessage) - if err != nil { - return 1, fmt.Errorf("failed to decode message from aws: %w", err) - } - - log.WithField("msg", wsMessage).Info("New Message from AWS function") - - switch wsMessage.Type { - default: - log.WithContext(ctx).WithField("data", wsMessage).Warn("unexpected message from aws function") - case dto.WebSocketExit: - return wsMessage.ExitCode, nil - case dto.WebSocketOutputStdout: - // We do not check the written bytes as the rawToCodeOceanWriter receives everything or nothing. - _, err = stdout.Write([]byte(wsMessage.Data)) - case dto.WebSocketOutputStderr, dto.WebSocketOutputError: - _, err = stderr.Write([]byte(wsMessage.Data)) - } - if err != nil { - return 1, fmt.Errorf("failed to forward message: %w", err) - } - } - return 1, fmt.Errorf("receiveOutput stpped by context: %w", ctx.Err()) -} - -// handleRunnerTimeout listens for a runner timeout and aborts the execution in that case. -// It listens via a context in runningExecutions that is canceled on the timeout event. -func (w *AWSFunctionWorkload) handleRunnerTimeout(ctx context.Context, - exitInternal <-chan ExitInfo, exit chan<- ExitInfo, executionID string) { - executionCtx, cancelExecution := context.WithCancel(ctx) - w.runningExecutions[executionID] = cancelExecution - defer delete(w.runningExecutions, executionID) - defer close(exit) - - select { - case exitInfo := <-exitInternal: - exit <- exitInfo - case <-executionCtx.Done(): - exit <- ExitInfo{255, ErrorRunnerInactivityTimeout} - } -} - -// hideEnvironmentVariables sets the CODEOCEAN variable and unsets all variables starting with the passed prefix. -func hideEnvironmentVariables(request *dto.ExecutionRequest, unsetPrefix string) { - if request.Environment == nil { - request.Environment = make(map[string]string) - } - request.Command = "unset \"${!" + unsetPrefix + "@}\" && " + request.Command -} diff --git a/internal/runner/aws_runner_test.go b/internal/runner/aws_runner_test.go deleted file mode 100644 index edfc826..0000000 --- a/internal/runner/aws_runner_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package runner - -import ( - "context" - "encoding/base64" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "io" - "net/http" - "net/http/httptest" - "strings" - "time" -) - -func (s *MainTestSuite) TestAWSExecutionRequestIsStored() { - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - r, err := NewAWSFunctionWorkload(environment, func(_ Runner) error { return nil }) - s.NoError(err) - executionRequest := &dto.ExecutionRequest{ - Command: "command", - TimeLimit: 10, - Environment: nil, - } - r.StoreExecution(tests.DefaultEnvironmentIDAsString, executionRequest) - s.True(r.ExecutionExists(tests.DefaultEnvironmentIDAsString)) - storedExecutionRunner, ok := r.executions.Pop(tests.DefaultEnvironmentIDAsString) - s.True(ok, "Getting an execution should not return ok false") - s.Equal(executionRequest, storedExecutionRunner) - - err = r.Destroy(nil) - s.NoError(err) -} - -type awsEndpointMock struct { - hasConnected bool - ctx context.Context - receivedData string -} - -func (a *awsEndpointMock) handler(w http.ResponseWriter, r *http.Request) { - upgrader := websocket.Upgrader{} - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - return - } - defer c.Close() - - a.hasConnected = true - for a.ctx.Err() == nil { - _, message, err := c.ReadMessage() - if err != nil { - break - } - a.receivedData = string(message) - } -} - -func (s *MainTestSuite) TestAWSFunctionWorkload_ExecuteInteractively() { - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - environment.On("Image").Return("testImage or AWS endpoint") - r, err := NewAWSFunctionWorkload(environment, func(_ Runner) error { return nil }) - s.Require().NoError(err) - - var cancel context.CancelFunc - awsMock := &awsEndpointMock{} - sv := httptest.NewServer(http.HandlerFunc(awsMock.handler)) - defer sv.Close() - - s.Run("establishes WebSocket connection to AWS endpoint", func() { - // Convert http://127.0.0.1 to ws://127.0.0.1 - config.Config.AWS.Endpoint = "ws" + strings.TrimPrefix(sv.URL, "http") - awsMock.ctx, cancel = context.WithCancel(context.Background()) - cancel() - - r.StoreExecution(tests.DefaultEnvironmentIDAsString, &dto.ExecutionRequest{}) - exit, _, err := r.ExecuteInteractively( - tests.DefaultEnvironmentIDAsString, nil, io.Discard, io.Discard, s.TestCtx) - s.Require().NoError(err) - <-exit - s.True(awsMock.hasConnected) - }) - - s.Run("sends execution request", func() { - s.T().Skip("The AWS runner ignores its context for executions and waits infinetly for the exit message.") // ToDo - awsMock.ctx, cancel = context.WithTimeout(context.Background(), tests.ShortTimeout) - defer cancel() - command := "sl" - request := &dto.ExecutionRequest{Command: command} - r.StoreExecution(tests.DefaultEnvironmentIDAsString, request) - - _, cancel, err := r.ExecuteInteractively( - tests.DefaultEnvironmentIDAsString, nil, io.Discard, io.Discard, s.TestCtx) - s.Require().NoError(err) - <-time.After(tests.ShortTimeout) - cancel() - - expectedRequestData := `{"action":"` + environment.Image() + - `","cmd":["/bin/bash","-c","env CODEOCEAN=true /bin/bash -c \"unset \\\"\\${!AWS@}\\\" \u0026\u0026 ` + command + - `\""],"files":{}}` - s.Equal(expectedRequestData, awsMock.receivedData) - }) - - err = r.Destroy(nil) - s.NoError(err) -} - -func (s *MainTestSuite) TestAWSFunctionWorkload_UpdateFileSystem() { - s.T().Skip("The AWS runner ignores its context for executions and waits infinetly for the exit message.") // ToDo - - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - environment.On("Image").Return("testImage or AWS endpoint") - r, err := NewAWSFunctionWorkload(environment, nil) - s.Require().NoError(err) - - var cancel context.CancelFunc - awsMock := &awsEndpointMock{} - sv := httptest.NewServer(http.HandlerFunc(awsMock.handler)) - defer sv.Close() - - // Convert http://127.0.0.1 to ws://127.0.0.1 - config.Config.AWS.Endpoint = "ws" + strings.TrimPrefix(sv.URL, "http") - awsMock.ctx, cancel = context.WithTimeout(context.Background(), tests.ShortTimeout) - defer cancel() - command := "sl" - request := &dto.ExecutionRequest{Command: command} - r.StoreExecution(tests.DefaultEnvironmentIDAsString, request) - myFile := dto.File{Path: "myPath", Content: []byte("myContent")} - - err = r.UpdateFileSystem(&dto.UpdateFileSystemRequest{Copy: []dto.File{myFile}}, s.TestCtx) - s.NoError(err) - _, execCancel, err := r.ExecuteInteractively( - tests.DefaultEnvironmentIDAsString, nil, io.Discard, io.Discard, s.TestCtx) - s.Require().NoError(err) - <-time.After(tests.ShortTimeout) - execCancel() - - expectedRequestData := `{"action":"` + environment.Image() + - `","cmd":["/bin/bash","-c","env CODEOCEAN=true /bin/bash -c \"unset \\\"\\${!AWS@}\\\" \u0026\u0026 ` + command + - `\""],"files":{"` + string(myFile.Path) + `":"` + base64.StdEncoding.EncodeToString(myFile.Content) + `"}}` - s.Equal(expectedRequestData, awsMock.receivedData) - - err = r.Destroy(nil) - s.NoError(err) -} - -func (s *MainTestSuite) TestAWSFunctionWorkload_Destroy() { - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - hasDestroyBeenCalled := false - r, err := NewAWSFunctionWorkload(environment, func(_ Runner) error { - hasDestroyBeenCalled = true - return nil - }) - s.Require().NoError(err) - - var reason error - err = r.Destroy(reason) - s.NoError(err) - s.True(hasDestroyBeenCalled) -} diff --git a/internal/runner/constants_test.go b/internal/runner/constants_test.go deleted file mode 100644 index 420f26c..0000000 --- a/internal/runner/constants_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package runner - -import ( - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" -) - -const ( - defaultEnvironmentID = dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger) - anotherEnvironmentID = dto.EnvironmentID(tests.AnotherEnvironmentIDAsInteger) - defaultInactivityTimeout = 0 -) diff --git a/internal/runner/execution_environment_mock.go b/internal/runner/execution_environment_mock.go deleted file mode 100644 index 101213f..0000000 --- a/internal/runner/execution_environment_mock.go +++ /dev/null @@ -1,349 +0,0 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. - -package runner - -import ( - dto "github.com/openHPI/poseidon/pkg/dto" - mock "github.com/stretchr/testify/mock" -) - -// ExecutionEnvironmentMock is an autogenerated mock type for the ExecutionEnvironment type -type ExecutionEnvironmentMock struct { - mock.Mock -} - -// AddRunner provides a mock function with given fields: r -func (_m *ExecutionEnvironmentMock) AddRunner(r Runner) { - _m.Called(r) -} - -// ApplyPrewarmingPoolSize provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) ApplyPrewarmingPoolSize() error { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for ApplyPrewarmingPoolSize") - } - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// CPULimit provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) CPULimit() uint { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for CPULimit") - } - - var r0 uint - if rf, ok := ret.Get(0).(func() uint); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint) - } - - return r0 -} - -// Delete provides a mock function with given fields: reason -func (_m *ExecutionEnvironmentMock) Delete(reason DestroyReason) error { - ret := _m.Called(reason) - - if len(ret) == 0 { - panic("no return value specified for Delete") - } - - var r0 error - if rf, ok := ret.Get(0).(func(DestroyReason) error); ok { - r0 = rf(reason) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteRunner provides a mock function with given fields: id -func (_m *ExecutionEnvironmentMock) DeleteRunner(id string) (Runner, bool) { - ret := _m.Called(id) - - if len(ret) == 0 { - panic("no return value specified for DeleteRunner") - } - - var r0 Runner - var r1 bool - if rf, ok := ret.Get(0).(func(string) (Runner, bool)); ok { - return rf(id) - } - if rf, ok := ret.Get(0).(func(string) Runner); ok { - r0 = rf(id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(Runner) - } - } - - if rf, ok := ret.Get(1).(func(string) bool); ok { - r1 = rf(id) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// ID provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) ID() dto.EnvironmentID { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for ID") - } - - var r0 dto.EnvironmentID - if rf, ok := ret.Get(0).(func() dto.EnvironmentID); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(dto.EnvironmentID) - } - - return r0 -} - -// IdleRunnerCount provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) IdleRunnerCount() uint { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for IdleRunnerCount") - } - - var r0 uint - if rf, ok := ret.Get(0).(func() uint); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint) - } - - return r0 -} - -// Image provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) Image() string { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Image") - } - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// MarshalJSON provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) MarshalJSON() ([]byte, error) { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for MarshalJSON") - } - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func() ([]byte, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() []byte); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MemoryLimit provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) MemoryLimit() uint { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for MemoryLimit") - } - - var r0 uint - if rf, ok := ret.Get(0).(func() uint); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint) - } - - return r0 -} - -// NetworkAccess provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) NetworkAccess() (bool, []uint16) { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for NetworkAccess") - } - - var r0 bool - var r1 []uint16 - if rf, ok := ret.Get(0).(func() (bool, []uint16)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func() []uint16); ok { - r1 = rf() - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).([]uint16) - } - } - - return r0, r1 -} - -// PrewarmingPoolSize provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) PrewarmingPoolSize() uint { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for PrewarmingPoolSize") - } - - var r0 uint - if rf, ok := ret.Get(0).(func() uint); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint) - } - - return r0 -} - -// Register provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) Register() error { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Register") - } - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Sample provides a mock function with given fields: -func (_m *ExecutionEnvironmentMock) Sample() (Runner, bool) { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Sample") - } - - var r0 Runner - var r1 bool - if rf, ok := ret.Get(0).(func() (Runner, bool)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() Runner); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(Runner) - } - } - - if rf, ok := ret.Get(1).(func() bool); ok { - r1 = rf() - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// SetCPULimit provides a mock function with given fields: limit -func (_m *ExecutionEnvironmentMock) SetCPULimit(limit uint) { - _m.Called(limit) -} - -// SetConfigFrom provides a mock function with given fields: environment -func (_m *ExecutionEnvironmentMock) SetConfigFrom(environment ExecutionEnvironment) { - _m.Called(environment) -} - -// SetID provides a mock function with given fields: id -func (_m *ExecutionEnvironmentMock) SetID(id dto.EnvironmentID) { - _m.Called(id) -} - -// SetImage provides a mock function with given fields: image -func (_m *ExecutionEnvironmentMock) SetImage(image string) { - _m.Called(image) -} - -// SetMemoryLimit provides a mock function with given fields: limit -func (_m *ExecutionEnvironmentMock) SetMemoryLimit(limit uint) { - _m.Called(limit) -} - -// SetNetworkAccess provides a mock function with given fields: allow, ports -func (_m *ExecutionEnvironmentMock) SetNetworkAccess(allow bool, ports []uint16) { - _m.Called(allow, ports) -} - -// SetPrewarmingPoolSize provides a mock function with given fields: count -func (_m *ExecutionEnvironmentMock) SetPrewarmingPoolSize(count uint) { - _m.Called(count) -} - -// NewExecutionEnvironmentMock creates a new instance of ExecutionEnvironmentMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewExecutionEnvironmentMock(t interface { - mock.TestingT - Cleanup(func()) -}) *ExecutionEnvironmentMock { - mock := &ExecutionEnvironmentMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/runner/inactivity_timer_mock.go b/internal/runner/inactivity_timer_mock.go deleted file mode 100644 index 366cd55..0000000 --- a/internal/runner/inactivity_timer_mock.go +++ /dev/null @@ -1,43 +0,0 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. - -package runner - -import ( - time "time" - - mock "github.com/stretchr/testify/mock" -) - -// InactivityTimerMock is an autogenerated mock type for the InactivityTimer type -type InactivityTimerMock struct { - mock.Mock -} - -// ResetTimeout provides a mock function with given fields: -func (_m *InactivityTimerMock) ResetTimeout() { - _m.Called() -} - -// SetupTimeout provides a mock function with given fields: duration -func (_m *InactivityTimerMock) SetupTimeout(duration time.Duration) { - _m.Called(duration) -} - -// StopTimeout provides a mock function with given fields: -func (_m *InactivityTimerMock) StopTimeout() { - _m.Called() -} - -// TimeoutPassed provides a mock function with given fields: -func (_m *InactivityTimerMock) TimeoutPassed() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} diff --git a/internal/runner/inactivity_timer_test.go b/internal/runner/inactivity_timer_test.go deleted file mode 100644 index 1aaf2fc..0000000 --- a/internal/runner/inactivity_timer_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package runner - -import ( - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "testing" - "time" -) - -func TestInactivityTimerTestSuite(t *testing.T) { - suite.Run(t, new(InactivityTimerTestSuite)) -} - -type InactivityTimerTestSuite struct { - tests.MemoryLeakTestSuite - runner Runner - returned chan bool -} - -func (s *InactivityTimerTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.returned = make(chan bool, 1) - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", tests.DefaultRunnerID).Return(nil) - s.runner = NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { - s.returned <- true - return nil - }) - - s.runner.SetupTimeout(tests.ShortTimeout) -} - -func (s *InactivityTimerTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - go func() { - select { - case <-s.returned: - case <-time.After(tests.ShortTimeout): - } - }() - - err := s.runner.Destroy(nil) - s.Require().NoError(err) -} - -func (s *InactivityTimerTestSuite) TestRunnerIsReturnedAfterTimeout() { - s.True(tests.ChannelReceivesSomething(s.returned, 2*tests.ShortTimeout)) -} - -func (s *InactivityTimerTestSuite) TestRunnerIsNotReturnedBeforeTimeout() { - s.False(tests.ChannelReceivesSomething(s.returned, tests.ShortTimeout/2)) -} - -func (s *InactivityTimerTestSuite) TestResetTimeoutExtendsTheDeadline() { - time.Sleep(3 * tests.ShortTimeout / 4) - s.runner.ResetTimeout() - s.False(tests.ChannelReceivesSomething(s.returned, 3*tests.ShortTimeout/4), - "Because of the reset, the timeout should not be reached by now.") - s.True(tests.ChannelReceivesSomething(s.returned, 5*tests.ShortTimeout/4), - "After reset, the timout should be reached by now.") -} - -func (s *InactivityTimerTestSuite) TestStopTimeoutStopsTimeout() { - s.runner.StopTimeout() - s.False(tests.ChannelReceivesSomething(s.returned, 2*tests.ShortTimeout)) -} - -func (s *InactivityTimerTestSuite) TestTimeoutPassedReturnsFalseBeforeDeadline() { - s.False(s.runner.TimeoutPassed()) -} - -func (s *InactivityTimerTestSuite) TestTimeoutPassedReturnsTrueAfterDeadline() { - <-time.After(2 * tests.ShortTimeout) - s.True(s.runner.TimeoutPassed()) -} - -func (s *InactivityTimerTestSuite) TestTimerIsNotResetAfterDeadline() { - time.Sleep(2 * tests.ShortTimeout) - // We need to empty the returned channel so Return can send to it again. - tests.ChannelReceivesSomething(s.returned, 0) - s.runner.ResetTimeout() - s.False(tests.ChannelReceivesSomething(s.returned, 2*tests.ShortTimeout)) -} - -func (s *InactivityTimerTestSuite) TestSetupTimeoutStopsOldTimeout() { - s.runner.SetupTimeout(3 * tests.ShortTimeout) - s.False(tests.ChannelReceivesSomething(s.returned, 2*tests.ShortTimeout)) - s.True(tests.ChannelReceivesSomething(s.returned, 2*tests.ShortTimeout)) -} - -func (s *InactivityTimerTestSuite) TestTimerIsInactiveWhenDurationIsZero() { - s.runner.SetupTimeout(0) - s.False(tests.ChannelReceivesSomething(s.returned, tests.ShortTimeout)) -} diff --git a/internal/runner/kubernetes_manager.go b/internal/runner/kubernetes_manager.go new file mode 100644 index 0000000..8bec060 --- /dev/null +++ b/internal/runner/kubernetes_manager.go @@ -0,0 +1,125 @@ +package runner + +import ( + "context" + "fmt" + "github.com/openHPI/poseidon/internal/kubernetes" + "github.com/openHPI/poseidon/internal/nomad" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/storage" + "github.com/openHPI/poseidon/pkg/util" + appv1 "k8s.io/api/apps/v1" + "strconv" + "time" +) + +type KubernetesRunnerManager struct { + *AbstractManager + apiClient kubernetes.ExecutorAPI + reloadingEnvironment storage.Storage[*alertData] +} + +func NewKubernetesRunnerManager(apiClient *kubernetes.ExecutorAPI, ctx context.Context) *KubernetesRunnerManager { + return &KubernetesRunnerManager{ + AbstractManager: NewAbstractManager(ctx), + apiClient: *apiClient, + reloadingEnvironment: storage.NewLocalStorage[*alertData](), + } +} + +// Load recovers all runners for all existing environments. +func (k *KubernetesRunnerManager) Load() { + log.Info("Loading runners") + newUsedRunners := storage.NewLocalStorage[Runner]() + for _, environment := range k.ListEnvironments() { + usedRunners, err := k.loadEnvironment(environment) + if err != nil { + log.WithError(err).WithField(dto.KeyEnvironmentID, environment.ID().ToString()). + Warn("Failed loading environment. Skipping...") + continue + } + for _, r := range usedRunners.List() { + newUsedRunners.Add(r.ID(), r) + } + } + // TODO MISSING IMPLEMENTATION + //k.updateUsedRunners(newUsedRunners, true) +} + +func (k *KubernetesRunnerManager) loadEnvironment(environment ExecutionEnvironment) (used storage.Storage[Runner], err error) { + used = storage.NewLocalStorage[Runner]() + + runnerJobs, err := k.apiClient.LoadRunnerJobs(environment.ID()) + if err != nil { + return nil, fmt.Errorf("failed fetching the runner jobs: %w", err) + } + for _, job := range runnerJobs { + r, isUsed, err := k.loadSingleJob(job, environment) + if err != nil { + log.WithError(err).WithField(dto.KeyEnvironmentID, environment.ID().ToString()). + WithField("used", isUsed).Warn("Failed loading job. Skipping...") + continue + } else if isUsed { + used.Add(r.ID(), r) + } + } + err = environment.ApplyPrewarmingPoolSize() + if err != nil { + return used, fmt.Errorf("couldn't scale environment: %w", err) + } + return used, nil +} + +func (k *KubernetesRunnerManager) loadSingleJob(deployment *appv1.Deployment, environment ExecutionEnvironment) (r Runner, isUsed bool, err error) { + configTaskGroup := deployment.Spec.Template + if err != nil { + return nil, false, fmt.Errorf("%w, %s", nomad.ErrorMissingTaskGroup, deployment.Name) + } + + isUsed = configTaskGroup.Annotations[nomad.ConfigMetaUsedKey] == nomad.ConfigMetaUsedValue + portMappings, err := k.apiClient.LoadRunnerPortMappings(deployment.Name) + if err != nil { + return nil, false, fmt.Errorf("error loading runner portMappings: %w", err) + } + + newJob := NewKubernetesDeployment(deployment.Name, portMappings, k.apiClient, k.onRunnerDestroyed) + log.WithField("isUsed", isUsed).WithField(dto.KeyRunnerID, newJob.ID()).Debug("Recovered Runner") + if isUsed { + timeout, err := strconv.Atoi(configTaskGroup.ObjectMeta.Annotations[nomad.ConfigMetaTimeoutKey]) + if err != nil { + log.WithField(dto.KeyRunnerID, newJob.ID()).WithError(err).Warn("failed loading timeout from meta values") + timeout = int(nomad.RunnerTimeoutFallback.Seconds()) + go k.markRunnerAsUsed(newJob, timeout) + } + newJob.SetupTimeout(time.Duration(timeout) * time.Second) + } else { + environment.AddRunner(newJob) + } + return newJob, isUsed, nil +} + +func (k *KubernetesRunnerManager) markRunnerAsUsed(runner Runner, timeoutDuration int) { + err := util.RetryExponential(func() (err error) { + if err = k.apiClient.MarkRunnerAsUsed(runner.ID(), timeoutDuration); err != nil { + err = fmt.Errorf("cannot mark runner as used: %w", err) + } + return + }) + if err != nil { + log.WithError(err).WithField(dto.KeyRunnerID, runner.ID()).Error("cannot mark runner as used") + err := k.Return(runner) + if err != nil { + log.WithError(err).WithField(dto.KeyRunnerID, runner.ID()).Error("can't mark runner as used and can't return runner") + } + } +} + +func (k *KubernetesRunnerManager) onRunnerDestroyed(r Runner) error { + k.usedRunners.Delete(r.ID()) + + environment, ok := k.GetEnvironment(r.Environment()) + if ok { + environment.DeleteRunner(r.ID()) + } + return nil +} diff --git a/internal/runner/kubernetes_runner.go b/internal/runner/kubernetes_runner.go new file mode 100644 index 0000000..2921946 --- /dev/null +++ b/internal/runner/kubernetes_runner.go @@ -0,0 +1,107 @@ +package runner + +import ( + "context" + "fmt" + "github.com/openHPI/poseidon/internal/kubernetes" + "github.com/openHPI/poseidon/internal/nomad" + "github.com/openHPI/poseidon/pkg/dto" + "github.com/openHPI/poseidon/pkg/monitoring" + "github.com/openHPI/poseidon/pkg/storage" + "io" + v1 "k8s.io/api/core/v1" + "net/http" + "time" +) + +// NomadJob is an abstraction to communicate with Nomad environments. +type KubernetesDeployment struct { + InactivityTimer + executions storage.Storage[*dto.ExecutionRequest] + id string + portMappings []v1.ContainerPort + api kubernetes.ExecutorAPI + onDestroy DestroyRunnerHandler + ctx context.Context + cancel context.CancelFunc +} + +func (r *KubernetesDeployment) MappedPorts() []*dto.MappedPort { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) StoreExecution(id string, executionRequest *dto.ExecutionRequest) { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) ExecutionExists(id string) bool { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) ExecuteInteractively(id string, stdin io.ReadWriter, stdout, stderr io.Writer, ctx context.Context) (exit <-chan ExitInfo, cancel context.CancelFunc, err error) { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) ListFileSystem(path string, recursive bool, result io.Writer, privilegedExecution bool, ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) UpdateFileSystem(request *dto.UpdateFileSystemRequest, ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) GetFileContent(path string, content http.ResponseWriter, privilegedExecution bool, ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) Destroy(reason DestroyReason) error { + //TODO implement me + panic("implement me") +} + +func (r *KubernetesDeployment) ID() string { + return r.id +} + +func (r *KubernetesDeployment) Environment() dto.EnvironmentID { + id, err := nomad.EnvironmentIDFromRunnerID(r.ID()) + if err != nil { + log.WithError(err).Error("Runners must have correct IDs") + } + return id +} + +// NewNomadJob creates a new NomadJob with the provided id. +// The InactivityTimer is used actively. It executes onDestroy when it has expired. +// The InactivityTimer is persisted in Nomad by the runner manager's Claim Function. +func NewKubernetesDeployment(id string, portMappings []v1.ContainerPort, + apiClient kubernetes.ExecutorAPI, onDestroy DestroyRunnerHandler, +) *KubernetesDeployment { + ctx := context.WithValue(context.Background(), dto.ContextKey(dto.KeyRunnerID), id) + ctx, cancel := context.WithCancel(ctx) + job := &KubernetesDeployment{ + id: id, + portMappings: portMappings, + api: apiClient, + onDestroy: onDestroy, + ctx: ctx, + cancel: cancel, + } + job.executions = storage.NewMonitoredLocalStorage[*dto.ExecutionRequest]( + monitoring.MeasurementExecutionsNomad, monitorExecutionsRunnerID(job.Environment(), id), time.Minute, ctx) + job.InactivityTimer = NewInactivityTimer(job, func(r Runner) error { + err := r.Destroy(ErrorRunnerInactivityTimeout) + if err != nil { + err = fmt.Errorf("NomadJob: %w", err) + } + return err + }) + return job +} diff --git a/internal/runner/manager_mock.go b/internal/runner/manager_mock.go deleted file mode 100644 index e31e61a..0000000 --- a/internal/runner/manager_mock.go +++ /dev/null @@ -1,178 +0,0 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. - -package runner - -import ( - dto "github.com/openHPI/poseidon/pkg/dto" - mock "github.com/stretchr/testify/mock" -) - -// ManagerMock is an autogenerated mock type for the Manager type -type ManagerMock struct { - mock.Mock -} - -// Claim provides a mock function with given fields: id, duration -func (_m *ManagerMock) Claim(id dto.EnvironmentID, duration int) (Runner, error) { - ret := _m.Called(id, duration) - - var r0 Runner - if rf, ok := ret.Get(0).(func(dto.EnvironmentID, int) Runner); ok { - r0 = rf(id, duration) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(Runner) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(dto.EnvironmentID, int) error); ok { - r1 = rf(id, duration) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// DeleteEnvironment provides a mock function with given fields: id -func (_m *ManagerMock) DeleteEnvironment(id dto.EnvironmentID) { - _m.Called(id) -} - -// EnvironmentStatistics provides a mock function with given fields: -func (_m *ManagerMock) EnvironmentStatistics() map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData { - ret := _m.Called() - - var r0 map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData - if rf, ok := ret.Get(0).(func() map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[dto.EnvironmentID]*dto.StatisticalExecutionEnvironmentData) - } - } - - return r0 -} - -// Get provides a mock function with given fields: runnerID -func (_m *ManagerMock) Get(runnerID string) (Runner, error) { - ret := _m.Called(runnerID) - - var r0 Runner - if rf, ok := ret.Get(0).(func(string) Runner); ok { - r0 = rf(runnerID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(Runner) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(runnerID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetEnvironment provides a mock function with given fields: id -func (_m *ManagerMock) GetEnvironment(id dto.EnvironmentID) (ExecutionEnvironment, bool) { - ret := _m.Called(id) - - var r0 ExecutionEnvironment - if rf, ok := ret.Get(0).(func(dto.EnvironmentID) ExecutionEnvironment); ok { - r0 = rf(id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(ExecutionEnvironment) - } - } - - var r1 bool - if rf, ok := ret.Get(1).(func(dto.EnvironmentID) bool); ok { - r1 = rf(id) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// HasNextHandler provides a mock function with given fields: -func (_m *ManagerMock) HasNextHandler() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// ListEnvironments provides a mock function with given fields: -func (_m *ManagerMock) ListEnvironments() []ExecutionEnvironment { - ret := _m.Called() - - var r0 []ExecutionEnvironment - if rf, ok := ret.Get(0).(func() []ExecutionEnvironment); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]ExecutionEnvironment) - } - } - - return r0 -} - -// Load provides a mock function with given fields: -func (_m *ManagerMock) Load() { - _m.Called() -} - -// NextHandler provides a mock function with given fields: -func (_m *ManagerMock) NextHandler() AccessorHandler { - ret := _m.Called() - - var r0 AccessorHandler - if rf, ok := ret.Get(0).(func() AccessorHandler); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(AccessorHandler) - } - } - - return r0 -} - -// Return provides a mock function with given fields: r -func (_m *ManagerMock) Return(r Runner) error { - ret := _m.Called(r) - - var r0 error - if rf, ok := ret.Get(0).(func(Runner) error); ok { - r0 = rf(r) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetNextHandler provides a mock function with given fields: m -func (_m *ManagerMock) SetNextHandler(m AccessorHandler) { - _m.Called(m) -} - -// StoreEnvironment provides a mock function with given fields: environment -func (_m *ManagerMock) StoreEnvironment(environment ExecutionEnvironment) { - _m.Called(environment) -} diff --git a/internal/runner/nomad_manager.go b/internal/runner/nomad_manager.go index c08cffc..a85c83d 100644 --- a/internal/runner/nomad_manager.go +++ b/internal/runner/nomad_manager.go @@ -42,6 +42,7 @@ func NewNomadRunnerManager(apiClient nomad.ExecutorAPI, ctx context.Context) *No return &NomadRunnerManager{NewAbstractManager(ctx), apiClient, storage.NewLocalStorage[*alertData]()} } +// Claim returns a runner for the given environment. The runner will be marked as used for the given duration. func (m *NomadRunnerManager) Claim(environmentID dto.EnvironmentID, duration int) (Runner, error) { environment, ok := m.GetEnvironment(environmentID) if !ok { @@ -185,6 +186,7 @@ func (m *NomadRunnerManager) checkPrewarmingPoolAlert(environment ExecutionEnvir func (m *NomadRunnerManager) loadEnvironment(environment ExecutionEnvironment) (used storage.Storage[Runner], err error) { used = storage.NewLocalStorage[Runner]() + runnerJobs, err := m.apiClient.LoadRunnerJobs(environment.ID()) if err != nil { return nil, fmt.Errorf("failed fetching the runner jobs: %w", err) diff --git a/internal/runner/nomad_manager_test.go b/internal/runner/nomad_manager_test.go deleted file mode 100644 index 5dea5c0..0000000 --- a/internal/runner/nomad_manager_test.go +++ /dev/null @@ -1,716 +0,0 @@ -package runner - -import ( - "context" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/storage" - "github.com/openHPI/poseidon/pkg/util" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "strconv" - "testing" - "time" -) - -func TestGetNextRunnerTestSuite(t *testing.T) { - suite.Run(t, new(ManagerTestSuite)) -} - -type ManagerTestSuite struct { - tests.MemoryLeakTestSuite - apiMock *nomad.ExecutorAPIMock - nomadRunnerManager *NomadRunnerManager - exerciseEnvironment *ExecutionEnvironmentMock - exerciseRunner Runner -} - -func (s *ManagerTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.apiMock = &nomad.ExecutorAPIMock{} - mockRunnerQueries(s.TestCtx, s.apiMock, []string{}) - // Instantly closed context to manually start the update process in some cases - ctx, cancel := context.WithCancel(context.Background()) - cancel() - s.nomadRunnerManager = NewNomadRunnerManager(s.apiMock, ctx) - - s.exerciseRunner = NewNomadJob(tests.DefaultRunnerID, nil, s.apiMock, s.nomadRunnerManager.onRunnerDestroyed) - s.exerciseEnvironment = createBasicEnvironmentMock(defaultEnvironmentID) - s.nomadRunnerManager.StoreEnvironment(s.exerciseEnvironment) -} - -func (s *ManagerTestSuite) TearDownTest() { - defer s.MemoryLeakTestSuite.TearDownTest() - err := s.exerciseRunner.Destroy(nil) - s.Require().NoError(err) -} - -func mockRunnerQueries(ctx context.Context, apiMock *nomad.ExecutorAPIMock, returnedRunnerIds []string) { - // reset expected calls to allow new mocked return values - apiMock.ExpectedCalls = []*mock.Call{} - call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) - call.Run(func(args mock.Arguments) { - <-ctx.Done() - call.ReturnArguments = mock.Arguments{nil} - }) - apiMock.On("LoadEnvironmentJobs").Return([]*nomadApi.Job{}, nil) - apiMock.On("LoadRunnerJobs", mock.AnythingOfType("dto.EnvironmentID")).Return([]*nomadApi.Job{}, nil) - apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string"), mock.AnythingOfType("int")).Return(nil) - apiMock.On("LoadRunnerIDs", tests.DefaultRunnerID).Return(returnedRunnerIds, nil) - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - apiMock.On("JobScale", tests.DefaultRunnerID).Return(uint(len(returnedRunnerIds)), nil) - apiMock.On("SetJobScale", tests.DefaultRunnerID, mock.AnythingOfType("uint"), "Runner Requested").Return(nil) - apiMock.On("RegisterRunnerJob", mock.Anything).Return(nil) - apiMock.On("MonitorEvaluation", mock.Anything, mock.Anything).Return(nil) -} - -func mockIdleRunners(environmentMock *ExecutionEnvironmentMock) { - tests.RemoveMethodFromMock(&environmentMock.Mock, "DeleteRunner") - idleRunner := storage.NewLocalStorage[Runner]() - environmentMock.On("AddRunner", mock.Anything).Run(func(args mock.Arguments) { - r, ok := args.Get(0).(Runner) - if !ok { - return - } - idleRunner.Add(r.ID(), r) - }) - sampleCall := environmentMock.On("Sample", mock.Anything) - sampleCall.Run(func(args mock.Arguments) { - r, ok := idleRunner.Sample() - sampleCall.ReturnArguments = mock.Arguments{r, ok} - }) - deleteCall := environmentMock.On("DeleteRunner", mock.AnythingOfType("string")) - deleteCall.Run(func(args mock.Arguments) { - id, ok := args.Get(0).(string) - if !ok { - log.Fatal("Cannot parse ID") - } - r, ok := idleRunner.Get(id) - deleteCall.ReturnArguments = mock.Arguments{r, ok} - if !ok { - return - } - idleRunner.Delete(id) - }) -} - -func (s *ManagerTestSuite) waitForRunnerRefresh() { - <-time.After(tests.ShortTimeout) -} - -func (s *ManagerTestSuite) TestSetEnvironmentAddsNewEnvironment() { - anotherEnvironment := createBasicEnvironmentMock(anotherEnvironmentID) - s.nomadRunnerManager.StoreEnvironment(anotherEnvironment) - - job, ok := s.nomadRunnerManager.environments.Get(anotherEnvironmentID.ToString()) - s.True(ok) - s.NotNil(job) -} - -func (s *ManagerTestSuite) TestClaimReturnsNotFoundErrorIfEnvironmentNotFound() { - runner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout) - s.Nil(runner) - s.Equal(ErrUnknownExecutionEnvironment, err) -} - -func (s *ManagerTestSuite) TestClaimReturnsRunnerIfAvailable() { - s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) - receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.NoError(err) - s.Equal(s.exerciseRunner, receivedRunner) -} - -func (s *ManagerTestSuite) TestClaimReturnsErrorIfNoRunnerAvailable() { - s.waitForRunnerRefresh() - s.exerciseEnvironment.On("Sample", mock.Anything).Return(nil, false) - runner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.Nil(runner) - s.Equal(ErrNoRunnersAvailable, err) -} - -func (s *ManagerTestSuite) TestClaimReturnsNoRunnerOfDifferentEnvironment() { - s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) - receivedRunner, err := s.nomadRunnerManager.Claim(anotherEnvironmentID, defaultInactivityTimeout) - s.Nil(receivedRunner) - s.Error(err) -} - -func (s *ManagerTestSuite) TestClaimDoesNotReturnTheSameRunnerTwice() { - s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true).Once() - secondRunner := NewNomadJob(tests.AnotherRunnerID, nil, s.apiMock, s.nomadRunnerManager.onRunnerDestroyed) - s.exerciseEnvironment.On("Sample", mock.Anything).Return(secondRunner, true).Once() - - firstReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.NoError(err) - secondReceivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.NoError(err) - s.NotEqual(firstReceivedRunner, secondReceivedRunner) - - err = secondRunner.Destroy(nil) - s.NoError(err) -} - -func (s *ManagerTestSuite) TestClaimAddsRunnerToUsedRunners() { - s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) - receivedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.Require().NoError(err) - savedRunner, ok := s.nomadRunnerManager.usedRunners.Get(receivedRunner.ID()) - s.True(ok) - s.Equal(savedRunner, receivedRunner) -} - -func (s *ManagerTestSuite) TestClaimRemovesRunnerWhenMarkAsUsedFails() { - s.exerciseEnvironment.On("Sample", mock.Anything).Return(s.exerciseRunner, true) - s.exerciseEnvironment.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil, false) - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - util.MaxConnectionRetriesExponential = 1 - modifyMockedCall(s.apiMock, "MarkRunnerAsUsed", func(call *mock.Call) { - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{tests.ErrDefault} - }) - }) - - claimedRunner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.Require().NoError(err) - <-time.After(time.Second + tests.ShortTimeout) // Claimed runners are marked as used asynchronously - s.apiMock.AssertCalled(s.T(), "DeleteJob", claimedRunner.ID()) - _, ok := s.nomadRunnerManager.usedRunners.Get(claimedRunner.ID()) - s.False(ok) -} - -func (s *ManagerTestSuite) TestGetReturnsRunnerIfRunnerIsUsed() { - s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner.ID(), s.exerciseRunner) - savedRunner, err := s.nomadRunnerManager.Get(s.exerciseRunner.ID()) - s.NoError(err) - s.Equal(savedRunner, s.exerciseRunner) -} - -func (s *ManagerTestSuite) TestGetReturnsErrorIfRunnerNotFound() { - savedRunner, err := s.nomadRunnerManager.Get(tests.DefaultRunnerID) - s.Nil(savedRunner) - s.Error(err) -} - -func (s *ManagerTestSuite) TestReturnRemovesRunnerFromUsedRunners() { - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.exerciseEnvironment.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil, false) - s.nomadRunnerManager.usedRunners.Add(s.exerciseRunner.ID(), s.exerciseRunner) - err := s.nomadRunnerManager.Return(s.exerciseRunner) - s.Nil(err) - _, ok := s.nomadRunnerManager.usedRunners.Get(s.exerciseRunner.ID()) - s.False(ok) -} - -func (s *ManagerTestSuite) TestReturnCallsDeleteRunnerApiMethod() { - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.exerciseEnvironment.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil, false) - err := s.nomadRunnerManager.Return(s.exerciseRunner) - s.Nil(err) - s.apiMock.AssertCalled(s.T(), "DeleteJob", s.exerciseRunner.ID()) -} - -func (s *ManagerTestSuite) TestReturnReturnsErrorWhenApiCallFailed() { - tests.RemoveMethodFromMock(&s.apiMock.Mock, "DeleteJob") - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(tests.ErrDefault) - defer s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - defer tests.RemoveMethodFromMock(&s.apiMock.Mock, "DeleteJob") - s.exerciseEnvironment.On("DeleteRunner", mock.AnythingOfType("string")).Return(nil, false) - - util.MaxConnectionRetriesExponential = 1 - util.InitialWaitingDuration = 2 * tests.ShortTimeout - - chReturnDone := make(chan error) - go func(done chan<- error) { - err := s.nomadRunnerManager.Return(s.exerciseRunner) - select { - case <-s.TestCtx.Done(): - case done <- err: - } - close(done) - }(chReturnDone) - - select { - case <-chReturnDone: - s.Fail("Return should not return if the API request failed") - case <-time.After(tests.ShortTimeout): - } - - select { - case err := <-chReturnDone: - s.ErrorIs(err, tests.ErrDefault) - case <-time.After(2 * tests.ShortTimeout): - s.Fail("Return should return after the retry mechanism") - // note: MaxConnectionRetriesExponential and InitialWaitingDuration is decreased extremely here. - } -} - -func (s *ManagerTestSuite) TestUpdateRunnersLogsErrorFromWatchAllocation() { - var hook *test.Hook - logger, hook := test.NewNullLogger() - log = logger.WithField("pkg", "runner") - modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { - call.Run(func(args mock.Arguments) { - call.ReturnArguments = mock.Arguments{tests.ErrDefault} - }) - }) - - err := s.nomadRunnerManager.SynchronizeRunners(s.TestCtx) - if err != nil { - log.WithError(err).Error("failed to synchronize runners") - } - - s.Require().Equal(2, len(hook.Entries)) - s.Equal(logrus.ErrorLevel, hook.LastEntry().Level) - err, ok := hook.LastEntry().Data[logrus.ErrorKey].(error) - s.Require().True(ok) - s.ErrorIs(err, tests.ErrDefault) -} - -func (s *ManagerTestSuite) TestUpdateRunnersAddsIdleRunner() { - allocation := &nomadApi.Allocation{ID: tests.DefaultRunnerID} - environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID.ToString()) - s.Require().True(ok) - allocation.JobID = environment.ID().ToString() - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - _, ok = environment.Sample() - s.Require().False(ok) - - modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { - call.Run(func(args mock.Arguments) { - callbacks, ok := args.Get(1).(*nomad.AllocationProcessing) - s.Require().True(ok) - callbacks.OnNew(allocation, 0) - call.ReturnArguments = mock.Arguments{nil} - }) - }) - - go func() { - err := s.nomadRunnerManager.SynchronizeRunners(s.TestCtx) - if err != nil { - log.WithError(err).Error("failed to synchronize runners") - } - }() - <-time.After(10 * time.Millisecond) - - r, ok := environment.Sample() - s.True(ok) - s.NoError(r.Destroy(nil)) -} - -func (s *ManagerTestSuite) TestUpdateRunnersRemovesIdleAndUsedRunner() { - allocation := &nomadApi.Allocation{JobID: tests.DefaultRunnerID} - environment, ok := s.nomadRunnerManager.environments.Get(defaultEnvironmentID.ToString()) - s.Require().True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - testRunner := NewNomadJob(allocation.JobID, nil, s.apiMock, s.nomadRunnerManager.onRunnerDestroyed) - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - environment.AddRunner(testRunner) - s.nomadRunnerManager.usedRunners.Add(testRunner.ID(), testRunner) - - modifyMockedCall(s.apiMock, "WatchEventStream", func(call *mock.Call) { - call.Run(func(args mock.Arguments) { - callbacks, ok := args.Get(1).(*nomad.AllocationProcessing) - s.Require().True(ok) - callbacks.OnDeleted(allocation.JobID, nil) - call.ReturnArguments = mock.Arguments{nil} - }) - }) - - go func() { - err := s.nomadRunnerManager.SynchronizeRunners(s.TestCtx) - if err != nil { - log.WithError(err).Error("failed to synchronize runners") - } - }() - <-time.After(tests.ShortTimeout) - - _, ok = environment.Sample() - s.False(ok) - _, ok = s.nomadRunnerManager.usedRunners.Get(allocation.JobID) - s.False(ok) -} - -func modifyMockedCall(apiMock *nomad.ExecutorAPIMock, method string, modifier func(call *mock.Call)) { - for _, c := range apiMock.ExpectedCalls { - if c.Method == method { - modifier(c) - } - } -} - -func (s *ManagerTestSuite) TestOnAllocationAdded() { - s.Run("does not add environment template id job", func() { - environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsString) - s.True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - alloc := &nomadApi.Allocation{JobID: nomad.TemplateJobID(tests.DefaultEnvironmentIDAsInteger)} - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - - _, ok = environment.Sample() - s.False(ok) - }) - s.Run("does not panic when environment id cannot be parsed", func() { - alloc := &nomadApi.Allocation{JobID: ""} - s.NotPanics(func() { - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - }) - }) - s.Run("does not panic when environment does not exist", func() { - nonExistentEnvironment := dto.EnvironmentID(1234) - _, ok := s.nomadRunnerManager.environments.Get(nonExistentEnvironment.ToString()) - s.Require().False(ok) - - alloc := &nomadApi.Allocation{JobID: nomad.RunnerJobID(nonExistentEnvironment, "1-1-1-1")} - s.NotPanics(func() { - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - }) - }) - s.Run("adds correct job", func() { - s.Run("without allocated resources", func() { - environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsString) - s.True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - _, ok = environment.Sample() - s.Require().False(ok) - - alloc := &nomadApi.Allocation{ - JobID: tests.DefaultRunnerID, - AllocatedResources: nil, - } - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - - runner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.NoError(err) - nomadJob, ok := runner.(*NomadJob) - s.True(ok) - s.Equal(nomadJob.id, tests.DefaultRunnerID) - s.Empty(nomadJob.portMappings) - - s.Run("but not again", func() { - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - runner, err = s.nomadRunnerManager.Claim(defaultEnvironmentID, defaultInactivityTimeout) - s.Error(err) - }) - - err = nomadJob.Destroy(nil) - s.NoError(err) - }) - s.nomadRunnerManager.usedRunners.Purge() - s.Run("with mapped ports", func() { - environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsString) - s.True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - alloc := &nomadApi.Allocation{ - JobID: tests.DefaultRunnerID, - AllocatedResources: &nomadApi.AllocatedResources{ - Shared: nomadApi.AllocatedSharedResources{Ports: tests.DefaultPortMappings}, - }, - } - s.nomadRunnerManager.onAllocationAdded(alloc, 0) - - runner, ok := environment.Sample() - s.True(ok) - nomadJob, ok := runner.(*NomadJob) - s.True(ok) - s.Equal(nomadJob.id, tests.DefaultRunnerID) - s.Equal(nomadJob.portMappings, tests.DefaultPortMappings) - - err := runner.Destroy(nil) - s.NoError(err) - }) - }) -} - -func (s *ManagerTestSuite) TestOnAllocationStopped() { - s.Run("returns false for idle runner", func() { - environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsString) - s.Require().True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - r := NewNomadJob(tests.DefaultRunnerID, []nomadApi.PortMapping{}, s.apiMock, func(r Runner) error { return nil }) - environment.AddRunner(r) - alreadyRemoved := s.nomadRunnerManager.onAllocationStopped(tests.DefaultRunnerID, nil) - s.False(alreadyRemoved) - s.Error(r.ctx.Err(), "The runner should be destroyed and its context canceled") - }) - s.Run("returns false and stops inactivity timer", func() { - runner, runnerDestroyed := testStoppedInactivityTimer(s) - - alreadyRemoved := s.nomadRunnerManager.onAllocationStopped(runner.ID(), nil) - s.False(alreadyRemoved) - - select { - case <-time.After(time.Second + tests.ShortTimeout): - s.Fail("runner was stopped too late") - case <-runnerDestroyed: - s.False(runner.TimeoutPassed()) - } - }) - s.Run("stops inactivity timer - counter check", func() { - runner, runnerDestroyed := testStoppedInactivityTimer(s) - - select { - case <-time.After(time.Second + tests.ShortTimeout): - s.Fail("runner was stopped too late") - case <-runnerDestroyed: - s.True(runner.TimeoutPassed()) - } - }) - s.Run("returns true when the runner is already removed", func() { - s.Run("by the inactivity timer", func() { - runner, _ := testStoppedInactivityTimer(s) - - <-time.After(time.Second) - s.Require().True(runner.TimeoutPassed()) - - alreadyRemoved := s.nomadRunnerManager.onAllocationStopped(runner.ID(), nil) - s.True(alreadyRemoved) - }) - }) -} - -func testStoppedInactivityTimer(s *ManagerTestSuite) (r Runner, destroyed chan struct{}) { - s.T().Helper() - environment, ok := s.nomadRunnerManager.environments.Get(tests.DefaultEnvironmentIDAsString) - s.Require().True(ok) - mockIdleRunners(environment.(*ExecutionEnvironmentMock)) - - runnerDestroyed := make(chan struct{}) - environment.AddRunner(NewNomadJob(tests.DefaultRunnerID, []nomadApi.PortMapping{}, s.apiMock, func(r Runner) error { - go func() { - select { - case runnerDestroyed <- struct{}{}: - case <-s.TestCtx.Done(): - } - }() - return s.nomadRunnerManager.onRunnerDestroyed(r) - })) - - runner, err := s.nomadRunnerManager.Claim(defaultEnvironmentID, 1) - s.Require().NoError(err) - s.Require().False(runner.TimeoutPassed()) - select { - case runnerDestroyed <- struct{}{}: - s.Fail("The runner should not be removed by now") - case <-time.After(tests.ShortTimeout): - } - - return runner, runnerDestroyed -} - -func (s *MainTestSuite) TestNomadRunnerManager_Load() { - apiMock := &nomad.ExecutorAPIMock{} - mockWatchAllocations(s.TestCtx, apiMock) - apiMock.On("LoadRunnerPortMappings", mock.AnythingOfType("string")). - Return([]nomadApi.PortMapping{}, nil) - call := apiMock.On("LoadRunnerJobs", dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - runnerManager := NewNomadRunnerManager(apiMock, s.TestCtx) - environmentMock := createBasicEnvironmentMock(tests.DefaultEnvironmentIDAsInteger) - environmentMock.On("ApplyPrewarmingPoolSize").Return(nil) - runnerManager.StoreEnvironment(environmentMock) - - s.Run("Stores unused runner", func() { - tests.RemoveMethodFromMock(&environmentMock.Mock, "DeleteRunner") - environmentMock.On("AddRunner", mock.AnythingOfType("*runner.NomadJob")).Once() - - _, job := helpers.CreateTemplateJob() - jobID := tests.DefaultRunnerID - job.ID = &jobID - job.Name = &jobID - s.ExpectedGoroutineIncrease++ // We dont care about destroying the created runner. - call.Return([]*nomadApi.Job{job}, nil) - - runnerManager.Load() - environmentMock.AssertExpectations(s.T()) - }) - - s.Run("Stores used runner", func() { - apiMock.On("MarkRunnerAsUsed", mock.AnythingOfType("string"), mock.AnythingOfType("int")).Return(nil) - _, job := helpers.CreateTemplateJob() - jobID := tests.DefaultRunnerID - job.ID = &jobID - job.Name = &jobID - configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName) - s.Require().NotNil(configTaskGroup) - configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue - s.ExpectedGoroutineIncrease++ // We don't care about destroying the created runner. - call.Return([]*nomadApi.Job{job}, nil) - - s.Require().Zero(runnerManager.usedRunners.Length()) - runnerManager.Load() - _, ok := runnerManager.usedRunners.Get(tests.DefaultRunnerID) - s.True(ok) - }) - - runnerManager.usedRunners.Purge() - s.Run("Restart timeout of used runner", func() { - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - environmentMock.On("DeleteRunner", mock.AnythingOfType("string")).Once().Return(nil, false) - timeout := 1 - - _, job := helpers.CreateTemplateJob() - jobID := tests.DefaultRunnerID - job.ID = &jobID - job.Name = &jobID - configTaskGroup := nomad.FindTaskGroup(job, nomad.ConfigTaskGroupName) - s.Require().NotNil(configTaskGroup) - configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue - configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey] = strconv.Itoa(timeout) - call.Return([]*nomadApi.Job{job}, nil) - - s.Require().Zero(runnerManager.usedRunners.Length()) - runnerManager.Load() - s.Require().NotZero(runnerManager.usedRunners.Length()) - - <-time.After(time.Duration(timeout*2) * time.Second) - s.Require().Zero(runnerManager.usedRunners.Length()) - }) -} - -func (s *MainTestSuite) TestNomadRunnerManager_checkPrewarmingPoolAlert() { - timeout := uint(1) - config.Config.Server.Alert.PrewarmingPoolReloadTimeout = timeout - config.Config.Server.Alert.PrewarmingPoolThreshold = 0.5 - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - environment.On("Image").Return("") - environment.On("CPULimit").Return(uint(0)) - environment.On("MemoryLimit").Return(uint(0)) - environment.On("NetworkAccess").Return(false, nil) - apiMock := &nomad.ExecutorAPIMock{} - m := NewNomadRunnerManager(apiMock, s.TestCtx) - m.StoreEnvironment(environment) - s.Run("checks the alert condition again after the reload timeout", func() { - environment.On("PrewarmingPoolSize").Return(uint(1)).Once() - environment.On("IdleRunnerCount").Return(uint(0)).Once() - environment.On("PrewarmingPoolSize").Return(uint(1)).Once() - environment.On("IdleRunnerCount").Return(uint(1)).Once() - - checkDone := make(chan struct{}) - go func() { - m.checkPrewarmingPoolAlert(environment, false) - close(checkDone) - }() - - select { - case <-checkDone: - s.Fail("checkPrewarmingPoolAlert returned before the reload timeout") - case <-time.After(time.Duration(timeout) * time.Second / 2): - } - - select { - case <-time.After(time.Duration(timeout) * time.Second): - s.Fail("checkPrewarmingPoolAlert did not return after checking the alert condition again") - case <-checkDone: - } - environment.AssertExpectations(s.T()) - }) - s.Run("checks the alert condition again after the reload timeout", func() { - environment.On("PrewarmingPoolSize").Return(uint(1)).Twice() - environment.On("IdleRunnerCount").Return(uint(0)).Twice() - apiMock.On("LoadRunnerJobs", environment.ID()).Return([]*nomadApi.Job{}, nil).Once() - environment.On("ApplyPrewarmingPoolSize").Return(nil).Once() - - checkDone := make(chan struct{}) - go func() { - m.checkPrewarmingPoolAlert(environment, false) - close(checkDone) - }() - - select { - case <-time.After(time.Duration(timeout) * time.Second * 2): - s.Fail("checkPrewarmingPoolAlert did not return") - case <-checkDone: - } - environment.AssertExpectations(s.T()) - }) - s.Run("is canceled by an added runner", func() { - environment.On("PrewarmingPoolSize").Return(uint(1)).Twice() - environment.On("IdleRunnerCount").Return(uint(0)).Once() - environment.On("IdleRunnerCount").Return(uint(1)).Once() - - checkDone := make(chan struct{}) - go func() { - m.checkPrewarmingPoolAlert(environment, false) - close(checkDone) - }() - - <-time.After(tests.ShortTimeout) - go m.checkPrewarmingPoolAlert(environment, true) - <-time.After(tests.ShortTimeout) - - select { - case <-time.After(100 * time.Duration(timeout) * time.Second): - s.Fail("checkPrewarmingPoolAlert was not canceled") - case <-checkDone: - } - environment.AssertExpectations(s.T()) - }) -} - -func (s *MainTestSuite) TestNomadRunnerManager_checkPrewarmingPoolAlert_reloadsRunners() { - config.Config.Server.Alert.PrewarmingPoolReloadTimeout = uint(1) - config.Config.Server.Alert.PrewarmingPoolThreshold = 0.5 - environment := &ExecutionEnvironmentMock{} - environment.On("ID").Return(dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - environment.On("Image").Return("") - environment.On("CPULimit").Return(uint(0)) - environment.On("MemoryLimit").Return(uint(0)) - environment.On("NetworkAccess").Return(false, nil) - apiMock := &nomad.ExecutorAPIMock{} - m := NewNomadRunnerManager(apiMock, s.TestCtx) - m.StoreEnvironment(environment) - - environment.On("PrewarmingPoolSize").Return(uint(1)).Twice() - environment.On("IdleRunnerCount").Return(uint(0)).Twice() - environment.On("DeleteRunner", mock.Anything).Return(nil, false).Once() - - s.Require().Empty(m.usedRunners.Length()) - _, usedJob := helpers.CreateTemplateJob() - id := tests.DefaultRunnerID - usedJob.ID = &id - configTaskGroup := nomad.FindTaskGroup(usedJob, nomad.ConfigTaskGroupName) - configTaskGroup.Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUsedValue - configTaskGroup.Meta[nomad.ConfigMetaTimeoutKey] = "42" - _, idleJob := helpers.CreateTemplateJob() - idleID := tests.AnotherRunnerID - idleJob.ID = &idleID - nomad.FindTaskGroup(idleJob, nomad.ConfigTaskGroupName).Meta[nomad.ConfigMetaUsedKey] = nomad.ConfigMetaUnusedValue - apiMock.On("LoadRunnerJobs", environment.ID()).Return([]*nomadApi.Job{usedJob, idleJob}, nil).Once() - apiMock.On("LoadRunnerPortMappings", mock.Anything).Return(nil, nil).Twice() - environment.On("ApplyPrewarmingPoolSize").Return(nil).Once() - environment.On("AddRunner", mock.Anything).Run(func(args mock.Arguments) { - job, ok := args[0].(*NomadJob) - s.Require().True(ok) - err := job.Destroy(ErrLocalDestruction) - s.NoError(err) - }).Return().Once() - - m.checkPrewarmingPoolAlert(environment, false) - - r, ok := m.usedRunners.Get(tests.DefaultRunnerID) - s.Require().True(ok) - err := r.Destroy(ErrLocalDestruction) - s.NoError(err) - - environment.AssertExpectations(s.T()) -} - -func mockWatchAllocations(ctx context.Context, apiMock *nomad.ExecutorAPIMock) { - call := apiMock.On("WatchEventStream", mock.Anything, mock.Anything, mock.Anything) - call.Run(func(args mock.Arguments) { - <-ctx.Done() - call.ReturnArguments = mock.Arguments{nil} - }) -} diff --git a/internal/runner/nomad_runner_test.go b/internal/runner/nomad_runner_test.go deleted file mode 100644 index 755ddd1..0000000 --- a/internal/runner/nomad_runner_test.go +++ /dev/null @@ -1,548 +0,0 @@ -package runner - -import ( - "archive/tar" - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/logging" - "github.com/openHPI/poseidon/pkg/nullio" - "github.com/openHPI/poseidon/pkg/storage" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "io" - "regexp" - "strings" - "testing" - "time" -) - -const defaultExecutionID = "execution-id" - -func (s *MainTestSuite) TestIdIsStored() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - runner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - s.Equal(tests.DefaultRunnerID, runner.ID()) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestMappedPortsAreStoredCorrectly() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - - runner := NewNomadJob(tests.DefaultRunnerID, tests.DefaultPortMappings, apiMock, func(_ Runner) error { return nil }) - s.Equal(tests.DefaultMappedPorts, runner.MappedPorts()) - s.NoError(runner.Destroy(nil)) - - runner = NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - s.Empty(runner.MappedPorts()) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestMarshalRunner() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - runner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - marshal, err := json.Marshal(runner) - s.NoError(err) - s.Equal("{\"runnerId\":\""+tests.DefaultRunnerID+"\"}", string(marshal)) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestExecutionRequestIsStored() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - runner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - executionRequest := &dto.ExecutionRequest{ - Command: "command", - TimeLimit: 10, - Environment: nil, - } - id := "test-execution" - runner.StoreExecution(id, executionRequest) - storedExecutionRunner, ok := runner.executions.Pop(id) - - s.True(ok, "Getting an execution should not return ok false") - s.Equal(executionRequest, storedExecutionRunner) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestNewContextReturnsNewContextWithRunner() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - runner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - ctx := context.Background() - newCtx := NewContext(ctx, runner) - storedRunner, ok := newCtx.Value(runnerContextKey).(Runner) - s.Require().True(ok) - - s.NotEqual(ctx, newCtx) - s.Equal(runner, storedRunner) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestFromContextReturnsRunner() { - apiMock := &nomad.ExecutorAPIMock{} - apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - runner := NewNomadJob(tests.DefaultRunnerID, nil, apiMock, func(_ Runner) error { return nil }) - ctx := NewContext(context.Background(), runner) - storedRunner, ok := FromContext(ctx) - - s.True(ok) - s.Equal(runner, storedRunner) - s.NoError(runner.Destroy(nil)) -} - -func (s *MainTestSuite) TestFromContextReturnsIsNotOkWhenContextHasNoRunner() { - ctx := context.Background() - _, ok := FromContext(ctx) - - s.False(ok) -} - -func (s *MainTestSuite) TestDestroyDoesNotPropagateToNomadForSomeReasons() { - apiMock := &nomad.ExecutorAPIMock{} - timer := &InactivityTimerMock{} - timer.On("StopTimeout").Return() - ctx, cancel := context.WithCancel(s.TestCtx) - r := &NomadJob{ - executions: storage.NewLocalStorage[*dto.ExecutionRequest](), - InactivityTimer: timer, - id: tests.DefaultRunnerID, - api: apiMock, - ctx: ctx, - cancel: cancel, - } - - s.Run("destroy removes the runner only locally for OOM Killed Allocations", func() { - err := r.Destroy(ErrOOMKilled) - s.NoError(err) - apiMock.AssertExpectations(s.T()) - }) - - s.Run("destroy removes the runner only locally for rescheduled allocations", func() { - err := r.Destroy(nomad.ErrorAllocationRescheduled) - s.NoError(err) - apiMock.AssertExpectations(s.T()) - }) -} - -func TestExecuteInteractivelyTestSuite(t *testing.T) { - suite.Run(t, new(ExecuteInteractivelyTestSuite)) -} - -type ExecuteInteractivelyTestSuite struct { - tests.MemoryLeakTestSuite - runner *NomadJob - apiMock *nomad.ExecutorAPIMock - timer *InactivityTimerMock - manager *ManagerMock - mockedExecuteCommandCall *mock.Call - mockedTimeoutPassedCall *mock.Call -} - -func (s *ExecuteInteractivelyTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.apiMock = &nomad.ExecutorAPIMock{} - s.mockedExecuteCommandCall = s.apiMock.On("ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, - true, false, mock.Anything, mock.Anything, mock.Anything). - Return(0, nil) - s.apiMock.On("DeleteJob", mock.AnythingOfType("string")).Return(nil) - s.timer = &InactivityTimerMock{} - s.timer.On("StopTimeout").Return() - s.timer.On("ResetTimeout").Return() - s.mockedTimeoutPassedCall = s.timer.On("TimeoutPassed").Return(false) - s.manager = &ManagerMock{} - s.manager.On("Return", mock.Anything).Return(nil) - - ctx, cancel := context.WithCancel(context.Background()) - s.runner = &NomadJob{ - executions: storage.NewLocalStorage[*dto.ExecutionRequest](), - InactivityTimer: s.timer, - id: tests.DefaultRunnerID, - api: s.apiMock, - ctx: ctx, - cancel: cancel, - } -} - -func (s *ExecuteInteractivelyTestSuite) TestReturnsErrorWhenExecutionDoesNotExist() { - _, _, err := s.runner.ExecuteInteractively("non-existent-id", nil, nil, nil, context.Background()) - s.ErrorIs(err, ErrorUnknownExecution) -} - -func (s *ExecuteInteractivelyTestSuite) TestCallsApi() { - request := &dto.ExecutionRequest{Command: "echo 'Hello World!'"} - s.runner.StoreExecution(defaultExecutionID, request) - _, _, err := s.runner.ExecuteInteractively(defaultExecutionID, nil, nil, nil, context.Background()) - s.Require().NoError(err) - - time.Sleep(tests.ShortTimeout) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", tests.DefaultRunnerID, mock.Anything, request.FullCommand(), - true, false, mock.Anything, mock.Anything, mock.Anything) -} - -func (s *ExecuteInteractivelyTestSuite) TestReturnsAfterTimeout() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - <-ctx.Done() - }).Return(0, nil) - - timeLimit := 1 - executionRequest := &dto.ExecutionRequest{TimeLimit: timeLimit} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - exit, _, err := s.runner.ExecuteInteractively(defaultExecutionID, &nullio.ReadWriter{}, nil, nil, context.Background()) - s.Require().NoError(err) - - select { - case <-exit: - s.FailNow("ExecuteInteractively should not terminate instantly") - case <-time.After(tests.ShortTimeout): - } - - select { - case <-time.After(time.Duration(timeLimit) * time.Second): - s.FailNow("ExecuteInteractively should return after the time limit") - case exitInfo := <-exit: - s.Equal(uint8(255), exitInfo.Code) - } -} - -func (s *ExecuteInteractivelyTestSuite) TestSendsSignalAfterTimeout() { - quit := make(chan struct{}) - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - stdin, ok := args.Get(5).(io.Reader) - s.Require().True(ok) - buffer := make([]byte, 1) //nolint:makezero,lll // If the length is zero, the Read call never reads anything. gofmt want this alignment. - for n := 0; !(n == 1 && buffer[0] == SIGQUIT); { - <-time.After(tests.ShortTimeout) - n, _ = stdin.Read(buffer) //nolint:errcheck,lll // Read returns EOF errors but that is expected. This nolint makes the line too long. - if n > 0 { - log.WithField("buffer", fmt.Sprintf("%x", buffer[0])).Info("Received Stdin") - } - } - log.Info("After loop") - close(quit) - }).Return(0, nil) - timeLimit := 1 - executionRequest := &dto.ExecutionRequest{TimeLimit: timeLimit} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - _, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, bytes.NewBuffer(make([]byte, 1)), nil, nil, context.Background()) - s.Require().NoError(err) - log.Info("Before waiting") - select { - case <-time.After(2 * (time.Duration(timeLimit) * time.Second)): - s.FailNow("The execution should receive a SIGQUIT after the timeout") - case <-quit: - log.Info("Received quit") - } -} - -func (s *ExecuteInteractivelyTestSuite) TestDestroysRunnerAfterTimeoutAndSignal() { - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - <-s.TestCtx.Done() - }) - runnerDestroyed := false - s.runner.onDestroy = func(_ Runner) error { - runnerDestroyed = true - return nil - } - timeLimit := 1 - executionRequest := &dto.ExecutionRequest{TimeLimit: timeLimit} - s.runner.cancel = func() {} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - - _, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, bytes.NewBuffer(make([]byte, 1)), nil, nil, context.Background()) - s.Require().NoError(err) - - <-time.After(executionTimeoutGracePeriod + time.Duration(timeLimit)*time.Second) - // Even if we expect the timeout to be exceeded now, Poseidon sometimes take a couple of hundred ms longer. - <-time.After(2 * tests.ShortTimeout) - s.manager.AssertNotCalled(s.T(), "Return", s.runner) - s.apiMock.AssertCalled(s.T(), "DeleteJob", s.runner.ID()) - s.True(runnerDestroyed) -} - -func (s *ExecuteInteractivelyTestSuite) TestResetTimerGetsCalled() { - executionRequest := &dto.ExecutionRequest{} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - _, _, err := s.runner.ExecuteInteractively(defaultExecutionID, nil, nil, nil, context.Background()) - s.Require().NoError(err) - s.timer.AssertCalled(s.T(), "ResetTimeout") -} - -func (s *ExecuteInteractivelyTestSuite) TestExitHasTimeoutErrorIfRunnerTimesOut() { - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - <-s.TestCtx.Done() - }).Return(0, nil) - s.mockedTimeoutPassedCall.Return(true) - executionRequest := &dto.ExecutionRequest{} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - - exitChannel, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, &nullio.ReadWriter{}, nil, nil, context.Background()) - s.Require().NoError(err) - err = s.runner.Destroy(ErrorRunnerInactivityTimeout) - s.Require().NoError(err) - exit := <-exitChannel - s.ErrorIs(exit.Err, ErrorRunnerInactivityTimeout) -} - -func (s *ExecuteInteractivelyTestSuite) TestDestroyReasonIsPassedToExecution() { - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - <-s.TestCtx.Done() - }).Return(0, nil) - s.mockedTimeoutPassedCall.Return(true) - executionRequest := &dto.ExecutionRequest{} - s.runner.StoreExecution(defaultExecutionID, executionRequest) - - exitChannel, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, &nullio.ReadWriter{}, nil, nil, context.Background()) - s.Require().NoError(err) - err = s.runner.Destroy(ErrOOMKilled) - s.Require().NoError(err) - exit := <-exitChannel - s.ErrorIs(exit.Err, ErrOOMKilled) -} - -func (s *ExecuteInteractivelyTestSuite) TestSuspectedOOMKilledExecutionWaitsForVerification() { - s.mockedExecuteCommandCall.Return(128, nil) - executionRequest := &dto.ExecutionRequest{} - s.Run("Actually OOM Killed", func() { - s.runner.StoreExecution(defaultExecutionID, executionRequest) - exitChannel, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, &nullio.ReadWriter{}, nil, nil, context.Background()) - s.Require().NoError(err) - - select { - case <-exitChannel: - s.FailNow("For exit code 128 Poseidon should wait a while to verify the OOM Kill assumption.") - case <-time.After(tests.ShortTimeout): - // All good. Poseidon waited. - } - - err = s.runner.Destroy(ErrOOMKilled) - s.Require().NoError(err) - exit := <-exitChannel - s.ErrorIs(exit.Err, ErrOOMKilled) - }) - - ctx, cancel := context.WithCancel(context.Background()) - s.runner.ctx = ctx - s.runner.cancel = cancel - s.Run("Not OOM Killed", func() { - s.runner.StoreExecution(defaultExecutionID, executionRequest) - exitChannel, _, err := s.runner.ExecuteInteractively( - defaultExecutionID, &nullio.ReadWriter{}, nil, nil, context.Background()) - s.Require().NoError(err) - - select { - case <-time.After(tests.ShortTimeout + time.Second): - s.FailNow("Poseidon should not wait too long for verifying the OOM Kill assumption.") - case exit := <-exitChannel: - s.Equal(uint8(128), exit.Code) - s.Nil(exit.Err) - } - }) -} - -func TestUpdateFileSystemTestSuite(t *testing.T) { - suite.Run(t, new(UpdateFileSystemTestSuite)) -} - -type UpdateFileSystemTestSuite struct { - tests.MemoryLeakTestSuite - runner *NomadJob - timer *InactivityTimerMock - apiMock *nomad.ExecutorAPIMock - mockedExecuteCommandCall *mock.Call - command string - stdin *bytes.Buffer -} - -func (s *UpdateFileSystemTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.apiMock = &nomad.ExecutorAPIMock{} - s.timer = &InactivityTimerMock{} - s.timer.On("ResetTimeout").Return() - s.timer.On("TimeoutPassed").Return(false) - s.runner = &NomadJob{ - executions: storage.NewLocalStorage[*dto.ExecutionRequest](), - InactivityTimer: s.timer, - id: tests.DefaultRunnerID, - api: s.apiMock, - } - s.mockedExecuteCommandCall = s.apiMock.On("ExecuteCommand", tests.DefaultRunnerID, mock.Anything, - mock.Anything, false, mock.AnythingOfType("bool"), mock.Anything, mock.Anything, mock.Anything). - Run(func(args mock.Arguments) { - var ok bool - s.command, ok = args.Get(2).(string) - s.Require().True(ok) - s.stdin, ok = args.Get(5).(*bytes.Buffer) - s.Require().True(ok) - }).Return(0, nil) -} - -func (s *UpdateFileSystemTestSuite) TestUpdateFileSystemForRunnerPerformsTarExtractionWithAbsoluteNamesOnRunner() { - // note: this method tests an implementation detail of the method UpdateFileSystemOfRunner method - // if the implementation changes, delete this test and write a new one - copyRequest := &dto.UpdateFileSystemRequest{} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.NoError(err) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, - false, mock.AnythingOfType("bool"), mock.Anything, mock.Anything, mock.Anything) - s.Regexp("tar --extract --absolute-names", s.command) -} - -func (s *UpdateFileSystemTestSuite) TestUpdateFileSystemForRunnerReturnsErrorIfExitCodeIsNotZero() { - s.mockedExecuteCommandCall.Return(1, nil) - copyRequest := &dto.UpdateFileSystemRequest{} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.ErrorIs(err, ErrorFileCopyFailed) -} - -func (s *UpdateFileSystemTestSuite) TestUpdateFileSystemForRunnerReturnsErrorIfApiCallDid() { - s.mockedExecuteCommandCall.Return(0, tests.ErrDefault) - copyRequest := &dto.UpdateFileSystemRequest{} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.ErrorIs(err, nomad.ErrorExecutorCommunicationFailed) -} - -func (s *UpdateFileSystemTestSuite) TestFilesToCopyAreIncludedInTarArchive() { - copyRequest := &dto.UpdateFileSystemRequest{Copy: []dto.File{ - {Path: tests.DefaultFileName, Content: []byte(tests.DefaultFileContent)}}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.NoError(err) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true, - mock.Anything, mock.Anything, mock.Anything) - - tarFiles := s.readFilesFromTarArchive(s.stdin) - s.Len(tarFiles, 1) - tarFile := tarFiles[0] - s.True(strings.HasSuffix(tarFile.Name, tests.DefaultFileName)) - s.Equal(byte(tar.TypeReg), tarFile.TypeFlag) - s.Equal(tests.DefaultFileContent, tarFile.Content) -} - -func (s *UpdateFileSystemTestSuite) TestTarFilesContainCorrectPathForRelativeFilePath() { - copyRequest := &dto.UpdateFileSystemRequest{Copy: []dto.File{ - {Path: tests.DefaultFileName, Content: []byte(tests.DefaultFileContent)}}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.Require().NoError(err) - - tarFiles := s.readFilesFromTarArchive(s.stdin) - s.Len(tarFiles, 1) - // tar is extracted in the active workdir of the container, file will be put relative to that - s.Equal(tests.DefaultFileName, tarFiles[0].Name) -} - -func (s *UpdateFileSystemTestSuite) TestFilesWithAbsolutePathArePutInAbsoluteLocation() { - copyRequest := &dto.UpdateFileSystemRequest{Copy: []dto.File{ - {Path: tests.FileNameWithAbsolutePath, Content: []byte(tests.DefaultFileContent)}}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.Require().NoError(err) - - tarFiles := s.readFilesFromTarArchive(s.stdin) - s.Len(tarFiles, 1) - s.Equal(tarFiles[0].Name, tests.FileNameWithAbsolutePath) -} - -func (s *UpdateFileSystemTestSuite) TestDirectoriesAreMarkedAsDirectoryInTar() { - copyRequest := &dto.UpdateFileSystemRequest{Copy: []dto.File{{Path: tests.DefaultDirectoryName, Content: []byte{}}}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.Require().NoError(err) - - tarFiles := s.readFilesFromTarArchive(s.stdin) - s.Len(tarFiles, 1) - tarFile := tarFiles[0] - s.True(strings.HasSuffix(tarFile.Name+"/", tests.DefaultDirectoryName)) - s.Equal(byte(tar.TypeDir), tarFile.TypeFlag) - s.Equal("", tarFile.Content) -} - -func (s *UpdateFileSystemTestSuite) TestFilesToRemoveGetRemoved() { - copyRequest := &dto.UpdateFileSystemRequest{Delete: []dto.FilePath{tests.DefaultFileName}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.NoError(err) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true, - mock.Anything, mock.Anything, mock.Anything) - s.Regexp(fmt.Sprintf("rm[^;]+%s' *;", regexp.QuoteMeta(tests.DefaultFileName)), s.command) -} - -func (s *UpdateFileSystemTestSuite) TestFilesToRemoveGetEscaped() { - copyRequest := &dto.UpdateFileSystemRequest{Delete: []dto.FilePath{"/some/potentially/harmful'filename"}} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.NoError(err) - s.apiMock.AssertCalled(s.T(), "ExecuteCommand", mock.Anything, mock.Anything, mock.Anything, false, true, - mock.Anything, mock.Anything, mock.Anything) - s.Contains(s.command, "'/some/potentially/harmful'\\\\''filename'") -} - -func (s *UpdateFileSystemTestSuite) TestResetTimerGetsCalled() { - copyRequest := &dto.UpdateFileSystemRequest{} - err := s.runner.UpdateFileSystem(copyRequest, context.Background()) - s.NoError(err) - s.timer.AssertCalled(s.T(), "ResetTimeout") -} - -type TarFile struct { - Name string - Content string - TypeFlag byte -} - -func (s *UpdateFileSystemTestSuite) readFilesFromTarArchive(tarArchive io.Reader) (files []TarFile) { - reader := tar.NewReader(tarArchive) - for { - hdr, err := reader.Next() - if err != nil { - break - } - bf, err := io.ReadAll(reader) - s.Require().NoError(err) - files = append(files, TarFile{Name: hdr.Name, Content: string(bf), TypeFlag: hdr.Typeflag}) - } - return files -} - -func (s *UpdateFileSystemTestSuite) TestGetFileContentReturnsErrorIfExitCodeIsNotZero() { - s.mockedExecuteCommandCall.RunFn = nil - s.mockedExecuteCommandCall.Return(1, nil) - err := s.runner.GetFileContent("", logging.NewLoggingResponseWriter(nil), false, context.Background()) - s.ErrorIs(err, ErrFileNotFound) -} - -func (s *UpdateFileSystemTestSuite) TestFileCopyIsCanceledOnRunnerDestroy() { - s.mockedExecuteCommandCall.Run(func(args mock.Arguments) { - ctx, ok := args.Get(1).(context.Context) - s.Require().True(ok) - - select { - case <-ctx.Done(): - s.Fail("mergeContext is done before any of its parents") - return - case <-time.After(tests.ShortTimeout): - } - - select { - case <-ctx.Done(): - case <-time.After(3 * tests.ShortTimeout): - s.Fail("mergeContext is not done after the earliest of its parents") - return - } - }) - ctx, cancel := context.WithCancel(context.Background()) - s.runner.ctx = ctx - s.runner.cancel = cancel - - <-time.After(2 * tests.ShortTimeout) - s.runner.cancel() -} diff --git a/internal/runner/runner_mock.go b/internal/runner/runner_mock.go deleted file mode 100644 index 03b5e36..0000000 --- a/internal/runner/runner_mock.go +++ /dev/null @@ -1,218 +0,0 @@ -// Code generated by mockery v2.30.16. DO NOT EDIT. - -package runner - -import ( - context "context" - http "net/http" - - dto "github.com/openHPI/poseidon/pkg/dto" - - io "io" - - mock "github.com/stretchr/testify/mock" - - time "time" -) - -// RunnerMock is an autogenerated mock type for the Runner type -type RunnerMock struct { - mock.Mock -} - -// Destroy provides a mock function with given fields: reason -func (_m *RunnerMock) Destroy(reason DestroyReason) error { - ret := _m.Called(reason) - - var r0 error - if rf, ok := ret.Get(0).(func(DestroyReason) error); ok { - r0 = rf(reason) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Environment provides a mock function with given fields: -func (_m *RunnerMock) Environment() dto.EnvironmentID { - ret := _m.Called() - - var r0 dto.EnvironmentID - if rf, ok := ret.Get(0).(func() dto.EnvironmentID); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(dto.EnvironmentID) - } - - return r0 -} - -// ExecuteInteractively provides a mock function with given fields: id, stdin, stdout, stderr, ctx -func (_m *RunnerMock) ExecuteInteractively(id string, stdin io.ReadWriter, stdout io.Writer, stderr io.Writer, ctx context.Context) (<-chan ExitInfo, context.CancelFunc, error) { - ret := _m.Called(id, stdin, stdout, stderr, ctx) - - var r0 <-chan ExitInfo - var r1 context.CancelFunc - var r2 error - if rf, ok := ret.Get(0).(func(string, io.ReadWriter, io.Writer, io.Writer, context.Context) (<-chan ExitInfo, context.CancelFunc, error)); ok { - return rf(id, stdin, stdout, stderr, ctx) - } - if rf, ok := ret.Get(0).(func(string, io.ReadWriter, io.Writer, io.Writer, context.Context) <-chan ExitInfo); ok { - r0 = rf(id, stdin, stdout, stderr, ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan ExitInfo) - } - } - - if rf, ok := ret.Get(1).(func(string, io.ReadWriter, io.Writer, io.Writer, context.Context) context.CancelFunc); ok { - r1 = rf(id, stdin, stdout, stderr, ctx) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(context.CancelFunc) - } - } - - if rf, ok := ret.Get(2).(func(string, io.ReadWriter, io.Writer, io.Writer, context.Context) error); ok { - r2 = rf(id, stdin, stdout, stderr, ctx) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// ExecutionExists provides a mock function with given fields: id -func (_m *RunnerMock) ExecutionExists(id string) bool { - ret := _m.Called(id) - - var r0 bool - if rf, ok := ret.Get(0).(func(string) bool); ok { - r0 = rf(id) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// GetFileContent provides a mock function with given fields: path, content, privilegedExecution, ctx -func (_m *RunnerMock) GetFileContent(path string, content http.ResponseWriter, privilegedExecution bool, ctx context.Context) error { - ret := _m.Called(path, content, privilegedExecution, ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(string, http.ResponseWriter, bool, context.Context) error); ok { - r0 = rf(path, content, privilegedExecution, ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ID provides a mock function with given fields: -func (_m *RunnerMock) ID() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// ListFileSystem provides a mock function with given fields: path, recursive, result, privilegedExecution, ctx -func (_m *RunnerMock) ListFileSystem(path string, recursive bool, result io.Writer, privilegedExecution bool, ctx context.Context) error { - ret := _m.Called(path, recursive, result, privilegedExecution, ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(string, bool, io.Writer, bool, context.Context) error); ok { - r0 = rf(path, recursive, result, privilegedExecution, ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MappedPorts provides a mock function with given fields: -func (_m *RunnerMock) MappedPorts() []*dto.MappedPort { - ret := _m.Called() - - var r0 []*dto.MappedPort - if rf, ok := ret.Get(0).(func() []*dto.MappedPort); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dto.MappedPort) - } - } - - return r0 -} - -// ResetTimeout provides a mock function with given fields: -func (_m *RunnerMock) ResetTimeout() { - _m.Called() -} - -// SetupTimeout provides a mock function with given fields: duration -func (_m *RunnerMock) SetupTimeout(duration time.Duration) { - _m.Called(duration) -} - -// StopTimeout provides a mock function with given fields: -func (_m *RunnerMock) StopTimeout() { - _m.Called() -} - -// StoreExecution provides a mock function with given fields: id, executionRequest -func (_m *RunnerMock) StoreExecution(id string, executionRequest *dto.ExecutionRequest) { - _m.Called(id, executionRequest) -} - -// TimeoutPassed provides a mock function with given fields: -func (_m *RunnerMock) TimeoutPassed() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// UpdateFileSystem provides a mock function with given fields: request, ctx -func (_m *RunnerMock) UpdateFileSystem(request *dto.UpdateFileSystemRequest, ctx context.Context) error { - ret := _m.Called(request, ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(*dto.UpdateFileSystemRequest, context.Context) error); ok { - r0 = rf(request, ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// NewRunnerMock creates a new instance of RunnerMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewRunnerMock(t interface { - mock.TestingT - Cleanup(func()) -}) *RunnerMock { - mock := &RunnerMock{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/pkg/logging/logging_test.go b/pkg/logging/logging_test.go deleted file mode 100644 index 589de27..0000000 --- a/pkg/logging/logging_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package logging - -import ( - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/suite" - "net/http" - "net/http/httptest" - "testing" -) - -func mockHTTPStatusHandler(status int) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(status) - }) -} - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestHTTPMiddlewareDebugsWhenStatusOK() { - var hook *test.Hook - log, hook = test.NewNullLogger() - InitializeLogging(logrus.DebugLevel.String(), dto.FormatterText) - - request, err := http.NewRequest(http.MethodGet, "/", http.NoBody) - if err != nil { - s.Fail(err.Error()) - } - recorder := httptest.NewRecorder() - HTTPLoggingMiddleware(mockHTTPStatusHandler(200)).ServeHTTP(recorder, request) - - s.Equal(1, len(hook.Entries)) - s.Equal(logrus.DebugLevel, hook.LastEntry().Level) -} diff --git a/pkg/nullio/content_length_test.go b/pkg/nullio/content_length_test.go deleted file mode 100644 index 4d82474..0000000 --- a/pkg/nullio/content_length_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package nullio - -import ( - "bytes" - "net/http" -) - -type responseWriterStub struct { - bytes.Buffer - header http.Header -} - -func (r *responseWriterStub) Header() http.Header { - return r.header -} -func (r *responseWriterStub) WriteHeader(_ int) { -} - -func (s *MainTestSuite) TestContentLengthWriter_Write() { - header := http.Header(make(map[string][]string)) - buf := &responseWriterStub{header: header} - writer := &ContentLengthWriter{Target: buf} - part1 := []byte("-rw-rw-r-- 1 kali ka") - contentLength := "42" - part2 := []byte("li " + contentLength + " 1660763446 flag\nFL") - part3 := []byte("AG") - - count, err := writer.Write(part1) - s.Require().NoError(err) - s.Equal(len(part1), count) - s.Empty(buf.String()) - s.Equal("", header.Get("Content-Length")) - - count, err = writer.Write(part2) - s.Require().NoError(err) - s.Equal(len(part2), count) - s.Equal("FL", buf.String()) - s.Equal(contentLength, header.Get("Content-Length")) - - count, err = writer.Write(part3) - s.Require().NoError(err) - s.Equal(len(part3), count) - s.Equal("FLAG", buf.String()) - s.Equal(contentLength, header.Get("Content-Length")) -} diff --git a/pkg/nullio/ls2json_test.go b/pkg/nullio/ls2json_test.go deleted file mode 100644 index c578661..0000000 --- a/pkg/nullio/ls2json_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package nullio - -import ( - "bytes" - "context" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "testing" -) - -func TestLs2JsonTestSuite(t *testing.T) { - suite.Run(t, new(Ls2JsonTestSuite)) -} - -type Ls2JsonTestSuite struct { - tests.MemoryLeakTestSuite - buf *bytes.Buffer - writer *Ls2JsonWriter -} - -func (s *Ls2JsonTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.buf = &bytes.Buffer{} - s.writer = &Ls2JsonWriter{Target: s.buf, Ctx: context.Background()} -} - -func (s *Ls2JsonTestSuite) TestLs2JsonWriter_WriteCreationAndClose() { - count, err := s.writer.Write([]byte("")) - s.Zero(count) - s.NoError(err) - - s.Equal("{\"files\": [", s.buf.String()) - - s.writer.Close() - s.Equal("{\"files\": []}", s.buf.String()) -} - -func (s *Ls2JsonTestSuite) TestLs2JsonWriter_WriteFile() { - input := "total 0\n-rw-rw-r-- 1 kali kali 0 1660763446 flag\n" - count, err := s.writer.Write([]byte(input)) - s.Equal(len(input), count) - s.NoError(err) - s.writer.Close() - - s.Equal("{\"files\": [{\"name\":\"flag\",\"entryType\":\"-\",\"size\":0,\"modificationTime\":1660763446"+ - ",\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"}]}", - s.buf.String()) -} - -func (s *Ls2JsonTestSuite) TestLs2JsonWriter_WriteRecursive() { - input := ".:\ntotal 4\ndrwxrwxr-x 2 kali kali 4096 1660764411 dir\n" + - "-rw-rw-r-- 1 kali kali 0 1660763446 flag\n" + - "\n./dir:\ntotal 4\n-rw-rw-r-- 1 kali kali 3 1660764366 another.txt\n" - count, err := s.writer.Write([]byte(input)) - s.Equal(len(input), count) - s.NoError(err) - s.writer.Close() - - s.Equal("{\"files\": ["+ - "{\"name\":\"./dir\",\"entryType\":\"d\",\"size\":4096,\"modificationTime\":1660764411,"+ - "\"permissions\":\"rwxrwxr-x\",\"owner\":\"kali\",\"group\":\"kali\"},"+ - "{\"name\":\"./flag\",\"entryType\":\"-\",\"size\":0,\"modificationTime\":1660763446,"+ - "\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"},"+ - "{\"name\":\"./dir/another.txt\",\"entryType\":\"-\",\"size\":3,\"modificationTime\":1660764366,"+ - "\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"}"+ - "]}", - s.buf.String()) -} - -func (s *Ls2JsonTestSuite) TestLs2JsonWriter_WriteRemaining() { - input1 := "total 4\n-rw-rw-r-- 1 kali kali 3 1660764366 an.txt\n-rw-rw-r-- 1 kal" - _, err := s.writer.Write([]byte(input1)) - s.NoError(err) - s.Equal("{\"files\": [{\"name\":\"an.txt\",\"entryType\":\"-\",\"size\":3,\"modificationTime\":1660764366,"+ - "\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"}", s.buf.String()) - - input2 := "i kali 0 1660763446 flag\n" - _, err = s.writer.Write([]byte(input2)) - s.NoError(err) - s.writer.Close() - s.Equal("{\"files\": [{\"name\":\"an.txt\",\"entryType\":\"-\",\"size\":3,\"modificationTime\":1660764366,"+ - "\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"},"+ - "{\"name\":\"flag\",\"entryType\":\"-\",\"size\":0,\"modificationTime\":1660763446,"+ - "\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"}]}", s.buf.String()) -} - -func (s *Ls2JsonTestSuite) TestLs2JsonWriter_WriteLink() { - input1 := "total 4\nlrw-rw-r-- 1 kali kali 3 1660764366 another.txt -> /bin/bash\n" - _, err := s.writer.Write([]byte(input1)) - s.NoError(err) - s.writer.Close() - s.Equal("{\"files\": [{\"name\":\"another.txt\",\"entryType\":\"l\",\"linkTarget\":\"/bin/bash\",\"size\":3,"+ - "\"modificationTime\":1660764366,\"permissions\":\"rw-rw-r--\",\"owner\":\"kali\",\"group\":\"kali\"}]}", - s.buf.String()) -} diff --git a/pkg/nullio/nullio_test.go b/pkg/nullio/nullio_test.go deleted file mode 100644 index 8d7460e..0000000 --- a/pkg/nullio/nullio_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package nullio - -import ( - "context" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "io" - "testing" - "time" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestReader_Read() { - read := func(reader io.Reader, ret chan<- bool) { - p := make([]byte, 0, 5) - _, err := reader.Read(p) - s.ErrorIs(io.EOF, err) - close(ret) - } - - s.Run("WithContext_DoesNotReturnImmediately", func() { - readingContext, cancel := context.WithCancel(context.Background()) - defer cancel() - - readerReturned := make(chan bool) - go read(&Reader{readingContext}, readerReturned) - - select { - case <-readerReturned: - s.Fail("The reader returned before the timeout was reached") - case <-time.After(tests.ShortTimeout): - } - }) - - s.Run("WithoutContext_DoesReturnImmediately", func() { - readerReturned := make(chan bool) - go read(&Reader{}, readerReturned) - - select { - case <-readerReturned: - case <-time.After(tests.ShortTimeout): - s.Fail("The reader returned before the timeout was reached") - } - }) -} - -func (s *MainTestSuite) TestReadWriterWritesEverything() { - readWriter := &ReadWriter{} - p := []byte{1, 2, 3} - n, err := readWriter.Write(p) - s.NoError(err) - s.Equal(len(p), n) -} diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go deleted file mode 100644 index 06843be..0000000 --- a/pkg/storage/storage_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package storage - -import ( - "context" - "github.com/influxdata/influxdb-client-go/v2/api/write" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "testing" - "time" -) - -func TestRunnerPoolTestSuite(t *testing.T) { - suite.Run(t, new(ObjectPoolTestSuite)) -} - -type ObjectPoolTestSuite struct { - tests.MemoryLeakTestSuite - objectStorage *localStorage[any] - object int -} - -func (s *ObjectPoolTestSuite) SetupTest() { - s.MemoryLeakTestSuite.SetupTest() - s.objectStorage = NewLocalStorage[any]() - s.object = 42 -} - -func (s *ObjectPoolTestSuite) TestAddedObjectCanBeRetrieved() { - s.objectStorage.Add("my_id", s.object) - retrievedRunner, ok := s.objectStorage.Get("my_id") - s.True(ok, "A saved object should be retrievable") - s.Equal(s.object, retrievedRunner) -} - -func (s *ObjectPoolTestSuite) TestLocalStorage_List() { - s.objectStorage.Add("my_id", s.object) - s.objectStorage.Add("my_id 2", 21) - retrievedRunners := s.objectStorage.List() - s.Len(retrievedRunners, 2) - s.Contains(retrievedRunners, s.object) - s.Contains(retrievedRunners, 21) -} - -func (s *ObjectPoolTestSuite) TestObjectWithSameIdOverwritesOldOne() { - otherObject := 21 - // assure object is actually different - s.NotEqual(s.object, otherObject) - - s.objectStorage.Add("my_id", s.object) - s.objectStorage.Add("my_id", otherObject) - retrievedObject, _ := s.objectStorage.Get("my_id") - s.NotEqual(s.object, retrievedObject) - s.Equal(otherObject, retrievedObject) -} - -func (s *ObjectPoolTestSuite) TestDeletedObjectsAreNotAccessible() { - s.objectStorage.Add("my_id", s.object) - s.objectStorage.Delete("my_id") - retrievedObject, ok := s.objectStorage.Get("my_id") - s.Nil(retrievedObject) - s.False(ok, "A deleted object should not be accessible") -} - -func (s *ObjectPoolTestSuite) TestSampleReturnsObjectWhenOneIsAvailable() { - s.objectStorage.Add("my_id", s.object) - sampledObject, ok := s.objectStorage.Sample() - s.NotNil(sampledObject) - s.True(ok) -} - -func (s *ObjectPoolTestSuite) TestSampleReturnsFalseWhenNoneIsAvailable() { - sampledObject, ok := s.objectStorage.Sample() - s.Nil(sampledObject) - s.False(ok) -} - -func (s *ObjectPoolTestSuite) TestSampleRemovesObjectFromPool() { - s.objectStorage.Add("my_id", s.object) - _, _ = s.objectStorage.Sample() - _, ok := s.objectStorage.Get("my_id") - s.False(ok) -} - -func (s *ObjectPoolTestSuite) TestLenOfEmptyPoolIsZero() { - s.Equal(uint(0), s.objectStorage.Length()) -} - -func (s *ObjectPoolTestSuite) TestLenChangesOnStoreContentChange() { - s.Run("len increases when object is added", func() { - s.objectStorage.Add("my_id_1", s.object) - s.Equal(uint(1), s.objectStorage.Length()) - }) - - s.Run("len does not increase when object with same id is added", func() { - s.objectStorage.Add("my_id_1", s.object) - s.Equal(uint(1), s.objectStorage.Length()) - }) - - s.Run("len increases again when different object is added", func() { - anotherObject := 21 - s.objectStorage.Add("my_id_2", anotherObject) - s.Equal(uint(2), s.objectStorage.Length()) - }) - - s.Run("len decreases when object is deleted", func() { - s.objectStorage.Delete("my_id_1") - s.Equal(uint(1), s.objectStorage.Length()) - }) - - s.Run("len decreases when object is sampled", func() { - _, _ = s.objectStorage.Sample() - s.Equal(uint(0), s.objectStorage.Length()) - }) -} - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestNewMonitoredLocalStorage_Callback() { - callbackCalls := 0 - callbackAdditions := 0 - callbackDeletions := 0 - os := NewMonitoredLocalStorage[string]("testMeasurement", func(p *write.Point, o string, eventType EventType) { - callbackCalls++ - if eventType == Deletion { - callbackDeletions++ - } else if eventType == Creation { - callbackAdditions++ - } - }, 0, context.Background()) - - assertCallbackCounts := func(test func(), totalCalls, additions, deletions int) { - beforeTotal := callbackCalls - beforeAdditions := callbackAdditions - beforeDeletions := callbackDeletions - test() - s.Equal(beforeTotal+totalCalls, callbackCalls) - s.Equal(beforeAdditions+additions, callbackAdditions) - s.Equal(beforeDeletions+deletions, callbackDeletions) - } - - s.Run("Add", func() { - assertCallbackCounts(func() { - os.Add("id 1", "object 1") - }, 1, 1, 0) - }) - - s.Run("Delete", func() { - assertCallbackCounts(func() { - os.Delete("id 1") - }, 1, 0, 1) - }) - - s.Run("List", func() { - assertCallbackCounts(func() { - os.List() - }, 0, 0, 0) - }) - - s.Run("Pop", func() { - os.Add("id 1", "object 1") - - assertCallbackCounts(func() { - o, ok := os.Pop("id 1") - s.True(ok) - s.Equal("object 1", o) - }, 1, 0, 1) - }) - - s.Run("Purge", func() { - os.Add("id 1", "object 1") - os.Add("id 2", "object 2") - - assertCallbackCounts(func() { - os.Purge() - }, 2, 0, 2) - }) -} -func (s *MainTestSuite) TestNewMonitoredLocalStorage_Periodically() { - callbackCalls := 0 - NewMonitoredLocalStorage[string]("testMeasurement", func(p *write.Point, o string, eventType EventType) { - callbackCalls++ - s.Equal(Periodically, eventType) - }, 2*tests.ShortTimeout, s.TestCtx) - - <-time.After(tests.ShortTimeout) - s.Equal(0, callbackCalls) - <-time.After(2 * tests.ShortTimeout) - s.Equal(1, callbackCalls) - <-time.After(2 * tests.ShortTimeout) - s.Equal(2, callbackCalls) -} diff --git a/pkg/util/merge_context_test.go b/pkg/util/merge_context_test.go deleted file mode 100644 index e964386..0000000 --- a/pkg/util/merge_context_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package util - -import ( - "context" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/stretchr/testify/suite" - "testing" - "time" -) - -type MainTestSuite struct { - tests.MemoryLeakTestSuite -} - -func TestMainTestSuite(t *testing.T) { - suite.Run(t, new(MainTestSuite)) -} - -func (s *MainTestSuite) TestMergeContext_Deadline() { - ctxWithoutDeadline := context.Background() - earlyDeadline := time.Now().Add(time.Second) - ctxWithEarlyDeadline, cancel := context.WithDeadline(context.Background(), earlyDeadline) - defer cancel() - ctxWithLateDeadline, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour)) - defer cancel() - - ctx := NewMergeContext([]context.Context{ctxWithoutDeadline, ctxWithEarlyDeadline, ctxWithLateDeadline}) - deadline, ok := ctx.Deadline() - - s.True(ok) - s.Equal(earlyDeadline, deadline, "The ealiest deadline is returned") -} - -func (s *MainTestSuite) TestMergeContext_Done() { - ctxWithoutDeadline := context.Background() - ctxWithEarlyDeadline, cancel := context.WithTimeout(context.Background(), 2*tests.ShortTimeout) - defer cancel() - ctxWithLateDeadline, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - ctx := NewMergeContext([]context.Context{ctxWithoutDeadline, ctxWithEarlyDeadline, ctxWithLateDeadline}) - - select { - case <-ctx.Done(): - s.Fail("mergeContext is done before any of its parents") - return - case <-time.After(tests.ShortTimeout): - } - - select { - case <-ctx.Done(): - case <-time.After(3 * tests.ShortTimeout): - s.Fail("mergeContext is not done after the earliest of its parents") - return - } -} - -func (s *MainTestSuite) TestMergeContext_Err() { - ctxWithoutDeadline := context.Background() - ctxCancelled, cancel := context.WithCancel(context.Background()) - ctx := NewMergeContext([]context.Context{ctxWithoutDeadline, ctxCancelled}) - - s.NoError(ctx.Err()) - cancel() - s.Error(ctx.Err()) -} - -func (s *MainTestSuite) TestMergeContext_Value() { - ctxWithAValue := context.WithValue(context.Background(), dto.ContextKey("keyA"), "valueA") - ctxWithAnotherValue := context.WithValue(context.Background(), dto.ContextKey("keyB"), "valueB") - ctx := NewMergeContext([]context.Context{ctxWithAValue, ctxWithAnotherValue}) - - s.Equal("valueA", ctx.Value(dto.ContextKey("keyA"))) - s.Equal("valueB", ctx.Value(dto.ContextKey("keyB"))) - s.Nil(ctx.Value("keyC")) -} diff --git a/tests/constants.go b/tests/constants.go deleted file mode 100644 index 2487762..0000000 --- a/tests/constants.go +++ /dev/null @@ -1,41 +0,0 @@ -package tests - -import ( - "errors" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/pkg/dto" - "time" -) - -const ( - NonExistingIntegerID = 9999 - NonExistingStringID = "n0n-3x1st1ng-1d" - DefaultFileName = "test.txt" - DefaultFileContent = "Hello, Codemoon!" - DefaultDirectoryName = "test/" - FileNameWithAbsolutePath = "/test.txt" - DefaultEnvironmentIDAsInteger = 0 - DefaultEnvironmentIDAsString = "0" - AnotherEnvironmentIDAsInteger = 42 - AnotherEnvironmentIDAsString = "42" - DefaultUUID = "MY-DEFAULT-RANDOM-UUID" - AnotherUUID = "another-uuid-43" - DefaultTemplateJobID = "template-" + DefaultEnvironmentIDAsString - DefaultRunnerID = DefaultEnvironmentIDAsString + "-" + DefaultUUID - AnotherRunnerID = AnotherEnvironmentIDAsString + "-" + AnotherUUID - DefaultExecutionID = "s0m3-3x3cu710n-1d" - DefaultMockID = "m0ck-1d" - ShortTimeout = 100 * time.Millisecond - DefaultTestTimeout = 10 * time.Minute - - defaultPort = 42 - anotherPort = 1337 -) - -var ( - ErrDefault = errors.New("an error occurred") - ErrCleanupDestroyReason = errors.New("destruction required for cleanup") - - DefaultPortMappings = []nomadApi.PortMapping{{To: defaultPort, Value: anotherPort, Label: "lit", HostIP: "127.0.0.1"}} - DefaultMappedPorts = []*dto.MappedPort{{ExposedPort: defaultPort, HostAddress: "127.0.0.1:1337"}} -) diff --git a/tests/e2e/e2e_helpers.go b/tests/e2e/e2e_helpers.go deleted file mode 100644 index fd3d68f..0000000 --- a/tests/e2e/e2e_helpers.go +++ /dev/null @@ -1,105 +0,0 @@ -package e2e - -import ( - "encoding/json" - "fmt" - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/logging" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "net/http" - "time" -) - -var log = logging.GetLogger("e2e-helpers") - -func CreateDefaultEnvironment(prewarmingPoolSize uint, image string) dto.ExecutionEnvironmentRequest { - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) - const smallCPULimit uint = 20 - const smallMemoryLimit uint = 100 - - defaultNomadEnvironment := dto.ExecutionEnvironmentRequest{ - PrewarmingPoolSize: prewarmingPoolSize, - CPULimit: smallCPULimit, - MemoryLimit: smallMemoryLimit, - Image: image, - NetworkAccess: false, - ExposedPorts: nil, - } - - resp, err := helpers.HTTPPutJSON(path, defaultNomadEnvironment) - if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { - log.WithError(err).Fatal("Couldn't create default environment for e2e tests") - } - err = resp.Body.Close() - if err != nil { - log.Fatal("Failed closing body") - } - return defaultNomadEnvironment -} - -func WaitForDefaultEnvironment() { - path := helpers.BuildURL(api.BasePath, api.RunnersPath) - body := &dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - InactivityTimeout: 1, - } - var code int - const maxRetries = 60 - for count := 0; count < maxRetries && code != http.StatusOK; count++ { - <-time.After(time.Second) - resp, err := helpers.HTTPPostJSON(path, body) - if err == nil { - code = resp.StatusCode - log.WithField("count", count).WithField("statusCode", code).Info("Waiting for idle runners") - } else { - log.WithField("count", count).WithError(err).Warn("Waiting for idle runners") - } - _ = resp.Body.Close() - } - if code != http.StatusOK { - log.Fatal("Failed to provide a runner") - } -} - -// ProvideRunner creates a runner with the given RunnerRequest via an external request. -// It needs a running Poseidon instance to work. -func ProvideRunner(request *dto.RunnerRequest) (string, error) { - runnerURL := helpers.BuildURL(api.BasePath, api.RunnersPath) - resp, err := helpers.HTTPPostJSON(runnerURL, request) - if err != nil { - return "", fmt.Errorf("cannot post provide runner: %w", err) - } - if resp.StatusCode != http.StatusOK { - //nolint:goerr113 // dynamic error is ok in here, as it is a test - return "", fmt.Errorf("expected response code 200 when getting runner, got %v", resp.StatusCode) - } - runnerResponse := new(dto.RunnerResponse) - err = json.NewDecoder(resp.Body).Decode(runnerResponse) - if err != nil { - return "", fmt.Errorf("cannot decode runner response: %w", err) - } - _ = resp.Body.Close() - return runnerResponse.ID, nil -} - -// ProvideWebSocketURL creates a WebSocket endpoint from the ExecutionRequest via an external api request. -// It requires a running Poseidon instance. -func ProvideWebSocketURL(runnerID string, request *dto.ExecutionRequest) (string, error) { - url := helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.ExecutePath) - resp, err := helpers.HTTPPostJSON(url, request) - if err != nil { - return "", fmt.Errorf("cannot post provide websocket url: %w", err) - } else if resp.StatusCode != http.StatusOK { - return "", dto.ErrMissingData - } - - executionResponse := new(dto.ExecutionResponse) - err = json.NewDecoder(resp.Body).Decode(executionResponse) - if err != nil { - return "", fmt.Errorf("cannot parse execution response: %w", err) - } - _ = resp.Body.Close() - return executionResponse.WebSocketURL, nil -} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go deleted file mode 100644 index 9b36203..0000000 --- a/tests/e2e/e2e_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package e2e - -import ( - "flag" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/suite" - "net/http" - "os" - "testing" - "time" -) - -/* -* # E2E Tests -* -* For the e2e tests a nomad cluster must be connected and poseidon must be running. - */ - -var ( - testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests") - nomadClient *nomadApi.Client - nomadNamespace string - environmentIDs []dto.EnvironmentID - defaultNomadEnvironment dto.ExecutionEnvironmentRequest -) - -type E2ETestSuite struct { - suite.Suite -} - -func (s *E2ETestSuite) SetupTest() { - // Waiting one second before each test allows Nomad to rescale after tests requested runners. - <-time.After(time.Second) -} - -func TestE2ETestSuite(t *testing.T) { - suite.Run(t, new(E2ETestSuite)) -} - -// Overwrite TestMain for custom setup. -func TestMain(m *testing.M) { - log.Info("Test Setup") - if err := config.InitConfig(); err != nil { - log.WithError(err).Fatal("Could not initialize configuration") - } - initNomad() - initAWS() - - // wait for environment to become ready - <-time.After(10 * time.Second) - log.Info("Test Run") - code := m.Run() - - deleteE2EEnvironments() - cleanupJobsForEnvironment(&testing.T{}, tests.DefaultEnvironmentIDAsString) - os.Exit(code) -} - -func initAWS() { - for i, function := range config.Config.AWS.Functions { - log.WithField("function", function[0:3]).Info("Yes, we do have AWS functions.") - id := dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger + i + 1) - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, id.ToString()) - request := dto.ExecutionEnvironmentRequest{Image: function} - resp, err := helpers.HTTPPutJSON(path, request) - if err != nil || resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { - log.WithField("function", function).WithError(err).Fatal("Couldn't create default environment for e2e tests") - } - environmentIDs = append(environmentIDs, id) - err = resp.Body.Close() - if err != nil { - log.Fatal("Failed closing body") - } - } -} - -func initNomad() { - nomadNamespace = config.Config.Nomad.Namespace - var err error - nomadClient, err = nomadApi.NewClient(&nomadApi.Config{ - Address: config.Config.Nomad.URL().String(), - TLSConfig: &nomadApi.TLSConfig{}, - Namespace: nomadNamespace, - }) - if err != nil { - log.WithError(err).Fatal("Could not create Nomad client") - return - } - createDefaultEnvironment() - WaitForDefaultEnvironment() -} - -func createDefaultEnvironment() { - if *testDockerImage == "" { - log.Fatal("You must specify the -dockerImage flag!") - } - defaultNomadEnvironment = CreateDefaultEnvironment(10, *testDockerImage) - environmentIDs = append(environmentIDs, tests.DefaultEnvironmentIDAsInteger) -} - -func deleteE2EEnvironments() { - for _, id := range environmentIDs { - deleteEnvironment(&testing.T{}, id.ToString()) - } -} diff --git a/tests/e2e/environments_test.go b/tests/e2e/environments_test.go deleted file mode 100644 index b7ba237..0000000 --- a/tests/e2e/environments_test.go +++ /dev/null @@ -1,403 +0,0 @@ -package e2e - -import ( - "encoding/json" - "fmt" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "io" - "net/http" - "strings" - "testing" - "time" -) - -var isAWSEnvironment = []bool{false} - -func TestCreateOrUpdateEnvironment(t *testing.T) { - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.AnotherEnvironmentIDAsString) - - t.Run("returns bad request with empty body", func(t *testing.T) { - resp, err := helpers.HTTPPut(path, strings.NewReader("")) - require.Nil(t, err) - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - - _ = resp.Body.Close() - }) - - request := dto.ExecutionEnvironmentRequest{ - PrewarmingPoolSize: 1, - CPULimit: 100, - MemoryLimit: 100, - Image: *testDockerImage, - NetworkAccess: false, - ExposedPorts: nil, - } - - t.Run("creates correct environment in Nomad", func(t *testing.T) { - assertPutReturnsStatusAndZeroContent(t, path, request, http.StatusCreated) - validateJob(t, request) - }) - - t.Run("updates limits in Nomad correctly", func(t *testing.T) { - updateRequest := request - updateRequest.CPULimit = 150 - updateRequest.MemoryLimit = 142 - - assertPutReturnsStatusAndZeroContent(t, path, updateRequest, http.StatusNoContent) - validateJob(t, updateRequest) - }) - - t.Run("adds network correctly", func(t *testing.T) { - updateRequest := request - updateRequest.NetworkAccess = true - updateRequest.ExposedPorts = []uint16{42, 1337} - - assertPutReturnsStatusAndZeroContent(t, path, updateRequest, http.StatusNoContent) - validateJob(t, updateRequest) - }) - - t.Run("removes network correctly", func(t *testing.T) { - require.False(t, request.NetworkAccess) - require.Nil(t, request.ExposedPorts) - assertPutReturnsStatusAndZeroContent(t, path, request, http.StatusNoContent) - validateJob(t, request) - }) - - deleteEnvironment(t, tests.AnotherEnvironmentIDAsString) -} - -func TestListEnvironments(t *testing.T) { - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath) - - t.Run("returns list with all static and the e2e environment", func(t *testing.T) { - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - - assert.Equal(t, http.StatusOK, response.StatusCode) - environmentsArray := assertEnvironmentArrayInResponse(t, response) - assert.Equal(t, len(environmentIDs), len(environmentsArray)) - }) - - t.Run("returns list including the default environment", func(t *testing.T) { - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - - environmentsArray := assertEnvironmentArrayInResponse(t, response) - require.Equal(t, len(environmentIDs), len(environmentsArray)) - foundIDs := parseIDsFromEnvironments(t, environmentsArray) - assert.Contains(t, foundIDs, dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - }) - - for _, useAWS := range isAWSEnvironment { - t.Run(fmt.Sprintf("AWS-%t", useAWS), func(t *testing.T) { - t.Run("Added environments can be retrieved without fetch", func(t *testing.T) { - createEnvironment(t, tests.AnotherEnvironmentIDAsString, useAWS) - - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - - environmentsArray := assertEnvironmentArrayInResponse(t, response) - require.Equal(t, len(environmentIDs)+1, len(environmentsArray)) - foundIDs := parseIDsFromEnvironments(t, environmentsArray) - assert.Contains(t, foundIDs, dto.EnvironmentID(tests.AnotherEnvironmentIDAsInteger)) - }) - deleteEnvironment(t, tests.AnotherEnvironmentIDAsString) - }) - } - - t.Run("Added environments can be retrieved with fetch", func(t *testing.T) { - // Add environment without Poseidon - _, job := helpers.CreateTemplateJob() - jobID := nomad.TemplateJobID(tests.AnotherEnvironmentIDAsInteger) - job.ID = &jobID - job.Name = &jobID - _, _, err := nomadClient.Jobs().Register(job, nil) - require.NoError(t, err) - <-time.After(tests.ShortTimeout) // Nomad needs a bit to create the job - - // List without fetch should not include the added environment - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - environmentsArray := assertEnvironmentArrayInResponse(t, response) - require.Equal(t, len(environmentIDs), len(environmentsArray)) - foundIDs := parseIDsFromEnvironments(t, environmentsArray) - assert.Contains(t, foundIDs, dto.EnvironmentID(tests.DefaultEnvironmentIDAsInteger)) - - // List with fetch should include the added environment - response, err = http.Get(path + "?fetch=true") //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - environmentsArray = assertEnvironmentArrayInResponse(t, response) - require.Equal(t, len(environmentIDs)+1, len(environmentsArray)) - foundIDs = parseIDsFromEnvironments(t, environmentsArray) - assert.Contains(t, foundIDs, dto.EnvironmentID(tests.AnotherEnvironmentIDAsInteger)) - }) - deleteEnvironment(t, tests.AnotherEnvironmentIDAsString) -} - -func TestGetEnvironment(t *testing.T) { - t.Run("returns the default environment", func(t *testing.T) { - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - - environment := getEnvironmentFromResponse(t, response) - assertEnvironment(t, environment, tests.DefaultEnvironmentIDAsInteger) - }) - - for _, useAWS := range isAWSEnvironment { - t.Run(fmt.Sprintf("AWS-%t", useAWS), func(t *testing.T) { - t.Run("Added environments can be retrieved without fetch", func(t *testing.T) { - createEnvironment(t, tests.AnotherEnvironmentIDAsString, useAWS) - - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.AnotherEnvironmentIDAsString) - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - - environment := getEnvironmentFromResponse(t, response) - assertEnvironment(t, environment, tests.AnotherEnvironmentIDAsInteger) - }) - deleteEnvironment(t, tests.AnotherEnvironmentIDAsString) - }) - } - - t.Run("Added environments can be retrieved with fetch", func(t *testing.T) { - // Add environment without Poseidon - _, job := helpers.CreateTemplateJob() - jobID := nomad.TemplateJobID(tests.AnotherEnvironmentIDAsInteger) - job.ID = &jobID - job.Name = &jobID - _, _, err := nomadClient.Jobs().Register(job, nil) - require.NoError(t, err) - <-time.After(tests.ShortTimeout) // Nomad needs a bit to create the job - - // List without fetch should not include the added environment - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.AnotherEnvironmentIDAsString) - response, err := http.Get(path) //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusNotFound, response.StatusCode) - - // List with fetch should include the added environment - response, err = http.Get(path + "?fetch=true") //nolint:gosec // because we build this path right above - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode) - environment := getEnvironmentFromResponse(t, response) - assertEnvironment(t, environment, tests.AnotherEnvironmentIDAsInteger) - }) - deleteEnvironment(t, tests.AnotherEnvironmentIDAsString) -} - -func TestDeleteEnvironment(t *testing.T) { - for _, useAWS := range isAWSEnvironment { - t.Run(fmt.Sprintf("AWS-%t", useAWS), func(t *testing.T) { - t.Run("Removes added environment", func(t *testing.T) { - createEnvironment(t, tests.AnotherEnvironmentIDAsString, useAWS) - - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.AnotherEnvironmentIDAsString) - response, err := helpers.HTTPDelete(path, nil) - assert.NoError(t, err) - assert.Equal(t, http.StatusNoContent, response.StatusCode) - }) - }) - } - - t.Run("Removes Nomad Job", func(t *testing.T) { - createEnvironment(t, tests.AnotherEnvironmentIDAsString, false) - - // Expect created Nomad job - jobID := nomad.TemplateJobID(tests.AnotherEnvironmentIDAsInteger) - job, _, err := nomadClient.Jobs().Info(jobID, nil) - assert.NoError(t, err) - assert.Equal(t, jobID, *job.ID) - - // Delete the job - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.AnotherEnvironmentIDAsString) - response, err := helpers.HTTPDelete(path, nil) - assert.NoError(t, err) - assert.Equal(t, http.StatusNoContent, response.StatusCode) - - // Expect not to find the Nomad job - _, _, err = nomadClient.Jobs().Info(jobID, nil) - assert.Error(t, err) - }) -} - -func parseIDsFromEnvironments(t *testing.T, environments []interface{}) (ids []dto.EnvironmentID) { - t.Helper() - for _, environment := range environments { - id, _ := parseEnvironment(t, environment) - ids = append(ids, id) - } - return ids -} - -func assertEnvironment(t *testing.T, environment interface{}, expectedID dto.EnvironmentID) { - t.Helper() - id, defaultEnvironmentParams := parseEnvironment(t, environment) - - assert.Equal(t, expectedID, id) - expectedKeys := []string{"prewarmingPoolSize", "cpuLimit", "memoryLimit", "image", "networkAccess", "exposedPorts"} - for _, key := range expectedKeys { - _, ok := defaultEnvironmentParams[key] - assert.True(t, ok) - } -} - -func parseEnvironment(t *testing.T, environment interface{}) (id dto.EnvironmentID, params map[string]interface{}) { - t.Helper() - environmentParams, ok := environment.(map[string]interface{}) - require.True(t, ok) - idInterface, ok := environmentParams["id"] - require.True(t, ok) - idFloat, ok := idInterface.(float64) - require.True(t, ok) - return dto.EnvironmentID(int(idFloat)), environmentParams -} - -func assertEnvironmentArrayInResponse(t *testing.T, response *http.Response) []interface{} { - t.Helper() - paramMap := make(map[string]interface{}) - err := json.NewDecoder(response.Body).Decode(¶mMap) - require.NoError(t, err) - environments, ok := paramMap["executionEnvironments"] - assert.True(t, ok) - environmentsArray, ok := environments.([]interface{}) - assert.True(t, ok) - return environmentsArray -} - -func getEnvironmentFromResponse(t *testing.T, response *http.Response) interface{} { - t.Helper() - var environment interface{} - err := json.NewDecoder(response.Body).Decode(&environment) - require.NoError(t, err) - return environment -} - -//nolint:unparam // Because its more clear if the environment id is written in the real test -func deleteEnvironment(t *testing.T, id string) { - t.Helper() - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, id) - _, err := helpers.HTTPDelete(path, nil) - require.NoError(t, err) -} - -func cleanupJobsForEnvironment(t *testing.T, environmentID string) { - t.Helper() - - jobListStub, _, err := nomadClient.Jobs().List(&nomadApi.QueryOptions{Prefix: environmentID}) - if err != nil { - t.Fatalf("Error when listing test jobs: %v", err) - } - - for _, j := range jobListStub { - _, _, err := nomadClient.Jobs().DeregisterOpts(j.ID, &nomadApi.DeregisterOptions{Purge: true}, nil) - if err != nil { - t.Fatalf("Error when removing test job %v", err) - } - } -} - -//nolint:unparam // Because its more clear if the environment id is written in the real test -func createEnvironment(t *testing.T, environmentID string, aws bool) { - t.Helper() - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, environmentID) - request := dto.ExecutionEnvironmentRequest{ - PrewarmingPoolSize: 1, - CPULimit: 20, - MemoryLimit: 100, - NetworkAccess: false, - ExposedPorts: nil, - } - if aws { - functions := config.Config.AWS.Functions - require.NotZero(t, len(functions)) - request.Image = functions[0] - } else { - request.Image = *testDockerImage - } - assertPutReturnsStatusAndZeroContent(t, path, request, http.StatusCreated) -} - -func assertPutReturnsStatusAndZeroContent(t *testing.T, path string, - request dto.ExecutionEnvironmentRequest, status int) { - t.Helper() - resp, err := helpers.HTTPPutJSON(path, request) - require.Nil(t, err) - assert.Equal(t, status, resp.StatusCode) - assert.Equal(t, int64(0), resp.ContentLength) - content, err := io.ReadAll(resp.Body) - require.NoError(t, err) - assert.Empty(t, string(content)) - _ = resp.Body.Close() -} - -func validateJob(t *testing.T, expected dto.ExecutionEnvironmentRequest) { - t.Helper() - job := findTemplateJob(t, tests.AnotherEnvironmentIDAsInteger) - - assertEqualValueStringPointer(t, nomadNamespace, job.Namespace) - assertEqualValueStringPointer(t, "batch", job.Type) - require.Equal(t, 2, len(job.TaskGroups)) - - taskGroup := job.TaskGroups[0] - require.NotNil(t, taskGroup.Count) - // Poseidon might have already scaled the registered job up once. - prewarmingPoolSizeInt := int(expected.PrewarmingPoolSize) - taskGroupCount := *taskGroup.Count - assert.True(t, prewarmingPoolSizeInt == taskGroupCount || prewarmingPoolSizeInt+1 == taskGroupCount) - assertEqualValueIntPointer(t, int(expected.PrewarmingPoolSize), taskGroup.Count) - require.Equal(t, 1, len(taskGroup.Tasks)) - - task := taskGroup.Tasks[0] - assertEqualValueIntPointer(t, int(expected.CPULimit), task.Resources.CPU) - assertEqualValueIntPointer(t, int(expected.MemoryLimit), task.Resources.MemoryMaxMB) - assert.Equal(t, expected.Image, task.Config["image"]) - - if expected.NetworkAccess { - assert.Equal(t, "", task.Config["network_mode"]) - require.Equal(t, 1, len(taskGroup.Networks)) - network := taskGroup.Networks[0] - assert.Equal(t, len(expected.ExposedPorts), len(network.DynamicPorts)) - for _, port := range network.DynamicPorts { - assert.Contains(t, expected.ExposedPorts, uint16(port.To)) - } - } else { - assert.Equal(t, "none", task.Config["network_mode"]) - assert.Equal(t, 0, len(taskGroup.Networks)) - } -} - -func findTemplateJob(t *testing.T, id dto.EnvironmentID) *nomadApi.Job { - t.Helper() - job, _, err := nomadClient.Jobs().Info(nomad.TemplateJobID(id), nil) - if err != nil { - t.Fatalf("Error retrieving Nomad job: %v", err) - } - return job -} - -func assertEqualValueStringPointer(t *testing.T, value string, valueP *string) { - t.Helper() - require.NotNil(t, valueP) - assert.Equal(t, value, *valueP) -} - -func assertEqualValueIntPointer(t *testing.T, value int, valueP *int) { - t.Helper() - require.NotNil(t, valueP) - assert.Equal(t, value, *valueP) -} diff --git a/tests/e2e/health_test.go b/tests/e2e/health_test.go deleted file mode 100644 index a7e4209..0000000 --- a/tests/e2e/health_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package e2e - -import ( - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/assert" - "net/http" - "testing" -) - -func TestHealthRoute(t *testing.T) { - resp, err := http.Get(helpers.BuildURL(api.BasePath, api.HealthPath)) - if assert.NoError(t, err) { - assert.Equal(t, http.StatusNoContent, resp.StatusCode, "The response code should be NoContent") - } -} diff --git a/tests/e2e/runners_test.go b/tests/e2e/runners_test.go deleted file mode 100644 index a6e56e4..0000000 --- a/tests/e2e/runners_test.go +++ /dev/null @@ -1,490 +0,0 @@ -package e2e - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/gorilla/websocket" - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/require" - "io" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -func (s *E2ETestSuite) TestProvideRunnerRoute() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - runnerRequestByteString, err := json.Marshal(dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - s.Require().NoError(err) - reader := bytes.NewReader(runnerRequestByteString) - - s.Run("valid request returns a runner", func() { - resp, err := http.Post(helpers.BuildURL(api.BasePath, api.RunnersPath), "application/json", reader) - s.Require().NoError(err) - s.Equal(http.StatusOK, resp.StatusCode) - - runnerResponse := new(dto.RunnerResponse) - err = json.NewDecoder(resp.Body).Decode(runnerResponse) - s.Require().NoError(err) - s.NotEmpty(runnerResponse.ID) - }) - - s.Run("invalid request returns bad request", func() { - resp, err := http.Post(helpers.BuildURL(api.BasePath, api.RunnersPath), "application/json", strings.NewReader("")) - s.Require().NoError(err) - s.Equal(http.StatusBadRequest, resp.StatusCode) - }) - - s.Run("requesting runner of unknown execution environment returns not found", func() { - runnerRequestByteString, err := json.Marshal(dto.RunnerRequest{ - ExecutionEnvironmentID: tests.NonExistingIntegerID, - }) - s.Require().NoError(err) - reader := bytes.NewReader(runnerRequestByteString) - resp, err := http.Post(helpers.BuildURL(api.BasePath, api.RunnersPath), "application/json", reader) - s.Require().NoError(err) - s.Equal(http.StatusNotFound, resp.StatusCode) - }) - }) - } -} - -// CopyFiles sends the dto.UpdateFileSystemRequest to Poseidon. -// It needs a running Poseidon instance to work. -func CopyFiles(runnerID string, request *dto.UpdateFileSystemRequest) (*http.Response, error) { - data, err := json.Marshal(request) - if err != nil { - return nil, err - } - r := bytes.NewReader(data) - return helpers.HTTPPatch(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath), - "application/json", r) -} - -func (s *E2ETestSuite) TestDeleteRunnerRoute() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - s.NoError(err) - - s.Run("Deleting the runner returns NoContent", func() { - resp, err := helpers.HTTPDelete(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID), nil) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - }) - - s.Run("Deleting it again returns Gone", func() { - resp, err := helpers.HTTPDelete(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID), nil) - s.NoError(err) - s.Equal(http.StatusGone, resp.StatusCode) - }) - - s.Run("Deleting non-existing runner returns Gone", func() { - resp, err := helpers.HTTPDelete(helpers.BuildURL(api.BasePath, api.RunnersPath, tests.NonExistingStringID), nil) - s.NoError(err) - s.Equal(http.StatusGone, resp.StatusCode) - }) - }) - } -} - -func (s *E2ETestSuite) TestListFileSystem_Nomad() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - }) - require.NoError(s.T(), err) - - s.Run("No files", func() { - getFileURL, err := url.Parse(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath)) - s.Require().NoError(err) - response, err := http.Get(getFileURL.String()) - s.Require().NoError(err) - s.Equal(http.StatusOK, response.StatusCode) - data, err := io.ReadAll(response.Body) - s.NoError(err) - s.Equal("{\"files\": []}", string(data)) - }) - - s.Run("With file", func() { - resp, err := CopyFiles(runnerID, &dto.UpdateFileSystemRequest{ - Copy: []dto.File{{Path: tests.DefaultFileName, Content: []byte{}}}, - }) - s.Require().NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - getFileURL, err := url.Parse(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath)) - s.Require().NoError(err) - response, err := http.Get(getFileURL.String()) - s.Require().NoError(err) - s.Equal(http.StatusOK, response.StatusCode) - - listFilesResponse := new(dto.ListFileSystemResponse) - err = json.NewDecoder(response.Body).Decode(listFilesResponse) - s.Require().NoError(err) - s.Require().Equal(1, len(listFilesResponse.Files)) - fileHeader := listFilesResponse.Files[0] - s.Equal(dto.FilePath("./"+tests.DefaultFileName), fileHeader.Name) - s.Equal(dto.EntryTypeRegularFile, fileHeader.EntryType) - // ToDo: Reconsider if those files should be owned by root. - s.Equal("root", fileHeader.Owner) - s.Equal("root", fileHeader.Group) - s.Equal("rwxr--r--", fileHeader.Permissions) - }) -} - -//nolint:funlen // there are a lot of tests for the files route, this function can be a little longer than 100 lines ;) -func (s *E2ETestSuite) TestCopyFilesRoute() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - s.NoError(err) - - request := &dto.UpdateFileSystemRequest{ - Copy: []dto.File{{Path: tests.DefaultFileName, Content: []byte(tests.DefaultFileContent)}}, - } - - s.Run("File copy with valid payload succeeds", func() { - resp, err := CopyFiles(runnerID, request) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - s.Run("File content can be printed on runner", func() { - s.assertFileContent(runnerID, tests.DefaultFileName, tests.DefaultFileContent) - }) - }) - - s.Run("Files are put in correct location", func() { - relativeFilePath := "relative/file/path.txt" - relativeFileContent := "Relative file content" - absoluteFilePath := "/tmp/absolute/file/path.txt" - absoluteFileContent := "Absolute file content" - request = &dto.UpdateFileSystemRequest{ - Copy: []dto.File{ - {Path: dto.FilePath(relativeFilePath), Content: []byte(relativeFileContent)}, - {Path: dto.FilePath(absoluteFilePath), Content: []byte(absoluteFileContent)}, - }, - } - s.Require().NoError(err) - - resp, err := CopyFiles(runnerID, request) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - s.Run("File content of file with relative path can be printed on runner", func() { - // the print command is executed in the context of the default working directory of the container - s.assertFileContent(runnerID, relativeFilePath, relativeFileContent) - }) - - s.Run("File content of file with absolute path can be printed on runner", func() { - s.assertFileContent(runnerID, absoluteFilePath, absoluteFileContent) - }) - }) - - s.Run("File deletion request deletes file on runner", func() { - request = &dto.UpdateFileSystemRequest{ - Delete: []dto.FilePath{tests.DefaultFileName}, - } - s.Require().NoError(err) - - resp, err := CopyFiles(runnerID, request) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - s.Run("File content can no longer be printed", func() { - stdout, stderr := s.PrintContentOfFileOnRunner(runnerID, tests.DefaultFileName) - s.Equal("", stdout) - s.Contains(stderr, "No such file or directory") - }) - }) - - s.Run("File copy happens after file deletion", func() { - request = &dto.UpdateFileSystemRequest{ - Delete: []dto.FilePath{tests.DefaultFileName}, - Copy: []dto.File{{Path: tests.DefaultFileName, Content: []byte(tests.DefaultFileContent)}}, - } - s.Require().NoError(err) - - resp, err := CopyFiles(runnerID, request) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - _ = resp.Body.Close() - - s.Run("File content can be printed on runner", func() { - s.assertFileContent(runnerID, tests.DefaultFileName, tests.DefaultFileContent) - }) - }) - - s.Run("File copy with invalid payload returns bad request", func() { - resp, err := helpers.HTTPPatch(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath), - "text/html", strings.NewReader("")) - s.NoError(err) - s.Equal(http.StatusBadRequest, resp.StatusCode) - }) - - s.Run("Copying to non-existing runner returns Gone", func() { - resp, err := CopyFiles(tests.NonExistingStringID, request) - s.NoError(err) - s.Equal(http.StatusGone, resp.StatusCode) - }) - }) - } -} - -func (s *E2ETestSuite) TestCopyFilesRoute_PermissionDenied() { - s.Run("Nomad/If one file produces permission denied error, others are still copied", func() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - }) - s.NoError(err) - - newFileContent := []byte("New content") - copyFilesRequestByteString, err := json.Marshal(&dto.UpdateFileSystemRequest{ - Copy: []dto.File{ - {Path: "/proc/1/environ", Content: []byte(tests.DefaultFileContent)}, - {Path: tests.DefaultFileName, Content: newFileContent}, - }, - }) - s.Require().NoError(err) - - resp, err := helpers.HTTPPatch(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath), - "application/json", bytes.NewReader(copyFilesRequestByteString)) - s.NoError(err) - s.Equal(http.StatusInternalServerError, resp.StatusCode) - internalServerError := new(dto.InternalServerError) - err = json.NewDecoder(resp.Body).Decode(internalServerError) - s.NoError(err) - s.Contains(internalServerError.Message, "Cannot open: ") - _ = resp.Body.Close() - - s.Run("File content can be printed on runner", func() { - s.assertFileContent(runnerID, tests.DefaultFileName, string(newFileContent)) - }) - }) - - s.Run("AWS/If one file produces permission denied error, others are still copied", func() { - for _, environmentID := range environmentIDs { - if environmentID == tests.DefaultEnvironmentIDAsInteger { - continue - } - s.Run(environmentID.ToString(), func() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - s.NoError(err) - - newFileContent := []byte("New content") - copyFilesRequestByteString, err := json.Marshal(&dto.UpdateFileSystemRequest{ - Copy: []dto.File{ - {Path: "/proc/1/environ", Content: []byte(tests.DefaultFileContent)}, - {Path: tests.DefaultFileName, Content: newFileContent}, - }, - }) - s.Require().NoError(err) - - resp, err := helpers.HTTPPatch(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath), - "application/json", bytes.NewReader(copyFilesRequestByteString)) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - _ = resp.Body.Close() - - stdout, stderr := s.PrintContentOfFileOnRunner(runnerID, tests.DefaultFileName) - s.Equal(string(newFileContent), stdout) - s.Contains(stderr, "Exception") - }) - } - }) -} - -func (s *E2ETestSuite) TestCopyFilesRoute_ProtectedFolders() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger}) - s.NoError(err) - - // Initialization of protected folder - newFileContent := []byte("New content") - protectedFolderPath := dto.FilePath("protectedFolder/") - copyFilesRequestByteString, err := json.Marshal(&dto.UpdateFileSystemRequest{ - Copy: []dto.File{ - {Path: protectedFolderPath}, - {Path: protectedFolderPath + tests.DefaultFileName, Content: newFileContent}, - }, - }) - s.Require().NoError(err) - - resp, err := helpers.HTTPPatch(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.UpdateFileSystemPath), - "application/json", bytes.NewReader(copyFilesRequestByteString)) - s.NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - // User manipulates protected folder - s.Run("User can create files", func() { - log.WithField("id", runnerID).Debug("Runner ID") - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ - Command: fmt.Sprintf("touch %s%s", protectedFolderPath, "userfile"), - TimeLimit: int(tests.DefaultTestTimeout.Seconds()), - PrivilegedExecution: false, - }) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) - s.Empty(stdout) - s.Empty(stderr) - }) - - s.Run("User can not delete protected folder", func() { - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ - Command: fmt.Sprintf("rm -fr %s", protectedFolderPath), - TimeLimit: int(tests.DefaultTestTimeout.Seconds()), - PrivilegedExecution: false, - }) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) - s.Empty(stdout) - s.Contains(stderr, "Operation not permitted") - }) - - s.Run("User can not delete protected file", func() { - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ - Command: fmt.Sprintf("rm -f %s", protectedFolderPath+tests.DefaultFileName), - TimeLimit: int(tests.DefaultTestTimeout.Seconds()), - PrivilegedExecution: false, - }) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) - s.Empty(stdout) - s.Contains(stderr, "Operation not permitted") - }) - - s.Run("User can not write protected file", func() { - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ - Command: fmt.Sprintf("echo Hi >> %s", protectedFolderPath+tests.DefaultFileName), - TimeLimit: int(tests.DefaultTestTimeout.Seconds()), - PrivilegedExecution: false, - }) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - stdout, stderr, _ := helpers.WebSocketOutputMessages(messages) - s.Empty(stdout) - s.Contains(stderr, "Permission denied") - }) - - s.Run("File content is not manipulated", func() { - s.assertFileContent(runnerID, string(protectedFolderPath+tests.DefaultFileName), string(newFileContent)) - }) -} - -func (s *E2ETestSuite) TestGetFileContent_Nomad() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - }) - require.NoError(s.T(), err) - - s.Run("Not Found", func() { - getFileURL, err := url.Parse(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.FileContentRawPath)) - s.Require().NoError(err) - getFileURL.RawQuery = fmt.Sprintf("%s=%s", api.PathKey, tests.DefaultFileName) - response, err := http.Get(getFileURL.String()) - s.Require().NoError(err) - s.Equal(http.StatusFailedDependency, response.StatusCode) - }) - - s.Run("Ok", func() { - newFileContent := []byte("New content") - resp, err := CopyFiles(runnerID, &dto.UpdateFileSystemRequest{ - Copy: []dto.File{{Path: tests.DefaultFileName, Content: newFileContent}}, - }) - s.Require().NoError(err) - s.Equal(http.StatusNoContent, resp.StatusCode) - - getFileURL, err := url.Parse(helpers.BuildURL(api.BasePath, api.RunnersPath, runnerID, api.FileContentRawPath)) - s.Require().NoError(err) - getFileURL.RawQuery = fmt.Sprintf("%s=%s", api.PathKey, tests.DefaultFileName) - response, err := http.Get(getFileURL.String()) - s.Require().NoError(err) - s.Equal(http.StatusOK, response.StatusCode) - s.Equal(strconv.Itoa(len(newFileContent)), response.Header.Get("Content-Length")) - s.Equal("attachment; filename=\""+tests.DefaultFileName+"\"", response.Header.Get("Content-Disposition")) - content, err := io.ReadAll(response.Body) - s.Require().NoError(err) - s.Equal(newFileContent, content) - }) -} - -func (s *E2ETestSuite) TestRunnerGetsDestroyedAfterInactivityTimeout() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - inactivityTimeout := 2 // seconds - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: int(environmentID), - InactivityTimeout: inactivityTimeout, - }) - s.Require().NoError(err) - - executionTerminated := make(chan bool) - var lastMessage *dto.WebSocketMessage - go func() { - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "sleep infinity"}) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - if !s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) { - s.Fail("websocket abnormal closure") - } - controlMessages := helpers.WebSocketControlMessages(messages) - s.Require().NotEmpty(controlMessages) - lastMessage = controlMessages[len(controlMessages)-1] - log.Warn("") - executionTerminated <- true - }() - s.Require().True(tests.ChannelReceivesSomething(executionTerminated, time.Duration(inactivityTimeout+5)*time.Second)) - s.Equal(dto.WebSocketMetaTimeout, lastMessage.Type) - }) - } -} - -func (s *E2ETestSuite) assertFileContent(runnerID, fileName, expectedContent string) { - stdout, stderr := s.PrintContentOfFileOnRunner(runnerID, fileName) - s.Equal(expectedContent, stdout) - s.Equal("", stderr) -} - -func (s *E2ETestSuite) PrintContentOfFileOnRunner(runnerID, filename string) (stdout, stderr string) { - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{ - Command: fmt.Sprintf("cat %s", filename), - TimeLimit: int(tests.DefaultTestTimeout.Seconds()), - }) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - - stdout, stderr, _ = helpers.WebSocketOutputMessages(messages) - return stdout, stderr -} diff --git a/tests/e2e/websocket_test.go b/tests/e2e/websocket_test.go deleted file mode 100644 index 9b112ad..0000000 --- a/tests/e2e/websocket_test.go +++ /dev/null @@ -1,398 +0,0 @@ -package e2e - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/gorilla/websocket" - "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/nomad" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/logging" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "net/http" - "os" - "regexp" - "strconv" - "strings" - "testing" - "time" -) - -const ( - EnvPoseidonLogFile = "POSEIDON_LOG_FILE" - EnvPoseidonLogFormatter = "POSEIDON_LOGGER_FORMATTER" -) - -func (s *E2ETestSuite) TestExecuteCommandRoute() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - s.Require().NoError(err) - - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "true"}) - s.Require().NoError(err) - s.NotEqual("", webSocketURL) - - var connection *websocket.Conn - var connectionClosed bool - - connection, err = ConnectToWebSocket(webSocketURL) - s.Require().NoError(err, "websocket connects") - closeHandler := connection.CloseHandler() - connection.SetCloseHandler(func(code int, text string) error { - connectionClosed = true - return closeHandler(code, text) - }) - - startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaStart}, startMessage) - - exitMessage, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(&dto.WebSocketMessage{Type: dto.WebSocketExit}, exitMessage) - - _, err = helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - - _, _, err = connection.ReadMessage() - s.True(websocket.IsCloseError(err, websocket.CloseNormalClosure)) - s.True(connectionClosed, "connection should be closed") - }) - } -} - -func (s *E2ETestSuite) TestOutputToStdout() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - stdout, _, _ := ExecuteNonInteractive(&s.Suite, environmentID, - &dto.ExecutionRequest{Command: "echo -n Hello World"}, nil) - s.Require().Equal("Hello World", stdout) - }) - } -} - -func (s *E2ETestSuite) TestOutputToStderr() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - stdout, stderr, exitCode := ExecuteNonInteractive(&s.Suite, environmentID, - &dto.ExecutionRequest{Command: "cat -invalid"}, nil) - - s.NotContains(stdout, "cat: invalid option", "Stdout should not contain the error") - s.Contains(stderr, "cat: invalid option", "Stderr should contain the error") - s.Equal(uint8(1), exitCode) - }) - } -} - -func (s *E2ETestSuite) TestUserNomad() { - s.Run("unprivileged", func() { - stdout, _, _ := ExecuteNonInteractive(&s.Suite, tests.DefaultEnvironmentIDAsInteger, - &dto.ExecutionRequest{Command: "id --name --user", PrivilegedExecution: false}, nil) - s.Require().NotEqual("root", stdout) - }) - s.Run("privileged", func() { - stdout, _, _ := ExecuteNonInteractive(&s.Suite, tests.DefaultEnvironmentIDAsInteger, - &dto.ExecutionRequest{Command: "id --name --user", PrivilegedExecution: true}, nil) - s.Contains(stdout, "root") - }) -} - -// AWS environments do not support stdin at this moment therefore they cannot take this test. -func (s *E2ETestSuite) TestCommandHead() { - hello := "Hello World!" - connection, err := ProvideWebSocketConnection(&s.Suite, tests.DefaultEnvironmentIDAsInteger, - &dto.ExecutionRequest{Command: "head -n 1"}, nil) - s.Require().NoError(err) - - startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, startMessage.Type) - - err = connection.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("%s\n", hello))) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure}) - stdout, _, _ := helpers.WebSocketOutputMessages(messages) - s.Regexp(fmt.Sprintf(`(%s\r\n?){2}`, hello), stdout) -} - -func (s *E2ETestSuite) TestCommandMake() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - expectedOutput := "MeinText" - request := &dto.UpdateFileSystemRequest{ - Copy: []dto.File{ - {Path: "Makefile", Content: []byte( - "run:\n\t@echo " + expectedOutput + "\n\n" + - "test:\n\t@echo Hi\n"), - }, - }, - } - - stdout, _, _ := ExecuteNonInteractive(&s.Suite, environmentID, &dto.ExecutionRequest{Command: "make run"}, request) - stdout = regexp.MustCompile(`\r?\n$`).ReplaceAllString(stdout, "") - s.Equal(expectedOutput, stdout) - }) - } -} - -func (s *E2ETestSuite) TestEnvironmentVariables() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - stdout, _, _ := ExecuteNonInteractive(&s.Suite, environmentID, &dto.ExecutionRequest{ - Command: "env", - Environment: map[string]string{"hello": "world"}, - }, nil) - - variables := s.expectEnvironmentVariables(stdout) - s.Contains(variables, "hello=world") - }) - } -} - -func (s *E2ETestSuite) TestCommandMakeEnvironmentVariables() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - request := &dto.UpdateFileSystemRequest{ - Copy: []dto.File{{Path: "Makefile", Content: []byte("run:\n\t@env\n")}}, - } - - stdout, _, _ := ExecuteNonInteractive(&s.Suite, environmentID, &dto.ExecutionRequest{Command: "make run"}, request) - s.expectEnvironmentVariables(stdout) - }) - } -} - -func (s *E2ETestSuite) expectEnvironmentVariables(stdout string) []string { - variables := strings.Split(strings.ReplaceAll(stdout, "\r\n", "\n"), "\n") - s.Contains(variables, "CODEOCEAN=true") - for _, envVar := range variables { - s.False(strings.HasPrefix(envVar, "AWS")) - s.False(strings.HasPrefix(envVar, "NOMAD_")) - } - return variables -} - -func (s *E2ETestSuite) TestCommandReturnsAfterTimeout() { - for _, environmentID := range environmentIDs { - s.Run(environmentID.ToString(), func() { - connection, err := ProvideWebSocketConnection(&s.Suite, environmentID, - &dto.ExecutionRequest{Command: "sleep 4", TimeLimit: 1}, nil) - s.Require().NoError(err) - - c := make(chan bool) - var messages []*dto.WebSocketMessage - go func() { - messages, err = helpers.ReceiveAllWebSocketMessages(connection) - if !s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) { - s.T().Fail() - } - close(c) - }() - - select { - case <-time.After(2 * time.Second): - s.T().Fatal("The execution should have returned by now") - case <-c: - if s.Equal(&dto.WebSocketMessage{Type: dto.WebSocketMetaTimeout}, messages[len(messages)-1]) { - return - } - } - s.T().Fail() - }) - } -} - -func (s *E2ETestSuite) TestMemoryMaxLimit_Nomad() { - maxMemoryLimit := defaultNomadEnvironment.MemoryLimit - // The operating system is in charge to kill the process and sometimes tolerates small exceeding of the limit. - maxMemoryLimit = uint(1.1 * float64(maxMemoryLimit)) - - stdout, stderr, _ := ExecuteNonInteractive(&s.Suite, tests.DefaultEnvironmentIDAsInteger, &dto.ExecutionRequest{ - // This shell line tries to load maxMemoryLimit Bytes into the memory. - Command: " /dev/null", - }, nil) - s.Empty(stdout) - s.Contains(stderr, "Killed") -} - -func (s *E2ETestSuite) TestNomadStderrFifoIsRemoved() { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - }) - s.Require().NoError(err) - - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "ls -a /tmp/"}) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(&websocket.CloseError{Code: websocket.CloseNormalClosure}, err) - - stdout, _, _ := helpers.WebSocketOutputMessages(messages) - s.Contains(stdout, ".fifo", "there should be a .fifo file during the execution") - - s.NotContains(s.ListTempDirectory(runnerID), ".fifo", "/tmp/ should not contain any .fifo files after the execution") -} - -func (s *E2ETestSuite) TestTerminatedByClient() { - logFile, logFileOk := os.LookupEnv(EnvPoseidonLogFile) - logFormatter, logFormatterOk := os.LookupEnv(EnvPoseidonLogFormatter) - if !logFileOk || !logFormatterOk || logFormatter != dto.FormatterJSON { - s.T().Skipf("The environment variables %s and %s are not set", EnvPoseidonLogFile, EnvPoseidonLogFormatter) - return - } - start := time.Now() - - // The bug of #325 is triggered in about every second execution. Therefore, we perform - // 10 executions to have a high probability of triggering this (fixed) behavior. - const runs = 10 - for i := 0; i < runs; i++ { - <-time.After(time.Duration(i) * time.Second) - log.WithField("i", i).Info("Run") - runnerID, err := ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - }) - s.Require().NoError(err) - - webSocketURL, err := ProvideWebSocketURL(runnerID, &dto.ExecutionRequest{Command: "sleep 2"}) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - s.Require().NoError(err) - - go func() { - <-time.After(time.Millisecond) - err := connection.WriteControl(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) - s.Require().NoError(err) - err = connection.Close() - s.Require().NoError(err) - }() - - _, err = helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - } - - records := parseLogFile(s.T(), logFile, start, time.Now()) - for _, record := range records { - msg, ok := record["msg"].(string) - if !ok || msg == "Exec debug message could not be read completely" { - s.Failf("Found Error", "Ok: %t, message: %s", ok, msg) - } - } -} - -func parseLogFile(t *testing.T, name string, start time.Time, end time.Time) (logRecords []map[string]interface{}) { - t.Helper() - <-time.After(tests.ShortTimeout) - file, err := os.Open(name) - require.NoError(t, err) - defer func(t *testing.T, file *os.File) { - t.Helper() - err := file.Close() - require.NoError(t, err) - }(t, file) - fileScanner := bufio.NewScanner(file) - fileScanner.Split(bufio.ScanLines) - for fileScanner.Scan() { - logRecord := map[string]interface{}{} - err = json.Unmarshal(fileScanner.Bytes(), &logRecord) - require.NoError(t, err) - timeString, ok := logRecord["time"].(string) - require.True(t, ok) - entryTime, err := time.ParseInLocation(logging.TimestampFormat, timeString, start.Location()) - require.NoError(t, err) - if entryTime.Before(start) || entryTime.After(end) { - continue - } - logRecords = append(logRecords, logRecord) - } - return logRecords -} - -func (s *E2ETestSuite) ListTempDirectory(runnerID string) string { - allocListStub, _, err := nomadClient.Jobs().Allocations(runnerID, true, nil) - s.Require().NoError(err) - var runningAllocStub *api.AllocationListStub - for _, stub := range allocListStub { - if stub.ClientStatus == api.AllocClientStatusRunning && stub.DesiredStatus == api.AllocDesiredStatusRun { - runningAllocStub = stub - break - } - } - alloc, _, err := nomadClient.Allocations().Info(runningAllocStub.ID, nil) - s.Require().NoError(err) - - var stdout, stderr bytes.Buffer - exit, err := nomadClient.Allocations().Exec(context.Background(), alloc, nomad.TaskName, - false, []string{"ls", "-a", "/tmp/"}, strings.NewReader(""), &stdout, &stderr, nil, nil) - - s.Require().NoError(err) - s.Require().Equal(0, exit) - s.Require().Empty(stderr) - return stdout.String() -} - -// ExecuteNonInteractive Executes the passed executionRequest in the required environment without providing input. -func ExecuteNonInteractive(s *suite.Suite, environmentID dto.EnvironmentID, executionRequest *dto.ExecutionRequest, - copyRequest *dto.UpdateFileSystemRequest) (stdout, stderr string, exitCode uint8) { - connection, err := ProvideWebSocketConnection(s, environmentID, executionRequest, copyRequest) - s.Require().NoError(err) - - startMessage, err := helpers.ReceiveNextWebSocketMessage(connection) - s.Require().NoError(err) - s.Equal(dto.WebSocketMetaStart, startMessage.Type) - - messages, err := helpers.ReceiveAllWebSocketMessages(connection) - s.Require().Error(err) - s.Equal(err, &websocket.CloseError{Code: websocket.CloseNormalClosure}) - - controlMessages := helpers.WebSocketControlMessages(messages) - s.Require().Equal(1, len(controlMessages)) - exitMessage := controlMessages[0] - s.Require().Equal(dto.WebSocketExit, exitMessage.Type) - - stdout, stderr, errors := helpers.WebSocketOutputMessages(messages) - s.Empty(errors) - return stdout, stderr, exitMessage.ExitCode -} - -// ProvideWebSocketConnection establishes a client WebSocket connection to run the passed ExecutionRequest. -func ProvideWebSocketConnection(s *suite.Suite, environmentID dto.EnvironmentID, executionRequest *dto.ExecutionRequest, - copyRequest *dto.UpdateFileSystemRequest) (*websocket.Conn, error) { - runnerID, err := ProvideRunner(&dto.RunnerRequest{ExecutionEnvironmentID: int(environmentID)}) - if err != nil { - return nil, fmt.Errorf("error providing runner: %w", err) - } - if copyRequest != nil { - resp, err := CopyFiles(runnerID, copyRequest) - s.Require().NoError(err) - s.Require().Equal(http.StatusNoContent, resp.StatusCode) - } - webSocketURL, err := ProvideWebSocketURL(runnerID, executionRequest) - s.Require().NoError(err) - connection, err := ConnectToWebSocket(webSocketURL) - if err != nil { - return nil, fmt.Errorf("error connecting to WebSocket: %w", err) - } - return connection, nil -} - -// ConnectToWebSocket establish an external WebSocket connection to the provided url. -// It requires a running Poseidon instance. -func ConnectToWebSocket(url string) (conn *websocket.Conn, err error) { - conn, _, err = websocket.DefaultDialer.Dial(url, nil) - return -} diff --git a/tests/helpers/test_helpers.go b/tests/helpers/test_helpers.go deleted file mode 100644 index 9b767b1..0000000 --- a/tests/helpers/test_helpers.go +++ /dev/null @@ -1,233 +0,0 @@ -// Package helpers contains functions that help executing tests. -// The helper functions generally look from the client side - a Poseidon user. -package helpers - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "fmt" - "github.com/gorilla/mux" - "github.com/gorilla/websocket" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "io" - "net/http" - "net/http/httptest" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -// BuildURL joins multiple route paths. -func BuildURL(parts ...string) string { - url := config.Config.Server.URL().String() - for _, part := range parts { - if !strings.HasPrefix(part, "/") { - url += "/" - } - url += part - } - return url -} - -// WebSocketOutputMessages extracts all stdout, stderr and error messages from the passed messages. -// It is useful since Nomad splits the command output nondeterministic. -func WebSocketOutputMessages(messages []*dto.WebSocketMessage) (stdout, stderr string, errors []string) { - for _, msg := range messages { - switch msg.Type { - case dto.WebSocketOutputStdout: - stdout += msg.Data - case dto.WebSocketOutputStderr: - stderr += msg.Data - case dto.WebSocketOutputError: - errors = append(errors, msg.Data) - } - } - return stdout, stderr, errors -} - -// WebSocketControlMessages extracts all meta (and exit) messages from the passed messages. -func WebSocketControlMessages(messages []*dto.WebSocketMessage) (controls []*dto.WebSocketMessage) { - for _, msg := range messages { - switch msg.Type { - case dto.WebSocketMetaStart, dto.WebSocketMetaTimeout, dto.WebSocketExit: - controls = append(controls, msg) - } - } - return controls -} - -// ReceiveAllWebSocketMessages pulls all messages from the websocket connection without sending anything. -// This function does not return unless the server closes the connection or a readDeadline is set -// in the WebSocket connection. -func ReceiveAllWebSocketMessages(connection *websocket.Conn) (messages []*dto.WebSocketMessage, err error) { - for { - var message *dto.WebSocketMessage - message, err = ReceiveNextWebSocketMessage(connection) - if err != nil { - return messages, err - } - messages = append(messages, message) - } -} - -// ReceiveNextWebSocketMessage pulls the next message from the websocket connection. -// This function does not return unless the server sends a message, closes the connection or a readDeadline -// is set in the WebSocket connection. -func ReceiveNextWebSocketMessage(connection *websocket.Conn) (*dto.WebSocketMessage, error) { - _, reader, err := connection.NextReader() - if err != nil { - //nolint:wrapcheck // we could either wrap here and do complicated things with errors.As or just not wrap - // the error in this test function and allow tests to use equal - return nil, err - } - message := new(dto.WebSocketMessage) - err = json.NewDecoder(reader).Decode(message) - if err != nil { - return nil, fmt.Errorf("error decoding WebSocket message: %w", err) - } - return message, nil -} - -// StartTLSServer runs a httptest.Server with the passed mux.Router and TLS enabled. -func StartTLSServer(t *testing.T, router *mux.Router) (*httptest.Server, error) { - t.Helper() - dir := t.TempDir() - keyOut := filepath.Join(dir, "poseidon-test.key") - certOut := filepath.Join(dir, "poseidon-test.crt") - - err := exec.Command("openssl", "req", "-x509", "-nodes", "-newkey", "rsa:2048", - "-keyout", keyOut, "-out", certOut, "-days", "1", - "-subj", "/CN=Poseidon test", "-addext", "subjectAltName=IP:127.0.0.1,DNS:localhost").Run() - if err != nil { - return nil, fmt.Errorf("error creating self-signed cert: %w", err) - } - cert, err := tls.LoadX509KeyPair(certOut, keyOut) - if err != nil { - return nil, fmt.Errorf("error loading x509 key pair: %w", err) - } - - server := httptest.NewUnstartedServer(router) - server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13} - server.StartTLS() - return server, nil -} - -// httpRequest deduplicates the comment and error message wrapping the http.NewRequest call. -func httpRequest(method, url string, body io.Reader) (*http.Request, error) { - //nolint:noctx // we don't need a http.NewRequestWithContext in our tests - req, err := http.NewRequest(method, url, body) - if err != nil { - return nil, fmt.Errorf("error creating request: %w", err) - } - return req, nil -} - -// HTTPDelete sends a "Delete" Http Request with body to the passed url. -func HTTPDelete(url string, body io.Reader) (response *http.Response, err error) { - req, err := httpRequest(http.MethodDelete, url, body) - if err != nil { - return nil, err - } - client := &http.Client{} - response, err = client.Do(req) - if err != nil { - return nil, fmt.Errorf("error executing request: %w", err) - } - return response, nil -} - -// HTTPPatch sends a Patch Http Request with body to the passed url. -func HTTPPatch(url, contentType string, body io.Reader) (response *http.Response, err error) { - //nolint:noctx // we don't need a http.NewRequestWithContext in our tests - req, err := httpRequest(http.MethodPatch, url, body) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", contentType) - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("error executing request: %w", err) - } - return resp, nil -} - -func HTTPPut(url string, body io.Reader) (response *http.Response, err error) { - //nolint:noctx // we don't need a http.NewRequestWithContext in our tests - req, err := httpRequest(http.MethodPut, url, body) - if err != nil { - return nil, err - } - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("error executing request: %w", err) - } - return resp, nil -} - -func HTTPPutJSON(url string, body interface{}) (response *http.Response, err error) { - requestByteString, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("cannot marshal json http body: %w", err) - } - reader := bytes.NewReader(requestByteString) - return HTTPPut(url, reader) -} - -func HTTPPostJSON(url string, body interface{}) (response *http.Response, err error) { - requestByteString, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("cannot marshal passed http post body: %w", err) - } - bodyReader := bytes.NewReader(requestByteString) - - //nolint:noctx // we don't need a http.NewRequestWithContext in our tests - req, err := httpRequest(http.MethodPost, url, bodyReader) - if err != nil { - return nil, err - } - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("error executing request: %w", err) - } - return resp, nil -} - -const templateJobPriority = 100 - -func CreateTemplateJob() (base, job *nomadApi.Job) { - base = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority) - job = nomadApi.NewBatchJob(tests.DefaultTemplateJobID, tests.DefaultTemplateJobID, "global", templateJobPriority) - job.Datacenters = []string{"dc1"} - jobStatus := structs.JobStatusRunning - job.Status = &jobStatus - configTaskGroup := nomadApi.NewTaskGroup("config", 0) - configTaskGroup.Meta = make(map[string]string) - configTaskGroup.Meta["prewarmingPoolSize"] = "0" - configTask := nomadApi.NewTask("config", "exec") - configTask.Config = map[string]interface{}{"command": "true"} - configTaskGroup.AddTask(configTask) - - defaultTaskGroup := nomadApi.NewTaskGroup("default-group", 1) - defaultTaskGroup.Networks = []*nomadApi.NetworkResource{} - defaultTask := nomadApi.NewTask("default-task", "docker") - defaultTask.Config = map[string]interface{}{ - "image": "python:latest", - "command": "sleep", - "args": []string{"infinity"}, - "network_mode": "none", - } - defaultTask.Resources = nomadApi.DefaultResources() - defaultTaskGroup.AddTask(defaultTask) - - job.TaskGroups = []*nomadApi.TaskGroup{defaultTaskGroup, configTaskGroup} - return base, job -} diff --git a/tests/recovery/e2e_recovery_test.go b/tests/recovery/e2e_recovery_test.go deleted file mode 100644 index 93ca24c..0000000 --- a/tests/recovery/e2e_recovery_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package recovery - -import ( - "encoding/json" - "flag" - nomadApi "github.com/hashicorp/nomad/api" - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/internal/config" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/pkg/logging" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/e2e" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/stretchr/testify/suite" - "net/http" - "os" - "os/exec" - "strconv" - "strings" - "testing" - "time" -) - -/* -* # E2E Recovery Tests -* -* For the e2e tests a nomad cluster must be connected and poseidon must be running. -* These cases test the behavior of Poseidon when restarting / recovering. - */ - -var ( - log = logging.GetLogger("e2e-recovery") - testDockerImage = flag.String("dockerImage", "", "Docker image to use in E2E tests") - nomadClient *nomadApi.Client - nomadNamespace string -) - -// InactivityTimeout of the created runner in seconds. -const ( - InactivityTimeout = 1 - PrewarmingPoolSize = 2 -) - -type E2ERecoveryTestSuite struct { - suite.Suite - runnerID string -} - -// Overwrite TestMain for custom setup. -func TestMain(m *testing.M) { - if err := config.InitConfig(); err != nil { - log.WithError(err).Fatal("Could not initialize configuration") - } - if *testDockerImage == "" { - log.Fatal("You must specify the -dockerImage flag!") - } - - nomadNamespace = config.Config.Nomad.Namespace - var err error - nomadClient, err = nomadApi.NewClient(&nomadApi.Config{ - Address: config.Config.Nomad.URL().String(), - TLSConfig: &nomadApi.TLSConfig{}, - Namespace: nomadNamespace, - }) - if err != nil { - log.WithError(err).Fatal("Could not create Nomad client") - return - } - - os.Exit(m.Run()) -} - -func TestE2ERecoveryTests(t *testing.T) { - testSuite := new(E2ERecoveryTestSuite) - - e2e.CreateDefaultEnvironment(PrewarmingPoolSize, *testDockerImage) - e2e.WaitForDefaultEnvironment() - - suite.Run(t, testSuite) - - TearDown() -} - -func (s *E2ERecoveryTestSuite) TestInactivityTimer_Valid() { - _, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"}) - s.NoError(err) -} - -func (s *E2ERecoveryTestSuite) TestInactivityTimer_Expired() { - waitForPoseidon() // The timeout begins only when the runner is recovered. - <-time.After(InactivityTimeout * time.Second) - _, err := e2e.ProvideWebSocketURL(s.runnerID, &dto.ExecutionRequest{Command: "true"}) - s.Error(err) -} - -// We expect the runner count to be equal to the prewarming pool size plus the one provided runner. -// If the count does not include the provided runner, the evaluation of the runner status may be wrong. -func (s *E2ERecoveryTestSuite) TestRunnerCount() { - jobListStubs, _, err := nomadClient.Jobs().List(&nomadApi.QueryOptions{ - Prefix: tests.DefaultEnvironmentIDAsString, - Namespace: nomadNamespace, - }) - s.Require().NoError(err) - s.Equal(PrewarmingPoolSize+1, len(jobListStubs)) -} - -func (s *E2ERecoveryTestSuite) TestEnvironmentStatistics() { - url := helpers.BuildURL(api.BasePath, api.StatisticsPath, api.EnvironmentsPath) - response, err := http.Get(url) //nolint:gosec // The variability of this url is limited by our configurations. - s.Require().NoError(err) - s.Require().Equal(http.StatusOK, response.StatusCode) - - statistics := make(map[string]*dto.StatisticalExecutionEnvironmentData) - err = json.NewDecoder(response.Body).Decode(&statistics) - s.Require().NoError(err) - err = response.Body.Close() - s.Require().NoError(err) - - environmentStatistics, ok := statistics[tests.DefaultEnvironmentIDAsString] - s.Require().True(ok) - s.Equal(tests.DefaultEnvironmentIDAsInteger, environmentStatistics.ID) - s.Equal(uint(PrewarmingPoolSize), environmentStatistics.PrewarmingPoolSize) - s.Equal(uint(PrewarmingPoolSize), environmentStatistics.IdleRunners) - s.Equal(uint(1), environmentStatistics.UsedRunners) -} - -func (s *E2ERecoveryTestSuite) TestWatchdogNotifications() { - // Wait for `WatchdogSec` to be passed. - <-time.After((5 + 1) * time.Second) - - // If the Watchdog has not received the notification by now it will restart Poseidon. - cmd := exec.Command("/usr/bin/systemctl", "--user", "show", "poseidon.service", "-p", "NRestarts") - s.Require().NoError(cmd.Err) - out, err := cmd.Output() - s.Require().NoError(err) - - restarts, err := strconv.Atoi(strings.Trim(strings.ReplaceAll(string(out), "NRestarts=", ""), "\n")) - s.Require().NoError(err) - // If Poseidon would not notify the systemd watchdog, we would have one more restart than expected. - s.Equal(PoseidonRestartCount, restarts) -} diff --git a/tests/recovery/setup_test.go b/tests/recovery/setup_test.go deleted file mode 100644 index 1b8fe26..0000000 --- a/tests/recovery/setup_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package recovery - -import ( - "github.com/openHPI/poseidon/internal/api" - "github.com/openHPI/poseidon/pkg/dto" - "github.com/openHPI/poseidon/tests" - "github.com/openHPI/poseidon/tests/e2e" - "github.com/openHPI/poseidon/tests/helpers" - "github.com/shirou/gopsutil/v3/process" - "golang.org/x/sys/unix" - "net/http" - "time" -) - -func (s *E2ERecoveryTestSuite) SetupTest() { - <-time.After(InactivityTimeout * time.Second) - // We do not want runner from the previous tests - - var err error - s.runnerID, err = e2e.ProvideRunner(&dto.RunnerRequest{ - ExecutionEnvironmentID: tests.DefaultEnvironmentIDAsInteger, - InactivityTimeout: InactivityTimeout, - }) - if err != nil { - log.WithError(err).Fatal("Could not provide runner") - } - - <-time.After(tests.ShortTimeout) - killPoseidon() - <-time.After(tests.ShortTimeout) -} - -func TearDown() { - path := helpers.BuildURL(api.BasePath, api.EnvironmentsPath, tests.DefaultEnvironmentIDAsString) - _, err := helpers.HTTPDelete(path, nil) - if err != nil { - log.WithError(err).Fatal("Could not remove default environment") - } -} - -func waitForPoseidon() { - done := false - for !done { - <-time.After(time.Second) - resp, err := http.Get(helpers.BuildURL(api.BasePath, api.HealthPath)) - done = err == nil && resp.StatusCode == http.StatusNoContent - } -} - -var PoseidonRestartCount = 0 - -func killPoseidon() { - processes, err := process.Processes() - if err != nil { - log.WithError(err).Error("Error listing processes") - } - for _, p := range processes { - n, err := p.Name() - if err != nil { - continue - } - if n == "poseidon" { - err = p.SendSignal(unix.SIGTERM) - if err != nil { - log.WithError(err).Error("Error killing Poseidon") - } else { - log.Info("Killed Poseidon") - PoseidonRestartCount++ - } - } - } -} diff --git a/tests/util.go b/tests/util.go deleted file mode 100644 index 4d8030f..0000000 --- a/tests/util.go +++ /dev/null @@ -1,91 +0,0 @@ -package tests - -import ( - "bytes" - "context" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "io" - "os" - "regexp" - "runtime" - "runtime/pprof" - "strconv" - "time" -) - -// ChannelReceivesSomething waits timeout seconds for something to be received from channel ch. -// If something is received, it returns true. If the timeout expires without receiving anything, it return false. -func ChannelReceivesSomething(ch chan bool, timeout time.Duration) bool { - select { - case <-ch: - return true - case <-time.After(timeout): - return false - } -} - -var numGoroutines = regexp.MustCompile(`^goroutine profile: total (\d*)\n`) - -// MemoryLeakTestSuite adds an assertion for checking Goroutine leaks. -// Be aware not to overwrite the SetupTest or TearDownTest function! -type MemoryLeakTestSuite struct { - suite.Suite - ExpectedGoroutineIncrease int - TestCtx context.Context - testCtxCancel context.CancelFunc - goroutineCountBefore int - goroutinesBefore *bytes.Buffer -} - -func (s *MemoryLeakTestSuite) lookupGoroutines() (debugOutput *bytes.Buffer, goroutineCount int) { - debugOutput = &bytes.Buffer{} - err := pprof.Lookup("goroutine").WriteTo(debugOutput, 1) - s.Require().NoError(err) - match := numGoroutines.FindSubmatch(debugOutput.Bytes()) - if match == nil { - s.Fail("gouroutines could not be parsed: " + debugOutput.String()) - } - - // We do not use runtime.NumGoroutine() to not create inconsistency to the Lookup. - goroutineCount, err = strconv.Atoi(string(match[1])) - if err != nil { - s.Fail("number of goroutines could not be parsed: " + err.Error()) - } - return debugOutput, goroutineCount -} - -func (s *MemoryLeakTestSuite) SetupTest() { - runtime.Gosched() // Flush done Goroutines - <-time.After(ShortTimeout) // Just to make sure - s.ExpectedGoroutineIncrease = 0 - s.goroutinesBefore, s.goroutineCountBefore = s.lookupGoroutines() - - ctx, cancel := context.WithCancel(context.Background()) - s.TestCtx = ctx - s.testCtxCancel = cancel -} - -func (s *MemoryLeakTestSuite) TearDownTest() { - s.testCtxCancel() - runtime.Gosched() // Flush done Goroutines - <-time.After(ShortTimeout) // Just to make sure - - goroutinesAfter, goroutineCountAfter := s.lookupGoroutines() - s.Equal(s.goroutineCountBefore+s.ExpectedGoroutineIncrease, goroutineCountAfter) - if s.goroutineCountBefore+s.ExpectedGoroutineIncrease != goroutineCountAfter { - _, err := io.Copy(os.Stderr, s.goroutinesBefore) - s.NoError(err) - _, err = io.Copy(os.Stderr, goroutinesAfter) - s.NoError(err) - } -} - -func RemoveMethodFromMock(m *mock.Mock, method string) { - for i, call := range m.ExpectedCalls { - if call.Method == method { - m.ExpectedCalls = append(m.ExpectedCalls[:i], m.ExpectedCalls[i+1:]...) - return - } - } -}