AWS SAM deployment (#91)

* Generate AWS SAM application
with the Poseidon Java 11 Executor Lambda Function.

* Extend AWS Lambda documentation.

* Apply suggestions from code review

Co-authored-by: Sebastian Serth <MrSerth@users.noreply.github.com>

* Parse dynamic AWS region

Co-authored-by: Sebastian Serth <MrSerth@users.noreply.github.com>

deploy/aws/.gitignore

@@ -0,0 +1 @@
+java11Exec/target/

deploy/aws/README.md

@@ -0,0 +1,44 @@
+# Poseidon AWS Executors
+
+This project contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. It includes the following functions.
+
+- java11ExecFunction - Code for the application's Lambda function. It can execute Java files with JDK 11.
+- events - Invocation events that you can use to invoke the function.
+- template.yaml - A template that defines the application's AWS resources.
+
+The application uses several AWS resources, including Lambda functions and an API Gateway API. These resources are defined in the `template.yaml` file in this project. You can update the template to add AWS resources through the same deployment process that updates your application code.
+
+See the [AWS SAM developer guide](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html) for deployment, usage and an introduction to SAM specification, the SAM CLI, and serverless application concepts.
+
+## Interface
+
+You can establish a WebSocket connection to the WebSocketURI generated by the deployment. With this connection you can send requests to the lambda functions following this interface:
+
+```
+action:
+  description: The name of the requested function.
+  type: string
+cmd:
+  description: The command that should be executed.
+  type: []string
+files:
+  description: The files that will be copied before the execution.
+  type: map[string]string
+```
+
+So for example:
+```
+{
+  "action": "java11Exec",
+  "cmd": [
+    "sh",
+    "-c",
+    "javac org/example/RecursiveMath.java && java org/example/RecursiveMath"
+  ],
+  "files": {
+    "org/example/RecursiveMath.java":"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo="
+  }
+}
+```
+
+The messages sent by the function use the [WebSocket Schema](../../api/websocket.schema.json).

deploy/aws/events/event.json

@@ -0,0 +1,22 @@
+{
+  "headers": {
+    "disableOutput": "True"
+  },
+  "requestContext": {
+    "stage": "production",
+    "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef",
+    "apiId": "1234567890",
+    "connectedAt": 1641993862426,
+    "connectionId": "L1Z1Cc7iFiACEKQ=",
+    "domainName": "abcdef1234.execute-api.eu-central-1.amazonaws.com",
+    "eventType": "MESSAGE",
+    "extendedRequestId": "L1Z1CH3rliAFRhg=",
+    "messageDirection": "IN",
+    "messageId": "MUBfpelRFiACF9g=",
+    "requestTime": "12/Jan/2022:13:24:22 +0000",
+    "requestTimeEpoch": 1641993862426,
+    "routeKey": "java11Exec"
+  },
+  "body": "{\n    \"action\": \"java11Exec\",\n    \"cmd\": [\n        \"sh\",\n        \"-c\",\n        \"javac org/example/RecursiveMath.java && java org/example/RecursiveMath\"\n    ],\n    \"files\": {\n        \"org/example/RecursiveMath.java\": \"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo=\"\n    }\n}",
+  "isBase64Encoded": false
+}

deploy/aws/java11Exec/pom.xml

@@ -0,0 +1,62 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>poseidon</groupId>
+    <artifactId>java11Exec</artifactId>
+    <version>1.0</version>
+    <packaging>jar</packaging>
+    <name>A Java executor created for openHPI/Poseidon.</name>
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-core</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-apigatewaymanagementapi</artifactId>
+            <version>1.12.131</version>
+        </dependency>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-lambda-java-events</artifactId>
+          <version>3.6.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.9</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>3.2.4</version>
+          <configuration>
+          </configuration>
+          <executions>
+            <execution>
+              <phase>package</phase>
+              <goals>
+                <goal>shade</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </build>
+</project>

deploy/aws/java11Exec/src/main/java/poseidon/App.java

@@ -0,0 +1,152 @@
+package poseidon;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Scanner;
+
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApi;
+import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApiClientBuilder;
+import com.amazonaws.services.apigatewaymanagementapi.model.PostToConnectionRequest;
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
+import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+
+// AwsFunctionRequest contains the java files that needs to be executed.
+class AwsFunctionRequest {
+    String[] cmd;
+    Map<String, String> files;
+}
+
+// WebSocketMessageType are the types of messages that are being sent back over the WebSocket connection.
+enum WebSocketMessageType {
+    WebSocketOutputStdout("stdout"),
+    WebSocketOutputStderr("stderr"),
+    WebSocketOutputError("error"),
+    WebSocketExit("exit");
+
+    private final String typeName;
+
+    WebSocketMessageType(String name) {
+        this.typeName = name;
+    }
+
+    public String toString() {
+        return typeName;
+    }
+}
+
+/**
+ * Handler for requests to Lambda function.
+ * This Lambda function executes the passed command with the provided files in an isolated Java environment.
+ */
+public class App implements RequestHandler<APIGatewayV2WebSocketEvent, APIGatewayProxyResponseEvent> {
+
+    // gson helps parse the json objects.
+    private static final Gson gson = new Gson();
+
+    // gwClient is used to send messages back via the WebSocket connection.
+    private AmazonApiGatewayManagementApi gwClient;
+
+    // connectionID helps to identify the WebSocket connection that has called this function.
+    private String connectionID;
+
+    public static final String disableOutputHeaderKey = "disableOutput";
+
+    // disableOutput: If set to true, no output will be sent over the WebSocket connection.
+    private boolean disableOutput = false;
+
+    public APIGatewayProxyResponseEvent handleRequest(final APIGatewayV2WebSocketEvent input, final Context context) {
+        APIGatewayV2WebSocketEvent.RequestContext ctx = input.getRequestContext();
+        String[] domains = ctx.getDomainName().split("\\.");
+        String region = domains[domains.length-3];
+        this.gwClient = AmazonApiGatewayManagementApiClientBuilder.standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://" + ctx.getDomainName() + "/" + ctx.getStage(), region))
+                .build();
+        this.connectionID = ctx.getConnectionId();
+        this.disableOutput = input.getHeaders() != null && input.getHeaders().containsKey(disableOutputHeaderKey) && Boolean.parseBoolean(input.getHeaders().get(disableOutputHeaderKey));
+        AwsFunctionRequest execution = gson.fromJson(input.getBody(), AwsFunctionRequest.class);
+
+        try {
+            File workingDirectory = this.writeFS(execution.files);
+
+            ProcessBuilder pb = new ProcessBuilder(execution.cmd).redirectErrorStream(true);
+            pb.directory(workingDirectory);
+            Process p = pb.start();
+            InputStream stdout = p.getInputStream(), stderr = p.getErrorStream();
+            this.forwardOutput(p, stdout, stderr);
+            p.destroy();
+            return new APIGatewayProxyResponseEvent().withStatusCode(200);
+        } catch (Exception e) {
+            this.sendMessage(WebSocketMessageType.WebSocketOutputError, e.toString(), null);
+            return new APIGatewayProxyResponseEvent().withBody(e.toString()).withStatusCode(500);
+        }
+    }
+
+    // writeFS writes the files to the local filesystem.
+    private File writeFS(Map<String, String> files) throws IOException {
+        File workspace = Files.createTempDirectory("workspace").toFile();
+        for (Map.Entry<String, String> entry : files.entrySet()) {
+            File f = new File(workspace, entry.getKey());
+
+            f.getParentFile().mkdirs();
+            if (!f.getParentFile().exists()) {
+                throw new IOException("Cannot create parent directories.");
+            }
+
+            f.createNewFile();
+            if (!f.exists()) {
+                throw new IOException("Cannot create file.");
+            }
+
+            Files.write(f.toPath(), Base64.getDecoder().decode(entry.getValue()));
+        }
+        return workspace;
+    }
+
+    // forwardOutput sends the output of the process to the WebSocket connection.
+    private void forwardOutput(Process p, InputStream stdout, InputStream stderr) throws InterruptedException {
+        Thread output = new Thread(() -> scanForOutput(p, stdout, WebSocketMessageType.WebSocketOutputStdout));
+        Thread error = new Thread(() -> scanForOutput(p, stderr, WebSocketMessageType.WebSocketOutputStderr));
+        output.start();
+        error.start();
+
+        output.join();
+        error.join();
+        this.sendMessage(WebSocketMessageType.WebSocketExit, null, p.exitValue());
+    }
+
+    // scanForOutput reads the passed stream and forwards it via the WebSocket connection.
+    private void scanForOutput(Process p, InputStream stream, WebSocketMessageType type) {
+        Scanner outputScanner = new Scanner(stream);
+        while (p.isAlive() || outputScanner.hasNextLine()) {
+            this.sendMessage(type, outputScanner.nextLine(), null);
+        }
+    }
+
+    // sendMessage sends WebSocketMessage objects back to the requester of this Lambda function.
+    private void sendMessage(WebSocketMessageType type, String data, Integer exitCode) {
+        if (this.disableOutput) {
+            return;
+        }
+        JsonObject msg = new JsonObject();
+        msg.addProperty("type", type.toString());
+        if (type == WebSocketMessageType.WebSocketExit) {
+            msg.addProperty("data", exitCode);
+        } else {
+            msg.addProperty("data", data);
+        }
+
+        this.gwClient.postToConnection(new PostToConnectionRequest()
+                .withConnectionId(this.connectionID)
+                .withData(ByteBuffer.wrap(gson.toJson(msg).getBytes(StandardCharsets.UTF_8))));
+    }
+}

deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java

@@ -0,0 +1,28 @@
+package poseidon;
+
+import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
+import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class AppTest {
+  @Test
+  public void successfulResponse() {
+    App app = new App();
+    APIGatewayV2WebSocketEvent input = new APIGatewayV2WebSocketEvent();
+    APIGatewayV2WebSocketEvent.RequestContext ctx = new APIGatewayV2WebSocketEvent.RequestContext();
+    ctx.setDomainName("abcdef1234.execute-api.eu-central-1.amazonaws.com");
+    ctx.setConnectionId("myUUID");
+    input.setRequestContext(ctx);
+    Map<String, String> headers = new HashMap<>();
+    headers.put(App.disableOutputHeaderKey, "True");
+    input.setHeaders(headers);
+    input.setBody("{\n    \"action\": \"java11Exec\",\n    \"cmd\": [\n        \"sh\",\n        \"-c\",\n        \"javac org/example/RecursiveMath.java && java org/example/RecursiveMath\"\n    ],\n    \"files\": {\n        \"org/example/RecursiveMath.java\": \"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo=\"\n    }\n}");
+    APIGatewayProxyResponseEvent result = app.handleRequest(input, null);
+    assertEquals(200, result.getStatusCode().intValue());
+  }
+}

deploy/aws/template.yaml

@@ -0,0 +1,105 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: >
+  PoseidonExecutors
+
+  Execute untrusted code in AWS functions.
+
+# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
+Globals:
+  Function:
+    Timeout: 15
+
+Resources:
+  PoseidonExecWebSocket:
+    Type: AWS::ApiGatewayV2::Api
+    Properties:
+      Name: PoseidonExecWebSocket
+      ProtocolType: WEBSOCKET
+      RouteSelectionExpression: "$request.body.action"
+
+  Deployment:
+    Type: AWS::ApiGatewayV2::Deployment
+    DependsOn:
+      - java11ExecRoute
+    Properties:
+      ApiId: !Ref PoseidonExecWebSocket
+
+  Stage:
+    Type: AWS::ApiGatewayV2::Stage
+    Properties:
+      StageName: production
+      Description: Production Stage
+      DeploymentId: !Ref Deployment
+      ApiId: !Ref PoseidonExecWebSocket
+
+  java11ExecRoute: # More info about Routes: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigatewayv2-route.html
+    Type: AWS::ApiGatewayV2::Route
+    Properties:
+      ApiId: !Ref PoseidonExecWebSocket
+      RouteKey: java11Exec
+      AuthorizationType: NONE
+      OperationName: java11ExecRoute
+      Target: !Join
+        - '/'
+        - - 'integrations'
+          - !Ref java11ExecInteg
+
+  java11ExecInteg: # More info about Integrations: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
+    Type: AWS::ApiGatewayV2::Integration
+    Properties:
+      ApiId: !Ref PoseidonExecWebSocket
+      Description: Java 11 Exec Integration
+      IntegrationType: AWS_PROXY
+      IntegrationUri:
+        Fn::Sub:
+          arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${java11ExecFunction.Arn}/invocations
+
+  java11ExecFunction:
+    Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
+    Properties:
+      CodeUri: java11Exec/
+      Handler: poseidon.App::handleRequest
+      Runtime: java11
+      Architectures:
+        - arm64
+      MemorySize: 2048
+      Policies:
+        - Statement:
+            - Effect: Allow
+              Action:
+                - 'execute-api:*'
+              Resource: "*"
+            - Effect: Allow
+              Action:
+                - 'logs:CreateLogGroup'
+              Resource:
+                - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
+            - Effect: Allow
+              Action:
+                - 'logs:CreateLogStream'
+                - 'logs:PutLogEvents'
+              Resource:
+                - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${PoseidonExecWebSocket}:*'
+
+  java11ExecPermission:
+    Type: AWS::Lambda::Permission
+    DependsOn:
+      - PoseidonExecWebSocket
+    Properties:
+      Action: lambda:InvokeFunction
+      FunctionName: !Ref java11ExecFunction
+      Principal: apigateway.amazonaws.com
+
+Outputs:
+  WebSocketURI:
+    Description: "The WSS Protocol URI to connect to"
+    Value: !Join [ '', [ 'wss://', !Ref PoseidonExecWebSocket, '.execute-api.',!Ref 'AWS::Region','.amazonaws.com/',!Ref 'Stage' ] ]
+
+  java11ExecFunctionArn:
+    Description: "Java 11 Execution Lambda Function ARN"
+    Value: !GetAtt java11ExecFunction.Arn
+
+  java11ExecFunctionIamRole:
+    Description: "Implicit IAM Role created for the Java 11 Execution function"
+    Value: !GetAtt java11ExecFunctionRole.Arn