diff --git a/deploy/aws/.gitignore b/deploy/aws/.gitignore new file mode 100644 index 0000000..c8d0c0b --- /dev/null +++ b/deploy/aws/.gitignore @@ -0,0 +1 @@ +java11Exec/target/ diff --git a/deploy/aws/README.md b/deploy/aws/README.md new file mode 100644 index 0000000..3949d0e --- /dev/null +++ b/deploy/aws/README.md @@ -0,0 +1,44 @@ +# Poseidon AWS Executors + +This project contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. It includes the following functions. + +- java11ExecFunction - Code for the application's Lambda function. It can execute Java files with JDK 11. +- events - Invocation events that you can use to invoke the function. +- template.yaml - A template that defines the application's AWS resources. + +The application uses several AWS resources, including Lambda functions and an API Gateway API. These resources are defined in the `template.yaml` file in this project. You can update the template to add AWS resources through the same deployment process that updates your application code. + +See the [AWS SAM developer guide](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html) for deployment, usage and an introduction to SAM specification, the SAM CLI, and serverless application concepts. + +## Interface + +You can establish a WebSocket connection to the WebSocketURI generated by the deployment. With this connection you can send requests to the lambda functions following this interface: + +``` +action: + description: The name of the requested function. + type: string +cmd: + description: The command that should be executed. + type: []string +files: + description: The files that will be copied before the execution. + type: map[string]string +``` + +So for example: +``` +{ + "action": "java11Exec", + "cmd": [ + "sh", + "-c", + "javac org/example/RecursiveMath.java && java org/example/RecursiveMath" + ], + "files": { + "org/example/RecursiveMath.java":"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo=" + } +} +``` + +The messages sent by the function use the [WebSocket Schema](../../api/websocket.schema.json). diff --git a/deploy/aws/events/event.json b/deploy/aws/events/event.json new file mode 100644 index 0000000..767f65b --- /dev/null +++ b/deploy/aws/events/event.json @@ -0,0 +1,22 @@ +{ + "headers": { + "disableOutput": "True" + }, + "requestContext": { + "stage": "production", + "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", + "apiId": "1234567890", + "connectedAt": 1641993862426, + "connectionId": "L1Z1Cc7iFiACEKQ=", + "domainName": "abcdef1234.execute-api.eu-central-1.amazonaws.com", + "eventType": "MESSAGE", + "extendedRequestId": "L1Z1CH3rliAFRhg=", + "messageDirection": "IN", + "messageId": "MUBfpelRFiACF9g=", + "requestTime": "12/Jan/2022:13:24:22 +0000", + "requestTimeEpoch": 1641993862426, + "routeKey": "java11Exec" + }, + "body": "{\n \"action\": \"java11Exec\",\n \"cmd\": [\n \"sh\",\n \"-c\",\n \"javac org/example/RecursiveMath.java && java org/example/RecursiveMath\"\n ],\n \"files\": {\n \"org/example/RecursiveMath.java\": \"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo=\"\n }\n}", + "isBase64Encoded": false +} diff --git a/deploy/aws/java11Exec/pom.xml b/deploy/aws/java11Exec/pom.xml new file mode 100644 index 0000000..5fbbb37 --- /dev/null +++ b/deploy/aws/java11Exec/pom.xml @@ -0,0 +1,62 @@ + + 4.0.0 + poseidon + java11Exec + 1.0 + jar + A Java executor created for openHPI/Poseidon. + + 11 + 11 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-java-sdk-apigatewaymanagementapi + 1.12.131 + + + com.amazonaws + aws-lambda-java-events + 3.6.0 + + + com.google.code.gson + gson + 2.8.9 + + + junit + junit + 4.13.1 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/deploy/aws/java11Exec/src/main/java/poseidon/App.java b/deploy/aws/java11Exec/src/main/java/poseidon/App.java new file mode 100644 index 0000000..e916167 --- /dev/null +++ b/deploy/aws/java11Exec/src/main/java/poseidon/App.java @@ -0,0 +1,152 @@ +package poseidon; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Base64; +import java.util.Map; +import java.util.Scanner; + +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApi; +import com.amazonaws.services.apigatewaymanagementapi.AmazonApiGatewayManagementApiClientBuilder; +import com.amazonaws.services.apigatewaymanagementapi.model.PostToConnectionRequest; +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent; +import com.google.gson.Gson; +import com.google.gson.JsonObject; + +// AwsFunctionRequest contains the java files that needs to be executed. +class AwsFunctionRequest { + String[] cmd; + Map files; +} + +// WebSocketMessageType are the types of messages that are being sent back over the WebSocket connection. +enum WebSocketMessageType { + WebSocketOutputStdout("stdout"), + WebSocketOutputStderr("stderr"), + WebSocketOutputError("error"), + WebSocketExit("exit"); + + private final String typeName; + + WebSocketMessageType(String name) { + this.typeName = name; + } + + public String toString() { + return typeName; + } +} + +/** + * Handler for requests to Lambda function. + * This Lambda function executes the passed command with the provided files in an isolated Java environment. + */ +public class App implements RequestHandler { + + // gson helps parse the json objects. + private static final Gson gson = new Gson(); + + // gwClient is used to send messages back via the WebSocket connection. + private AmazonApiGatewayManagementApi gwClient; + + // connectionID helps to identify the WebSocket connection that has called this function. + private String connectionID; + + public static final String disableOutputHeaderKey = "disableOutput"; + + // disableOutput: If set to true, no output will be sent over the WebSocket connection. + private boolean disableOutput = false; + + public APIGatewayProxyResponseEvent handleRequest(final APIGatewayV2WebSocketEvent input, final Context context) { + APIGatewayV2WebSocketEvent.RequestContext ctx = input.getRequestContext(); + String[] domains = ctx.getDomainName().split("\\."); + String region = domains[domains.length-3]; + this.gwClient = AmazonApiGatewayManagementApiClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://" + ctx.getDomainName() + "/" + ctx.getStage(), region)) + .build(); + this.connectionID = ctx.getConnectionId(); + this.disableOutput = input.getHeaders() != null && input.getHeaders().containsKey(disableOutputHeaderKey) && Boolean.parseBoolean(input.getHeaders().get(disableOutputHeaderKey)); + AwsFunctionRequest execution = gson.fromJson(input.getBody(), AwsFunctionRequest.class); + + try { + File workingDirectory = this.writeFS(execution.files); + + ProcessBuilder pb = new ProcessBuilder(execution.cmd).redirectErrorStream(true); + pb.directory(workingDirectory); + Process p = pb.start(); + InputStream stdout = p.getInputStream(), stderr = p.getErrorStream(); + this.forwardOutput(p, stdout, stderr); + p.destroy(); + return new APIGatewayProxyResponseEvent().withStatusCode(200); + } catch (Exception e) { + this.sendMessage(WebSocketMessageType.WebSocketOutputError, e.toString(), null); + return new APIGatewayProxyResponseEvent().withBody(e.toString()).withStatusCode(500); + } + } + + // writeFS writes the files to the local filesystem. + private File writeFS(Map files) throws IOException { + File workspace = Files.createTempDirectory("workspace").toFile(); + for (Map.Entry entry : files.entrySet()) { + File f = new File(workspace, entry.getKey()); + + f.getParentFile().mkdirs(); + if (!f.getParentFile().exists()) { + throw new IOException("Cannot create parent directories."); + } + + f.createNewFile(); + if (!f.exists()) { + throw new IOException("Cannot create file."); + } + + Files.write(f.toPath(), Base64.getDecoder().decode(entry.getValue())); + } + return workspace; + } + + // forwardOutput sends the output of the process to the WebSocket connection. + private void forwardOutput(Process p, InputStream stdout, InputStream stderr) throws InterruptedException { + Thread output = new Thread(() -> scanForOutput(p, stdout, WebSocketMessageType.WebSocketOutputStdout)); + Thread error = new Thread(() -> scanForOutput(p, stderr, WebSocketMessageType.WebSocketOutputStderr)); + output.start(); + error.start(); + + output.join(); + error.join(); + this.sendMessage(WebSocketMessageType.WebSocketExit, null, p.exitValue()); + } + + // scanForOutput reads the passed stream and forwards it via the WebSocket connection. + private void scanForOutput(Process p, InputStream stream, WebSocketMessageType type) { + Scanner outputScanner = new Scanner(stream); + while (p.isAlive() || outputScanner.hasNextLine()) { + this.sendMessage(type, outputScanner.nextLine(), null); + } + } + + // sendMessage sends WebSocketMessage objects back to the requester of this Lambda function. + private void sendMessage(WebSocketMessageType type, String data, Integer exitCode) { + if (this.disableOutput) { + return; + } + JsonObject msg = new JsonObject(); + msg.addProperty("type", type.toString()); + if (type == WebSocketMessageType.WebSocketExit) { + msg.addProperty("data", exitCode); + } else { + msg.addProperty("data", data); + } + + this.gwClient.postToConnection(new PostToConnectionRequest() + .withConnectionId(this.connectionID) + .withData(ByteBuffer.wrap(gson.toJson(msg).getBytes(StandardCharsets.UTF_8)))); + } +} diff --git a/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java b/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java new file mode 100644 index 0000000..1e41e98 --- /dev/null +++ b/deploy/aws/java11Exec/src/test/java/poseidon/AppTest.java @@ -0,0 +1,28 @@ +package poseidon; + +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayV2WebSocketEvent; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class AppTest { + @Test + public void successfulResponse() { + App app = new App(); + APIGatewayV2WebSocketEvent input = new APIGatewayV2WebSocketEvent(); + APIGatewayV2WebSocketEvent.RequestContext ctx = new APIGatewayV2WebSocketEvent.RequestContext(); + ctx.setDomainName("abcdef1234.execute-api.eu-central-1.amazonaws.com"); + ctx.setConnectionId("myUUID"); + input.setRequestContext(ctx); + Map headers = new HashMap<>(); + headers.put(App.disableOutputHeaderKey, "True"); + input.setHeaders(headers); + input.setBody("{\n \"action\": \"java11Exec\",\n \"cmd\": [\n \"sh\",\n \"-c\",\n \"javac org/example/RecursiveMath.java && java org/example/RecursiveMath\"\n ],\n \"files\": {\n \"org/example/RecursiveMath.java\": \"cGFja2FnZSBvcmcuZXhhbXBsZTsKCnB1YmxpYyBjbGFzcyBSZWN1cnNpdmVNYXRoIHsKCiAgICBwdWJsaWMgc3RhdGljIHZvaWQgbWFpbihTdHJpbmdbXSBhcmdzKSB7CiAgICAgICAgU3lzdGVtLm91dC5wcmludGxuKCJNZWluIFRleHQiKTsKICAgIH0KCiAgICBwdWJsaWMgc3RhdGljIGRvdWJsZSBwb3dlcihpbnQgYmFzZSwgaW50IGV4cG9uZW50KSB7CiAgICAgICAgcmV0dXJuIDQyOwogICAgfQp9Cgo=\"\n }\n}"); + APIGatewayProxyResponseEvent result = app.handleRequest(input, null); + assertEquals(200, result.getStatusCode().intValue()); + } +} diff --git a/deploy/aws/template.yaml b/deploy/aws/template.yaml new file mode 100644 index 0000000..5e7c344 --- /dev/null +++ b/deploy/aws/template.yaml @@ -0,0 +1,105 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + PoseidonExecutors + + Execute untrusted code in AWS functions. + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 15 + +Resources: + PoseidonExecWebSocket: + Type: AWS::ApiGatewayV2::Api + Properties: + Name: PoseidonExecWebSocket + ProtocolType: WEBSOCKET + RouteSelectionExpression: "$request.body.action" + + Deployment: + Type: AWS::ApiGatewayV2::Deployment + DependsOn: + - java11ExecRoute + Properties: + ApiId: !Ref PoseidonExecWebSocket + + Stage: + Type: AWS::ApiGatewayV2::Stage + Properties: + StageName: production + Description: Production Stage + DeploymentId: !Ref Deployment + ApiId: !Ref PoseidonExecWebSocket + + java11ExecRoute: # More info about Routes: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigatewayv2-route.html + Type: AWS::ApiGatewayV2::Route + Properties: + ApiId: !Ref PoseidonExecWebSocket + RouteKey: java11Exec + AuthorizationType: NONE + OperationName: java11ExecRoute + Target: !Join + - '/' + - - 'integrations' + - !Ref java11ExecInteg + + java11ExecInteg: # More info about Integrations: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html + Type: AWS::ApiGatewayV2::Integration + Properties: + ApiId: !Ref PoseidonExecWebSocket + Description: Java 11 Exec Integration + IntegrationType: AWS_PROXY + IntegrationUri: + Fn::Sub: + arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${java11ExecFunction.Arn}/invocations + + java11ExecFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + CodeUri: java11Exec/ + Handler: poseidon.App::handleRequest + Runtime: java11 + Architectures: + - arm64 + MemorySize: 2048 + Policies: + - Statement: + - Effect: Allow + Action: + - 'execute-api:*' + Resource: "*" + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + Resource: + - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*' + - Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: + - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${PoseidonExecWebSocket}:*' + + java11ExecPermission: + Type: AWS::Lambda::Permission + DependsOn: + - PoseidonExecWebSocket + Properties: + Action: lambda:InvokeFunction + FunctionName: !Ref java11ExecFunction + Principal: apigateway.amazonaws.com + +Outputs: + WebSocketURI: + Description: "The WSS Protocol URI to connect to" + Value: !Join [ '', [ 'wss://', !Ref PoseidonExecWebSocket, '.execute-api.',!Ref 'AWS::Region','.amazonaws.com/',!Ref 'Stage' ] ] + + java11ExecFunctionArn: + Description: "Java 11 Execution Lambda Function ARN" + Value: !GetAtt java11ExecFunction.Arn + + java11ExecFunctionIamRole: + Description: "Implicit IAM Role created for the Java 11 Execution function" + Value: !GetAtt java11ExecFunctionRole.Arn