Restructure project
We previously didn't really had any structure in our project apart from creating a new folder for each package in our project root. Now that we have accumulated some packages, we use the well-known Golang project layout in order to clearly communicate our intent with packages. See https://github.com/golang-standards/project-layout
This commit is contained in:
226
pkg/dto/dto.go
Normal file
226
pkg/dto/dto.go
Normal file
@ -0,0 +1,226 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RunnerRequest is the expected json structure of the request body for the ProvideRunner function.
|
||||
type RunnerRequest struct {
|
||||
ExecutionEnvironmentID int `json:"executionEnvironmentId"`
|
||||
InactivityTimeout int `json:"inactivityTimeout"`
|
||||
}
|
||||
|
||||
// ExecutionRequest is the expected json structure of the request body for the ExecuteCommand function.
|
||||
type ExecutionRequest struct {
|
||||
Command string
|
||||
TimeLimit int
|
||||
Environment map[string]string
|
||||
}
|
||||
|
||||
func (er *ExecutionRequest) FullCommand() []string {
|
||||
command := make([]string, 0)
|
||||
command = append(command, "env", "-")
|
||||
for variable, value := range er.Environment {
|
||||
command = append(command, fmt.Sprintf("%s=%s", variable, value))
|
||||
}
|
||||
command = append(command, "sh", "-c", er.Command)
|
||||
return command
|
||||
}
|
||||
|
||||
// ExecutionEnvironmentRequest is the expected json structure of the request body
|
||||
// for the create execution environment function.
|
||||
type ExecutionEnvironmentRequest struct {
|
||||
PrewarmingPoolSize uint `json:"prewarmingPoolSize"`
|
||||
CPULimit uint `json:"cpuLimit"`
|
||||
MemoryLimit uint `json:"memoryLimit"`
|
||||
Image string `json:"image"`
|
||||
NetworkAccess bool `json:"networkAccess"`
|
||||
ExposedPorts []uint16 `json:"exposedPorts"`
|
||||
}
|
||||
|
||||
// MappedPort contains the mapping from exposed port inside the container to the host address
|
||||
// outside the container.
|
||||
type MappedPort struct {
|
||||
ExposedPort uint `json:"exposedPort"`
|
||||
HostAddress string `json:"hostAddress"`
|
||||
}
|
||||
|
||||
// RunnerResponse is the expected response when providing a runner.
|
||||
type RunnerResponse struct {
|
||||
ID string `json:"runnerId"`
|
||||
MappedPorts []*MappedPort `json:"mappedPorts"`
|
||||
}
|
||||
|
||||
// ExecutionResponse is the expected response when creating an execution for a runner.
|
||||
type ExecutionResponse struct {
|
||||
WebSocketURL string `json:"websocketUrl"`
|
||||
}
|
||||
|
||||
// UpdateFileSystemRequest is the expected json structure of the request body for the update file system route.
|
||||
type UpdateFileSystemRequest struct {
|
||||
Delete []FilePath `json:"delete"`
|
||||
Copy []File `json:"copy"`
|
||||
}
|
||||
|
||||
// FilePath specifies the path of a file and is part of the UpdateFileSystemRequest.
|
||||
type FilePath string
|
||||
|
||||
// File is a DTO for transmitting file contents. It is part of the UpdateFileSystemRequest.
|
||||
type File struct {
|
||||
Path FilePath `json:"path"`
|
||||
Content []byte `json:"content"`
|
||||
}
|
||||
|
||||
// Cleaned returns the cleaned path of the FilePath.
|
||||
func (f FilePath) Cleaned() string {
|
||||
return path.Clean(string(f))
|
||||
}
|
||||
|
||||
// CleanedPath returns the cleaned path of the file.
|
||||
func (f File) CleanedPath() string {
|
||||
return f.Path.Cleaned()
|
||||
}
|
||||
|
||||
// IsDirectory returns true iff the path of the File ends with a /.
|
||||
func (f File) IsDirectory() bool {
|
||||
return strings.HasSuffix(string(f.Path), "/")
|
||||
}
|
||||
|
||||
// ByteContent returns the content of the File. If the File is a directory, the content will be empty.
|
||||
func (f File) ByteContent() []byte {
|
||||
if f.IsDirectory() {
|
||||
return []byte("")
|
||||
} else {
|
||||
return f.Content
|
||||
}
|
||||
}
|
||||
|
||||
// WebSocketMessageType is the type for the messages from Poseidon to the client.
|
||||
type WebSocketMessageType string
|
||||
|
||||
const (
|
||||
WebSocketOutputStdout WebSocketMessageType = "stdout"
|
||||
WebSocketOutputStderr WebSocketMessageType = "stderr"
|
||||
WebSocketOutputError WebSocketMessageType = "error"
|
||||
WebSocketMetaStart WebSocketMessageType = "start"
|
||||
WebSocketMetaTimeout WebSocketMessageType = "timeout"
|
||||
WebSocketExit WebSocketMessageType = "exit"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnknownWebSocketMessageType = errors.New("unknown WebSocket message type")
|
||||
ErrMissingType = errors.New("type is missing")
|
||||
ErrMissingData = errors.New("data is missing")
|
||||
ErrInvalidType = errors.New("invalid type")
|
||||
)
|
||||
|
||||
// WebSocketMessage is the type for all messages send in the WebSocket to the client.
|
||||
// Depending on the MessageType the Data or ExitCode might not be included in the marshaled json message.
|
||||
type WebSocketMessage struct {
|
||||
Type WebSocketMessageType
|
||||
Data string
|
||||
ExitCode uint8
|
||||
}
|
||||
|
||||
// MarshalJSON implements the json.Marshaler interface.
|
||||
// This converts the WebSocketMessage into the expected schema (see docs/websocket.schema.json).
|
||||
func (m WebSocketMessage) MarshalJSON() (res []byte, err error) {
|
||||
switch m.Type {
|
||||
case WebSocketOutputStdout, WebSocketOutputStderr, WebSocketOutputError:
|
||||
res, err = json.Marshal(struct {
|
||||
MessageType WebSocketMessageType `json:"type"`
|
||||
Data string `json:"data"`
|
||||
}{m.Type, m.Data})
|
||||
case WebSocketMetaStart, WebSocketMetaTimeout:
|
||||
res, err = json.Marshal(struct {
|
||||
MessageType WebSocketMessageType `json:"type"`
|
||||
}{m.Type})
|
||||
case WebSocketExit:
|
||||
res, err = json.Marshal(struct {
|
||||
MessageType WebSocketMessageType `json:"type"`
|
||||
ExitCode uint8 `json:"data"`
|
||||
}{m.Type, m.ExitCode})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error marshaling WebSocketMessage: %w", err)
|
||||
} else if res == nil {
|
||||
return nil, ErrUnknownWebSocketMessageType
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements the json.Unmarshaler interface.
|
||||
// It is used by tests in order to ReceiveNextWebSocketMessage.
|
||||
func (m *WebSocketMessage) UnmarshalJSON(rawMessage []byte) error {
|
||||
messageMap := make(map[string]interface{})
|
||||
err := json.Unmarshal(rawMessage, &messageMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error unmarshiling raw WebSocket message: %w", err)
|
||||
}
|
||||
messageType, ok := messageMap["type"]
|
||||
if !ok {
|
||||
return ErrMissingType
|
||||
}
|
||||
messageTypeString, ok := messageType.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("value of key type must be a string: %w", ErrInvalidType)
|
||||
}
|
||||
switch messageType := WebSocketMessageType(messageTypeString); messageType {
|
||||
case WebSocketExit:
|
||||
data, ok := messageMap["data"]
|
||||
if !ok {
|
||||
return ErrMissingData
|
||||
}
|
||||
// json.Unmarshal converts any number to a float64 in the massageMap, so we must first cast it to the float.
|
||||
exit, ok := data.(float64)
|
||||
if !ok {
|
||||
return fmt.Errorf("value of key data must be a number: %w", ErrInvalidType)
|
||||
}
|
||||
if exit != float64(uint8(exit)) {
|
||||
return fmt.Errorf("value of key data must be uint8: %w", ErrInvalidType)
|
||||
}
|
||||
m.Type = messageType
|
||||
m.ExitCode = uint8(exit)
|
||||
case WebSocketOutputStdout, WebSocketOutputStderr, WebSocketOutputError:
|
||||
data, ok := messageMap["data"]
|
||||
if !ok {
|
||||
return ErrMissingData
|
||||
}
|
||||
text, ok := data.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("value of key data must be a string: %w", ErrInvalidType)
|
||||
}
|
||||
m.Type = messageType
|
||||
m.Data = text
|
||||
case WebSocketMetaStart, WebSocketMetaTimeout:
|
||||
m.Type = messageType
|
||||
default:
|
||||
return ErrUnknownWebSocketMessageType
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClientError is the response interface if the request is not valid.
|
||||
type ClientError struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// InternalServerError is the response interface that is returned when an error occurs.
|
||||
type InternalServerError struct {
|
||||
Message string `json:"message"`
|
||||
ErrorCode ErrorCode `json:"errorCode"`
|
||||
}
|
||||
|
||||
// ErrorCode is the type for error codes expected by CodeOcean.
|
||||
type ErrorCode string
|
||||
|
||||
const (
|
||||
ErrorNomadUnreachable ErrorCode = "NOMAD_UNREACHABLE"
|
||||
ErrorNomadOverload ErrorCode = "NOMAD_OVERLOAD"
|
||||
ErrorNomadInternalServerError ErrorCode = "NOMAD_INTERNAL_SERVER_ERROR"
|
||||
ErrorUnknown ErrorCode = "UNKNOWN"
|
||||
)
|
83
pkg/logging/logging.go
Normal file
83
pkg/logging/logging.go
Normal file
@ -0,0 +1,83 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var log = &logrus.Logger{
|
||||
Out: os.Stderr,
|
||||
Formatter: &logrus.TextFormatter{
|
||||
DisableColors: true,
|
||||
FullTimestamp: true,
|
||||
},
|
||||
Hooks: make(logrus.LevelHooks),
|
||||
Level: logrus.InfoLevel,
|
||||
}
|
||||
|
||||
func InitializeLogging(loglevel string) {
|
||||
level, err := logrus.ParseLevel(loglevel)
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("Error parsing loglevel")
|
||||
return
|
||||
}
|
||||
log.SetLevel(level)
|
||||
}
|
||||
|
||||
func GetLogger(pkg string) *logrus.Entry {
|
||||
return log.WithField("package", pkg)
|
||||
}
|
||||
|
||||
// loggingResponseWriter wraps the default http.ResponseWriter and catches the status code
|
||||
// that is written.
|
||||
type loggingResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
statusCode int
|
||||
}
|
||||
|
||||
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
|
||||
return &loggingResponseWriter{w, http.StatusOK}
|
||||
}
|
||||
|
||||
func (writer *loggingResponseWriter) WriteHeader(code int) {
|
||||
writer.statusCode = code
|
||||
writer.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (writer *loggingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
conn, rw, err := writer.ResponseWriter.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return conn, nil, fmt.Errorf("hijacking connection failed: %w", err)
|
||||
}
|
||||
return conn, rw, nil
|
||||
}
|
||||
|
||||
// HTTPLoggingMiddleware returns an http.Handler that logs different information about every request.
|
||||
func HTTPLoggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now().UTC()
|
||||
path := r.URL.Path
|
||||
|
||||
lrw := NewLoggingResponseWriter(w)
|
||||
next.ServeHTTP(lrw, r)
|
||||
|
||||
latency := time.Now().UTC().Sub(start)
|
||||
logEntry := log.WithFields(logrus.Fields{
|
||||
"code": lrw.statusCode,
|
||||
"method": r.Method,
|
||||
"path": path,
|
||||
"duration": latency,
|
||||
"user_agent": r.UserAgent(),
|
||||
})
|
||||
if lrw.statusCode >= http.StatusInternalServerError {
|
||||
logEntry.Warn()
|
||||
} else {
|
||||
logEntry.Debug()
|
||||
}
|
||||
})
|
||||
}
|
48
pkg/logging/logging_test.go
Normal file
48
pkg/logging/logging_test.go
Normal file
@ -0,0 +1,48 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mockHTTPStatusHandler(status int) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(status)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHTTPMiddlewareWarnsWhenInternalServerError(t *testing.T) {
|
||||
var hook *test.Hook
|
||||
log, hook = test.NewNullLogger()
|
||||
InitializeLogging(logrus.DebugLevel.String())
|
||||
|
||||
request, err := http.NewRequest(http.MethodGet, "/", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
HTTPLoggingMiddleware(mockHTTPStatusHandler(500)).ServeHTTP(recorder, request)
|
||||
|
||||
assert.Equal(t, 1, len(hook.Entries))
|
||||
assert.Equal(t, logrus.WarnLevel, hook.LastEntry().Level)
|
||||
}
|
||||
|
||||
func TestHTTPMiddlewareDebugsWhenStatusOK(t *testing.T) {
|
||||
var hook *test.Hook
|
||||
log, hook = test.NewNullLogger()
|
||||
InitializeLogging(logrus.DebugLevel.String())
|
||||
|
||||
request, err := http.NewRequest(http.MethodGet, "/", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
HTTPLoggingMiddleware(mockHTTPStatusHandler(200)).ServeHTTP(recorder, request)
|
||||
|
||||
assert.Equal(t, 1, len(hook.Entries))
|
||||
assert.Equal(t, logrus.DebugLevel, hook.LastEntry().Level)
|
||||
}
|
10
pkg/nullreader/nullreader.go
Normal file
10
pkg/nullreader/nullreader.go
Normal file
@ -0,0 +1,10 @@
|
||||
package nullreader
|
||||
|
||||
// NullReader is a struct that implements the io.Reader interface and returns nothing when reading
|
||||
// from it.
|
||||
type NullReader struct{}
|
||||
|
||||
func (r NullReader) Read(_ []byte) (int, error) {
|
||||
// An empty select blocks forever.
|
||||
select {}
|
||||
}
|
20
pkg/nullreader/nullreader_test.go
Normal file
20
pkg/nullreader/nullreader_test.go
Normal file
@ -0,0 +1,20 @@
|
||||
package nullreader
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNullReaderDoesNotReturnImmediately(t *testing.T) {
|
||||
reader := &NullReader{}
|
||||
readerReturned := make(chan bool)
|
||||
go func() {
|
||||
p := make([]byte, 0, 5)
|
||||
_, err := reader.Read(p)
|
||||
require.NoError(t, err)
|
||||
close(readerReturned)
|
||||
}()
|
||||
assert.False(t, tests.ChannelReceivesSomething(readerReturned, tests.ShortTimeout))
|
||||
}
|
Reference in New Issue
Block a user