Restructure project

We previously didn't really had any structure in our project apart
from creating a new folder for each package in our project root.
Now that we have accumulated some packages, we use the well-known
Golang project layout in order to clearly communicate our intent
with packages. See https://github.com/golang-standards/project-layout
This commit is contained in:
sirkrypt0
2021-07-16 09:19:42 +02:00
parent 2f1383b743
commit 8b26ecbe5f
66 changed files with 95 additions and 95 deletions

226
pkg/dto/dto.go Normal file
View File

@ -0,0 +1,226 @@
package dto
import (
"encoding/json"
"errors"
"fmt"
"path"
"strings"
)
// RunnerRequest is the expected json structure of the request body for the ProvideRunner function.
type RunnerRequest struct {
ExecutionEnvironmentID int `json:"executionEnvironmentId"`
InactivityTimeout int `json:"inactivityTimeout"`
}
// ExecutionRequest is the expected json structure of the request body for the ExecuteCommand function.
type ExecutionRequest struct {
Command string
TimeLimit int
Environment map[string]string
}
func (er *ExecutionRequest) FullCommand() []string {
command := make([]string, 0)
command = append(command, "env", "-")
for variable, value := range er.Environment {
command = append(command, fmt.Sprintf("%s=%s", variable, value))
}
command = append(command, "sh", "-c", er.Command)
return command
}
// ExecutionEnvironmentRequest is the expected json structure of the request body
// for the create execution environment function.
type ExecutionEnvironmentRequest struct {
PrewarmingPoolSize uint `json:"prewarmingPoolSize"`
CPULimit uint `json:"cpuLimit"`
MemoryLimit uint `json:"memoryLimit"`
Image string `json:"image"`
NetworkAccess bool `json:"networkAccess"`
ExposedPorts []uint16 `json:"exposedPorts"`
}
// MappedPort contains the mapping from exposed port inside the container to the host address
// outside the container.
type MappedPort struct {
ExposedPort uint `json:"exposedPort"`
HostAddress string `json:"hostAddress"`
}
// RunnerResponse is the expected response when providing a runner.
type RunnerResponse struct {
ID string `json:"runnerId"`
MappedPorts []*MappedPort `json:"mappedPorts"`
}
// ExecutionResponse is the expected response when creating an execution for a runner.
type ExecutionResponse struct {
WebSocketURL string `json:"websocketUrl"`
}
// UpdateFileSystemRequest is the expected json structure of the request body for the update file system route.
type UpdateFileSystemRequest struct {
Delete []FilePath `json:"delete"`
Copy []File `json:"copy"`
}
// FilePath specifies the path of a file and is part of the UpdateFileSystemRequest.
type FilePath string
// File is a DTO for transmitting file contents. It is part of the UpdateFileSystemRequest.
type File struct {
Path FilePath `json:"path"`
Content []byte `json:"content"`
}
// Cleaned returns the cleaned path of the FilePath.
func (f FilePath) Cleaned() string {
return path.Clean(string(f))
}
// CleanedPath returns the cleaned path of the file.
func (f File) CleanedPath() string {
return f.Path.Cleaned()
}
// IsDirectory returns true iff the path of the File ends with a /.
func (f File) IsDirectory() bool {
return strings.HasSuffix(string(f.Path), "/")
}
// ByteContent returns the content of the File. If the File is a directory, the content will be empty.
func (f File) ByteContent() []byte {
if f.IsDirectory() {
return []byte("")
} else {
return f.Content
}
}
// WebSocketMessageType is the type for the messages from Poseidon to the client.
type WebSocketMessageType string
const (
WebSocketOutputStdout WebSocketMessageType = "stdout"
WebSocketOutputStderr WebSocketMessageType = "stderr"
WebSocketOutputError WebSocketMessageType = "error"
WebSocketMetaStart WebSocketMessageType = "start"
WebSocketMetaTimeout WebSocketMessageType = "timeout"
WebSocketExit WebSocketMessageType = "exit"
)
var (
ErrUnknownWebSocketMessageType = errors.New("unknown WebSocket message type")
ErrMissingType = errors.New("type is missing")
ErrMissingData = errors.New("data is missing")
ErrInvalidType = errors.New("invalid type")
)
// WebSocketMessage is the type for all messages send in the WebSocket to the client.
// Depending on the MessageType the Data or ExitCode might not be included in the marshaled json message.
type WebSocketMessage struct {
Type WebSocketMessageType
Data string
ExitCode uint8
}
// MarshalJSON implements the json.Marshaler interface.
// This converts the WebSocketMessage into the expected schema (see docs/websocket.schema.json).
func (m WebSocketMessage) MarshalJSON() (res []byte, err error) {
switch m.Type {
case WebSocketOutputStdout, WebSocketOutputStderr, WebSocketOutputError:
res, err = json.Marshal(struct {
MessageType WebSocketMessageType `json:"type"`
Data string `json:"data"`
}{m.Type, m.Data})
case WebSocketMetaStart, WebSocketMetaTimeout:
res, err = json.Marshal(struct {
MessageType WebSocketMessageType `json:"type"`
}{m.Type})
case WebSocketExit:
res, err = json.Marshal(struct {
MessageType WebSocketMessageType `json:"type"`
ExitCode uint8 `json:"data"`
}{m.Type, m.ExitCode})
}
if err != nil {
return nil, fmt.Errorf("error marshaling WebSocketMessage: %w", err)
} else if res == nil {
return nil, ErrUnknownWebSocketMessageType
}
return res, nil
}
// UnmarshalJSON implements the json.Unmarshaler interface.
// It is used by tests in order to ReceiveNextWebSocketMessage.
func (m *WebSocketMessage) UnmarshalJSON(rawMessage []byte) error {
messageMap := make(map[string]interface{})
err := json.Unmarshal(rawMessage, &messageMap)
if err != nil {
return fmt.Errorf("error unmarshiling raw WebSocket message: %w", err)
}
messageType, ok := messageMap["type"]
if !ok {
return ErrMissingType
}
messageTypeString, ok := messageType.(string)
if !ok {
return fmt.Errorf("value of key type must be a string: %w", ErrInvalidType)
}
switch messageType := WebSocketMessageType(messageTypeString); messageType {
case WebSocketExit:
data, ok := messageMap["data"]
if !ok {
return ErrMissingData
}
// json.Unmarshal converts any number to a float64 in the massageMap, so we must first cast it to the float.
exit, ok := data.(float64)
if !ok {
return fmt.Errorf("value of key data must be a number: %w", ErrInvalidType)
}
if exit != float64(uint8(exit)) {
return fmt.Errorf("value of key data must be uint8: %w", ErrInvalidType)
}
m.Type = messageType
m.ExitCode = uint8(exit)
case WebSocketOutputStdout, WebSocketOutputStderr, WebSocketOutputError:
data, ok := messageMap["data"]
if !ok {
return ErrMissingData
}
text, ok := data.(string)
if !ok {
return fmt.Errorf("value of key data must be a string: %w", ErrInvalidType)
}
m.Type = messageType
m.Data = text
case WebSocketMetaStart, WebSocketMetaTimeout:
m.Type = messageType
default:
return ErrUnknownWebSocketMessageType
}
return nil
}
// ClientError is the response interface if the request is not valid.
type ClientError struct {
Message string `json:"message"`
}
// InternalServerError is the response interface that is returned when an error occurs.
type InternalServerError struct {
Message string `json:"message"`
ErrorCode ErrorCode `json:"errorCode"`
}
// ErrorCode is the type for error codes expected by CodeOcean.
type ErrorCode string
const (
ErrorNomadUnreachable ErrorCode = "NOMAD_UNREACHABLE"
ErrorNomadOverload ErrorCode = "NOMAD_OVERLOAD"
ErrorNomadInternalServerError ErrorCode = "NOMAD_INTERNAL_SERVER_ERROR"
ErrorUnknown ErrorCode = "UNKNOWN"
)

83
pkg/logging/logging.go Normal file
View File

@ -0,0 +1,83 @@
package logging
import (
"bufio"
"fmt"
"github.com/sirupsen/logrus"
"net"
"net/http"
"os"
"time"
)
var log = &logrus.Logger{
Out: os.Stderr,
Formatter: &logrus.TextFormatter{
DisableColors: true,
FullTimestamp: true,
},
Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
}
func InitializeLogging(loglevel string) {
level, err := logrus.ParseLevel(loglevel)
if err != nil {
log.WithError(err).Fatal("Error parsing loglevel")
return
}
log.SetLevel(level)
}
func GetLogger(pkg string) *logrus.Entry {
return log.WithField("package", pkg)
}
// loggingResponseWriter wraps the default http.ResponseWriter and catches the status code
// that is written.
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (writer *loggingResponseWriter) WriteHeader(code int) {
writer.statusCode = code
writer.ResponseWriter.WriteHeader(code)
}
func (writer *loggingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
conn, rw, err := writer.ResponseWriter.(http.Hijacker).Hijack()
if err != nil {
return conn, nil, fmt.Errorf("hijacking connection failed: %w", err)
}
return conn, rw, nil
}
// HTTPLoggingMiddleware returns an http.Handler that logs different information about every request.
func HTTPLoggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now().UTC()
path := r.URL.Path
lrw := NewLoggingResponseWriter(w)
next.ServeHTTP(lrw, r)
latency := time.Now().UTC().Sub(start)
logEntry := log.WithFields(logrus.Fields{
"code": lrw.statusCode,
"method": r.Method,
"path": path,
"duration": latency,
"user_agent": r.UserAgent(),
})
if lrw.statusCode >= http.StatusInternalServerError {
logEntry.Warn()
} else {
logEntry.Debug()
}
})
}

View File

@ -0,0 +1,48 @@
package logging
import (
"github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
func mockHTTPStatusHandler(status int) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(status)
})
}
func TestHTTPMiddlewareWarnsWhenInternalServerError(t *testing.T) {
var hook *test.Hook
log, hook = test.NewNullLogger()
InitializeLogging(logrus.DebugLevel.String())
request, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatal(err)
}
recorder := httptest.NewRecorder()
HTTPLoggingMiddleware(mockHTTPStatusHandler(500)).ServeHTTP(recorder, request)
assert.Equal(t, 1, len(hook.Entries))
assert.Equal(t, logrus.WarnLevel, hook.LastEntry().Level)
}
func TestHTTPMiddlewareDebugsWhenStatusOK(t *testing.T) {
var hook *test.Hook
log, hook = test.NewNullLogger()
InitializeLogging(logrus.DebugLevel.String())
request, err := http.NewRequest(http.MethodGet, "/", nil)
if err != nil {
t.Fatal(err)
}
recorder := httptest.NewRecorder()
HTTPLoggingMiddleware(mockHTTPStatusHandler(200)).ServeHTTP(recorder, request)
assert.Equal(t, 1, len(hook.Entries))
assert.Equal(t, logrus.DebugLevel, hook.LastEntry().Level)
}

View File

@ -0,0 +1,10 @@
package nullreader
// NullReader is a struct that implements the io.Reader interface and returns nothing when reading
// from it.
type NullReader struct{}
func (r NullReader) Read(_ []byte) (int, error) {
// An empty select blocks forever.
select {}
}

View File

@ -0,0 +1,20 @@
package nullreader
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.hpi.de/codeocean/codemoon/poseidon/tests"
"testing"
)
func TestNullReaderDoesNotReturnImmediately(t *testing.T) {
reader := &NullReader{}
readerReturned := make(chan bool)
go func() {
p := make([]byte, 0, 5)
_, err := reader.Read(p)
require.NoError(t, err)
close(readerReturned)
}()
assert.False(t, tests.ChannelReceivesSomething(readerReturned, tests.ShortTimeout))
}