Fix Runner DNS resolution

by adding public nameservers to the CNI secure bridge configuration.
This commit is contained in:
Maximilian Paß
2024-03-28 14:15:47 +01:00
parent 39d25d2223
commit 9deee186a7
4 changed files with 28 additions and 16 deletions

View File

@ -65,6 +65,14 @@ nomad:
# namespace: poseidon # namespace: poseidon
# Prefer local Docker images over pulling them from a registry. Images with the `latest` tag will always be force pulled by Nomad regardless of this configuration. # Prefer local Docker images over pulling them from a registry. Images with the `latest` tag will always be force pulled by Nomad regardless of this configuration.
disableforcepull: true disableforcepull: true
# Network configuration for network-enabled runners. See https://developer.hashicorp.com/nomad/docs/job-specification/network.
network:
# Available Modes: "none", "bridge", "host", "cni/*".
# "none": Even the network-enabled runners will be isolated.
# "bridge": Isolated network namespace with bridged interface. Linux-only.
# "host": Using the host network namespace. Less-secure.
# "cni/*": Configure an isolated network namespace using CNI. For example, this can be a more secured bridge network.
mode: "cni/secure-bridge"
aws: aws:
# Specifies whether AWS should be used as executor. # Specifies whether AWS should be used as executor.

View File

@ -8,6 +8,7 @@ import (
"flag" "flag"
"fmt" "fmt"
"github.com/getsentry/sentry-go" "github.com/getsentry/sentry-go"
nomadApi "github.com/hashicorp/nomad/api"
"github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/pkg/dto"
"github.com/openHPI/poseidon/pkg/logging" "github.com/openHPI/poseidon/pkg/logging"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
@ -54,6 +55,10 @@ var (
}, },
Namespace: "default", Namespace: "default",
DisableForcePull: false, DisableForcePull: false,
Network: nomadApi.NetworkResource{
Mode: "bridge",
DNS: nil,
},
}, },
AWS: AWS{ AWS: AWS{
Enabled: false, Enabled: false,
@ -120,6 +125,7 @@ type Nomad struct {
TLS TLS TLS TLS
Namespace string Namespace string
DisableForcePull bool DisableForcePull bool
Network nomadApi.NetworkResource
} }
// URL returns the URL for the configured Nomad cluster. // URL returns the URL for the configured Nomad cluster.

View File

@ -8,6 +8,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
nomadApi "github.com/hashicorp/nomad/api" nomadApi "github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/jobspec2" "github.com/hashicorp/nomad/jobspec2"
"github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/internal/nomad"
"github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/pkg/dto"
@ -170,16 +171,7 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) {
} }
if allow { if allow {
var networkResource *nomadApi.NetworkResource networkResource := config.Config.Nomad.Network
if len(defaultTaskGroup.Networks) == 0 {
networkResource = &nomadApi.NetworkResource{}
defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource}
} else {
networkResource = defaultTaskGroup.Networks[0]
}
// Prefer "bridge" network over "host" to have an isolated network namespace with bridged interface
// instead of joining the host network namespace.
networkResource.Mode = "cni/secure-bridge"
for _, portNumber := range exposedPorts { for _, portNumber := range exposedPorts {
port := nomadApi.Port{ port := nomadApi.Port{
Label: strconv.FormatUint(uint64(portNumber), portNumberBase), Label: strconv.FormatUint(uint64(portNumber), portNumberBase),
@ -187,6 +179,11 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) {
} }
networkResource.DynamicPorts = append(networkResource.DynamicPorts, port) networkResource.DynamicPorts = append(networkResource.DynamicPorts, port)
} }
if len(defaultTaskGroup.Networks) == 0 {
defaultTaskGroup.Networks = []*nomadApi.NetworkResource{&networkResource}
} else {
defaultTaskGroup.Networks[0] = &networkResource
}
// Explicitly set mode to override existing settings when updating job from without to with network. // Explicitly set mode to override existing settings when updating job from without to with network.
// Don't use bridge as it collides with the bridge mode above. This results in Docker using 'bridge' // Don't use bridge as it collides with the bridge mode above. This results in Docker using 'bridge'
@ -332,12 +329,12 @@ func (n *NomadEnvironment) SetConfigFrom(environment runner.ExecutionEnvironment
} }
func parseJob(jobHCL string) (*nomadApi.Job, error) { func parseJob(jobHCL string) (*nomadApi.Job, error) {
config := jobspec2.ParseConfig{ jobConfig := jobspec2.ParseConfig{
Body: []byte(jobHCL), Body: []byte(jobHCL),
AllowFS: false, AllowFS: false,
Strict: true, Strict: true,
} }
job, err := jobspec2.ParseWithConfig(&config) job, err := jobspec2.ParseWithConfig(&jobConfig)
if err != nil { if err != nil {
return job, fmt.Errorf("couldn't parse job HCL: %w", err) return job, fmt.Errorf("couldn't parse job HCL: %w", err)
} }

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
nomadApi "github.com/hashicorp/nomad/api" nomadApi "github.com/hashicorp/nomad/api"
"github.com/openHPI/poseidon/internal/config"
"github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/internal/nomad"
"github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/internal/runner"
"github.com/openHPI/poseidon/pkg/storage" "github.com/openHPI/poseidon/pkg/storage"
@ -32,14 +33,14 @@ func (s *MainTestSuite) TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkEx
defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job)
environment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil} environment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil}
networkResource := &nomadApi.NetworkResource{Mode: "cni/secure-bridge"} networkResource := config.Config.Nomad.Network
defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource} defaultTaskGroup.Networks = []*nomadApi.NetworkResource{&networkResource}
if s.Equal(1, len(defaultTaskGroup.Networks)) { if s.Equal(1, len(defaultTaskGroup.Networks)) {
environment.SetNetworkAccess(true, []uint16{}) environment.SetNetworkAccess(true, []uint16{})
s.Equal(1, len(defaultTaskGroup.Networks)) s.Equal(1, len(defaultTaskGroup.Networks))
s.Equal(networkResource, defaultTaskGroup.Networks[0]) s.Equal(&networkResource, defaultTaskGroup.Networks[0])
} }
} }
@ -80,7 +81,7 @@ func (s *MainTestSuite) TestConfigureNetworkSetsCorrectValues() {
s.Require().Equal(1, len(testTaskGroup.Networks)) s.Require().Equal(1, len(testTaskGroup.Networks))
networkResource := testTaskGroup.Networks[0] networkResource := testTaskGroup.Networks[0]
s.Equal("cni/secure-bridge", networkResource.Mode) s.Equal(config.Config.Nomad.Network.Mode, networkResource.Mode)
s.Require().Equal(len(ports), len(networkResource.DynamicPorts)) s.Require().Equal(len(ports), len(networkResource.DynamicPorts))
assertExpectedPorts(s.T(), ports, networkResource) assertExpectedPorts(s.T(), ports, networkResource)