diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83b2916..4d2b808 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: env: GO_VERSION: 1.19 NOMAD_VERSION: 1.3.5 + CNI_VERSION: 1.1.1 jobs: compile: @@ -141,10 +142,12 @@ jobs: # More details: https://github.com/golang/go/blob/d60ad1e068263832c711aaf17b6ccb1b7f71b000/src/cmd/go/internal/cache/cache.go#L255-L326 run: date +%s > ~/.cache/go-build/trim.txt continue-on-error: true - - name: Cache Nomad binary + - name: Cache Nomad and CNI binaries uses: actions/cache@v2 with: - path: ${{ github.workspace }}/nomad + path: | + ${{ github.workspace }}/nomad + ${{ github.workspace }}/cni/bin key: ${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }} restore-keys: | ${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }} @@ -155,8 +158,18 @@ jobs: wget -q "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_SHA256SUMS" grep "nomad_${NOMAD_VERSION}_linux_amd64.zip" nomad_${NOMAD_VERSION}_SHA256SUMS | sha256sum -c - unzip nomad_${NOMAD_VERSION}_linux_amd64.zip + - name: Download CNI binaries + run: | + if [[ -f ./cni/bin ]]; then exit 0; fi + wget -q "https://github.com/containernetworking/plugins/releases/download/v${CNI_VERSION}/cni-plugins-linux-amd64-v${CNI_VERSION}.tgz" + wget -q "https://github.com/containernetworking/plugins/releases/download/v${CNI_VERSION}/cni-plugins-linux-amd64-v${CNI_VERSION}.tgz.sha256" + grep "cni-plugins-linux-amd64-v${CNI_VERSION}.tgz" cni-plugins-linux-amd64-v${CNI_VERSION}.tgz.sha256 | sha256sum -c - + mkdir -p ./cni/bin + tar zxvf cni-plugins-linux-amd64-v${CNI_VERSION}.tgz -C ./cni/bin - name: Set Nomad Config - run: echo "server { default_scheduler_config { memory_oversubscription_enabled = true } }" > e2e-config.hcl + run: | + cp ./docs/resources/secure-bridge.conflist ./cni/secure-bridge.conflist + echo "server { default_scheduler_config { memory_oversubscription_enabled = true } }, client { cni_path = \"${{ github.workspace }}/cni/bin\", cni_config_dir = \"${{ github.workspace }}/cni\" }" > e2e-config.hcl - name: Download Poseidon binary uses: actions/download-artifact@v2 with: diff --git a/deploy/api.tpl.nomad b/deploy/api.tpl.nomad index f8a5bde..fb4843c 100644 --- a/deploy/api.tpl.nomad +++ b/deploy/api.tpl.nomad @@ -26,7 +26,7 @@ job "${NOMAD_SLUG}" { } network { - mode = "bridge" + mode = "cni/secure-bridge" port "http" { to = 7200 diff --git a/docs/configuration.md b/docs/configuration.md index 18a3f64..dfb2220 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,14 @@ In order to allow full networking support in Nomad, the `containernetworking-plu If the path is not set up correctly or the dependency is missing, the following error will be shown in Nomad: `failed to find plugin "bridge" in path [/opt/cni/bin]` +Additionally, we provide a [secure-bridge](./resources/secure-bridge.conflist) configuration for the `containernetworking-plugins`. We highly recommend to use this configuration, as it will automatically configure an appropriate firewall and isolate your local network. Store the [secure-bridge](./resources/secure-bridge.conflist) in an (otherwise) empty folder and specify that folder in `/etc/nomad.d/client.hcl`: + +```hcl + cni_config_dir = "" +``` + +If the path is not set up correctly or with a different name, the placement of allocations will fail in Nomad: `Constraint missing network filtered [all] nodes`. Be sure to set the "dns" and "dns-search" options in `/etc/docker/daemon.json` with reasonable defaults, for example with those shown in our [example configuration for Docker](./resources/docker.daemon.json). + ### Use gVisor as a sandbox We recommend using gVisor as a sandbox for the execution environments. First, [install gVisor following the official documentation](https://gvisor.dev/docs/user_guide/install/) and second, adapt the `/etc/docker/daemon.json` with reasonable defaults as shown in our [example configuration for Docker](./resources/docker.daemon.json). diff --git a/docs/resources/docker.daemon.json b/docs/resources/docker.daemon.json index 43accc3..add384b 100644 --- a/docs/resources/docker.daemon.json +++ b/docs/resources/docker.daemon.json @@ -1,4 +1,11 @@ { + "dns": [ + "8.8.8.8", + "8.8.4.4" + ], + "dns-search": [ + "codeocean.internal" + ], "default-runtime": "runsc", "runtimes": { "runsc": { diff --git a/docs/resources/secure-bridge.conflist b/docs/resources/secure-bridge.conflist new file mode 100644 index 0000000..0c2724a --- /dev/null +++ b/docs/resources/secure-bridge.conflist @@ -0,0 +1,156 @@ +{ + "cniVersion": "0.4.0", + "name": "secure-bridge", + "plugins": [ + { + "type": "loopback" + }, + { + "type": "bridge", + "bridge": "nomad-filtered", + "ipMasq": true, + "isGateway": true, + "forceAddress": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "172.26.80.0/20" + } + ] + ], + "routes": [ + { "dst": "1.0.0.0/8" }, + { "dst": "2.0.0.0/7" }, + { "dst": "4.0.0.0/6" }, + { "dst": "8.0.0.0/7" }, + { "dst": "11.0.0.0/8" }, + { "dst": "12.0.0.0/6" }, + { "dst": "16.0.0.0/4" }, + { "dst": "32.0.0.0/3" }, + { "dst": "64.0.0.0/3" }, + { "dst": "96.0.0.0/6" }, + { "dst": "100.0.0.0/10" }, + { "dst": "100.128.0.0/9" }, + { "dst": "101.0.0.0/8" }, + { "dst": "102.0.0.0/7" }, + { "dst": "104.0.0.0/5" }, + { "dst": "112.0.0.0/5" }, + { "dst": "120.0.0.0/6" }, + { "dst": "124.0.0.0/7" }, + { "dst": "126.0.0.0/8" }, + { "dst": "128.0.0.0/3" }, + { "dst": "160.0.0.0/5" }, + { "dst": "168.0.0.0/8" }, + { "dst": "169.0.0.0/9" }, + { "dst": "169.128.0.0/10" }, + { "dst": "169.192.0.0/11" }, + { "dst": "169.224.0.0/12" }, + { "dst": "169.240.0.0/13" }, + { "dst": "169.248.0.0/14" }, + { "dst": "169.252.0.0/15" }, + { "dst": "169.255.0.0/16" }, + { "dst": "170.0.0.0/7" }, + { "dst": "172.0.0.0/12" }, + { "dst": "172.32.0.0/11" }, + { "dst": "172.64.0.0/10" }, + { "dst": "172.128.0.0/9" }, + { "dst": "173.0.0.0/8" }, + { "dst": "174.0.0.0/7" }, + { "dst": "176.0.0.0/4" }, + { "dst": "192.0.1.0/24" }, + { "dst": "192.0.3.0/24" }, + { "dst": "192.0.4.0/22" }, + { "dst": "192.0.8.0/21" }, + { "dst": "192.0.16.0/20" }, + { "dst": "192.0.32.0/19" }, + { "dst": "192.0.64.0/18" }, + { "dst": "192.0.128.0/17" }, + { "dst": "192.1.0.0/16" }, + { "dst": "192.2.0.0/15" }, + { "dst": "192.4.0.0/14" }, + { "dst": "192.8.0.0/13" }, + { "dst": "192.16.0.0/12" }, + { "dst": "192.32.0.0/11" }, + { "dst": "192.64.0.0/12" }, + { "dst": "192.80.0.0/13" }, + { "dst": "192.88.0.0/18" }, + { "dst": "192.88.64.0/19" }, + { "dst": "192.88.96.0/23" }, + { "dst": "192.88.98.0/24" }, + { "dst": "192.88.100.0/22" }, + { "dst": "192.88.104.0/21" }, + { "dst": "192.88.112.0/20" }, + { "dst": "192.88.128.0/17" }, + { "dst": "192.89.0.0/16" }, + { "dst": "192.90.0.0/15" }, + { "dst": "192.92.0.0/14" }, + { "dst": "192.96.0.0/11" }, + { "dst": "192.128.0.0/11" }, + { "dst": "192.160.0.0/13" }, + { "dst": "192.169.0.0/16" }, + { "dst": "192.170.0.0/15" }, + { "dst": "192.172.0.0/14" }, + { "dst": "192.176.0.0/12" }, + { "dst": "192.192.0.0/10" }, + { "dst": "193.0.0.0/8" }, + { "dst": "194.0.0.0/7" }, + { "dst": "196.0.0.0/7" }, + { "dst": "198.0.0.0/12" }, + { "dst": "198.16.0.0/15" }, + { "dst": "198.20.0.0/14" }, + { "dst": "198.24.0.0/13" }, + { "dst": "198.32.0.0/12" }, + { "dst": "198.48.0.0/15" }, + { "dst": "198.50.0.0/16" }, + { "dst": "198.51.0.0/18" }, + { "dst": "198.51.64.0/19" }, + { "dst": "198.51.96.0/22" }, + { "dst": "198.51.101.0/24" }, + { "dst": "198.51.102.0/23" }, + { "dst": "198.51.104.0/21" }, + { "dst": "198.51.112.0/20" }, + { "dst": "198.51.128.0/17" }, + { "dst": "198.52.0.0/14" }, + { "dst": "198.56.0.0/13" }, + { "dst": "198.64.0.0/10" }, + { "dst": "198.128.0.0/9" }, + { "dst": "199.0.0.0/8" }, + { "dst": "200.0.0.0/7" }, + { "dst": "202.0.0.0/8" }, + { "dst": "203.0.0.0/18" }, + { "dst": "203.0.64.0/19" }, + { "dst": "203.0.96.0/20" }, + { "dst": "203.0.112.0/24" }, + { "dst": "203.0.114.0/23" }, + { "dst": "203.0.116.0/22" }, + { "dst": "203.0.120.0/21" }, + { "dst": "203.0.128.0/17" }, + { "dst": "203.1.0.0/16" }, + { "dst": "203.2.0.0/15" }, + { "dst": "203.4.0.0/14" }, + { "dst": "203.8.0.0/13" }, + { "dst": "203.16.0.0/12" }, + { "dst": "203.32.0.0/11" }, + { "dst": "203.64.0.0/10" }, + { "dst": "203.128.0.0/9" }, + { "dst": "204.0.0.0/6" }, + { "dst": "208.0.0.0/4" } + ] + } + }, + { + "type": "firewall", + "backend": "iptables", + "iptablesAdminChainName": "NOMAD-ADMIN-FILTERED" + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + }, + "snat": true + } + ] +} diff --git a/internal/environment/nomad_environment.go b/internal/environment/nomad_environment.go index 589ad0f..2950400 100644 --- a/internal/environment/nomad_environment.go +++ b/internal/environment/nomad_environment.go @@ -177,7 +177,7 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) { } // Prefer "bridge" network over "host" to have an isolated network namespace with bridged interface // instead of joining the host network namespace. - networkResource.Mode = "bridge" + networkResource.Mode = "cni/secure-bridge" for _, portNumber := range exposedPorts { port := nomadApi.Port{ Label: strconv.FormatUint(uint64(portNumber), portNumberBase), diff --git a/internal/environment/nomad_environment_test.go b/internal/environment/nomad_environment_test.go index f6ddb1d..6a6f4f7 100644 --- a/internal/environment/nomad_environment_test.go +++ b/internal/environment/nomad_environment_test.go @@ -32,7 +32,7 @@ func TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkExists(t *testing.T) defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) environment := &NomadEnvironment{nil, "", job, nil} - networkResource := &nomadApi.NetworkResource{Mode: "bridge"} + networkResource := &nomadApi.NetworkResource{Mode: "cni/secure-bridge"} defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource} if assert.Equal(t, 1, len(defaultTaskGroup.Networks)) { @@ -80,7 +80,7 @@ func TestConfigureNetworkSetsCorrectValues(t *testing.T) { require.Equal(t, 1, len(testTaskGroup.Networks)) networkResource := testTaskGroup.Networks[0] - assert.Equal(t, "bridge", networkResource.Mode) + assert.Equal(t, "cni/secure-bridge", networkResource.Mode) require.Equal(t, len(ports), len(networkResource.DynamicPorts)) assertExpectedPorts(t, ports, networkResource)