htwkmooc/poseidon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cni/secure-bridge to isolate host network

Browse Source
This commit is contained in:
Sebastian Serth
2022-09-09 00:35:37 +02:00
parent 1df9701a74
commit d372e37d1a
7 changed files with 191 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.github/workflows/ci.yml vendored
View File

@ -9,6 +9,7 @@ on:
env:
GO_VERSION: 1.19
NOMAD_VERSION: 1.3.5
CNI_VERSION: 1.1.1
jobs:
compile:
@ -141,10 +142,12 @@ jobs:
# More details: https://github.com/golang/go/blob/d60ad1e068263832c711aaf17b6ccb1b7f71b000/src/cmd/go/internal/cache/cache.go#L255-L326
run: date +%s > ~/.cache/go-build/trim.txt
continue-on-error: true
- name: Cache Nomad binary
- name: Cache Nomad and CNI binaries
uses: actions/cache@v2
with:
path: ${{ github.workspace }}/nomad
path: |
${{ github.workspace }}/nomad
${{ github.workspace }}/cni/bin
key: ${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }}
restore-keys: |
${{ runner.os }}-nomad-${{ env.NOMAD_VERSION }}
@ -155,8 +158,18 @@ jobs:
wget -q "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_SHA256SUMS"
grep "nomad_${NOMAD_VERSION}_linux_amd64.zip" nomad_${NOMAD_VERSION}_SHA256SUMS | sha256sum -c -
unzip nomad_${NOMAD_VERSION}_linux_amd64.zip
- name: Download CNI binaries
run: |
if [[ -f ./cni/bin ]]; then exit 0; fi
wget -q "https://github.com/containernetworking/plugins/releases/download/v${CNI_VERSION}/cni-plugins-linux-amd64-v${CNI_VERSION}.tgz"
wget -q "https://github.com/containernetworking/plugins/releases/download/v${CNI_VERSION}/cni-plugins-linux-amd64-v${CNI_VERSION}.tgz.sha256"
grep "cni-plugins-linux-amd64-v${CNI_VERSION}.tgz" cni-plugins-linux-amd64-v${CNI_VERSION}.tgz.sha256 | sha256sum -c -
mkdir -p ./cni/bin
tar zxvf cni-plugins-linux-amd64-v${CNI_VERSION}.tgz -C ./cni/bin
- name: Set Nomad Config
run: echo "server { default_scheduler_config { memory_oversubscription_enabled = true } }" > e2e-config.hcl
run: |
cp ./docs/resources/secure-bridge.conflist ./cni/secure-bridge.conflist
echo "server { default_scheduler_config { memory_oversubscription_enabled = true } }, client { cni_path = \"${{ github.workspace }}/cni/bin\", cni_config_dir = \"${{ github.workspace }}/cni\" }" > e2e-config.hcl
- name: Download Poseidon binary
uses: actions/download-artifact@v2
with:

2
deploy/api.tpl.nomad
View File

@ -26,7 +26,7 @@ job "${NOMAD_SLUG}" {
}
network {
mode = "bridge"
mode = "cni/secure-bridge"
port "http" {
to = 7200

8
docs/configuration.md
View File

@ -73,6 +73,14 @@ In order to allow full networking support in Nomad, the `containernetworking-plu
If the path is not set up correctly or the dependency is missing, the following error will be shown in Nomad: `failed to find plugin "bridge" in path [/opt/cni/bin]`
Additionally, we provide a [secure-bridge](./resources/secure-bridge.conflist) configuration for the `containernetworking-plugins`. We highly recommend to use this configuration, as it will automatically configure an appropriate firewall and isolate your local network. Store the [secure-bridge](./resources/secure-bridge.conflist) in an (otherwise) empty folder and specify that folder in `/etc/nomad.d/client.hcl`:
```hcl
cni_config_dir = "<path to folder with *.conflist>"
```
If the path is not set up correctly or with a different name, the placement of allocations will fail in Nomad: `Constraint missing network filtered [all] nodes`. Be sure to set the "dns" and "dns-search" options in `/etc/docker/daemon.json` with reasonable defaults, for example with those shown in our [example configuration for Docker](./resources/docker.daemon.json).
### Use gVisor as a sandbox
We recommend using gVisor as a sandbox for the execution environments. First, [install gVisor following the official documentation](https://gvisor.dev/docs/user_guide/install/) and second, adapt the `/etc/docker/daemon.json` with reasonable defaults as shown in our [example configuration for Docker](./resources/docker.daemon.json).

7
docs/resources/docker.daemon.json
View File

@ -1,4 +1,11 @@
{
"dns": [
"8.8.8.8",
"8.8.4.4"
],
"dns-search": [
"codeocean.internal"
],
"default-runtime": "runsc",
"runtimes": {
"runsc": {

156
docs/resources/secure-bridge.conflist Normal file
View File

@ -0,0 +1,156 @@
{
"cniVersion": "0.4.0",
"name": "secure-bridge",
"plugins": [
{
"type": "loopback"
},
{
"type": "bridge",
"bridge": "nomad-filtered",
"ipMasq": true,
"isGateway": true,
"forceAddress": true,
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "172.26.80.0/20"
}
]
],
"routes": [
{ "dst": "1.0.0.0/8" },
{ "dst": "2.0.0.0/7" },
{ "dst": "4.0.0.0/6" },
{ "dst": "8.0.0.0/7" },
{ "dst": "11.0.0.0/8" },
{ "dst": "12.0.0.0/6" },
{ "dst": "16.0.0.0/4" },
{ "dst": "32.0.0.0/3" },
{ "dst": "64.0.0.0/3" },
{ "dst": "96.0.0.0/6" },
{ "dst": "100.0.0.0/10" },
{ "dst": "100.128.0.0/9" },
{ "dst": "101.0.0.0/8" },
{ "dst": "102.0.0.0/7" },
{ "dst": "104.0.0.0/5" },
{ "dst": "112.0.0.0/5" },
{ "dst": "120.0.0.0/6" },
{ "dst": "124.0.0.0/7" },
{ "dst": "126.0.0.0/8" },
{ "dst": "128.0.0.0/3" },
{ "dst": "160.0.0.0/5" },
{ "dst": "168.0.0.0/8" },
{ "dst": "169.0.0.0/9" },
{ "dst": "169.128.0.0/10" },
{ "dst": "169.192.0.0/11" },
{ "dst": "169.224.0.0/12" },
{ "dst": "169.240.0.0/13" },
{ "dst": "169.248.0.0/14" },
{ "dst": "169.252.0.0/15" },
{ "dst": "169.255.0.0/16" },
{ "dst": "170.0.0.0/7" },
{ "dst": "172.0.0.0/12" },
{ "dst": "172.32.0.0/11" },
{ "dst": "172.64.0.0/10" },
{ "dst": "172.128.0.0/9" },
{ "dst": "173.0.0.0/8" },
{ "dst": "174.0.0.0/7" },
{ "dst": "176.0.0.0/4" },
{ "dst": "192.0.1.0/24" },
{ "dst": "192.0.3.0/24" },
{ "dst": "192.0.4.0/22" },
{ "dst": "192.0.8.0/21" },
{ "dst": "192.0.16.0/20" },
{ "dst": "192.0.32.0/19" },
{ "dst": "192.0.64.0/18" },
{ "dst": "192.0.128.0/17" },
{ "dst": "192.1.0.0/16" },
{ "dst": "192.2.0.0/15" },
{ "dst": "192.4.0.0/14" },
{ "dst": "192.8.0.0/13" },
{ "dst": "192.16.0.0/12" },
{ "dst": "192.32.0.0/11" },
{ "dst": "192.64.0.0/12" },
{ "dst": "192.80.0.0/13" },
{ "dst": "192.88.0.0/18" },
{ "dst": "192.88.64.0/19" },
{ "dst": "192.88.96.0/23" },
{ "dst": "192.88.98.0/24" },
{ "dst": "192.88.100.0/22" },
{ "dst": "192.88.104.0/21" },
{ "dst": "192.88.112.0/20" },
{ "dst": "192.88.128.0/17" },
{ "dst": "192.89.0.0/16" },
{ "dst": "192.90.0.0/15" },
{ "dst": "192.92.0.0/14" },
{ "dst": "192.96.0.0/11" },
{ "dst": "192.128.0.0/11" },
{ "dst": "192.160.0.0/13" },
{ "dst": "192.169.0.0/16" },
{ "dst": "192.170.0.0/15" },
{ "dst": "192.172.0.0/14" },
{ "dst": "192.176.0.0/12" },
{ "dst": "192.192.0.0/10" },
{ "dst": "193.0.0.0/8" },
{ "dst": "194.0.0.0/7" },
{ "dst": "196.0.0.0/7" },
{ "dst": "198.0.0.0/12" },
{ "dst": "198.16.0.0/15" },
{ "dst": "198.20.0.0/14" },
{ "dst": "198.24.0.0/13" },
{ "dst": "198.32.0.0/12" },
{ "dst": "198.48.0.0/15" },
{ "dst": "198.50.0.0/16" },
{ "dst": "198.51.0.0/18" },
{ "dst": "198.51.64.0/19" },
{ "dst": "198.51.96.0/22" },
{ "dst": "198.51.101.0/24" },
{ "dst": "198.51.102.0/23" },
{ "dst": "198.51.104.0/21" },
{ "dst": "198.51.112.0/20" },
{ "dst": "198.51.128.0/17" },
{ "dst": "198.52.0.0/14" },
{ "dst": "198.56.0.0/13" },
{ "dst": "198.64.0.0/10" },
{ "dst": "198.128.0.0/9" },
{ "dst": "199.0.0.0/8" },
{ "dst": "200.0.0.0/7" },
{ "dst": "202.0.0.0/8" },
{ "dst": "203.0.0.0/18" },
{ "dst": "203.0.64.0/19" },
{ "dst": "203.0.96.0/20" },
{ "dst": "203.0.112.0/24" },
{ "dst": "203.0.114.0/23" },
{ "dst": "203.0.116.0/22" },
{ "dst": "203.0.120.0/21" },
{ "dst": "203.0.128.0/17" },
{ "dst": "203.1.0.0/16" },
{ "dst": "203.2.0.0/15" },
{ "dst": "203.4.0.0/14" },
{ "dst": "203.8.0.0/13" },
{ "dst": "203.16.0.0/12" },
{ "dst": "203.32.0.0/11" },
{ "dst": "203.64.0.0/10" },
{ "dst": "203.128.0.0/9" },
{ "dst": "204.0.0.0/6" },
{ "dst": "208.0.0.0/4" }
]
}
},
{
"type": "firewall",
"backend": "iptables",
"iptablesAdminChainName": "NOMAD-ADMIN-FILTERED"
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
},
"snat": true
}
]
}

2
internal/environment/nomad_environment.go
View File

@ -177,7 +177,7 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) {
}
// Prefer "bridge" network over "host" to have an isolated network namespace with bridged interface
// instead of joining the host network namespace.
networkResource.Mode = "bridge"
networkResource.Mode = "cni/secure-bridge"
for _, portNumber := range exposedPorts {
port := nomadApi.Port{
Label: strconv.FormatUint(uint64(portNumber), portNumberBase),

4
internal/environment/nomad_environment_test.go
View File

@ -32,7 +32,7 @@ func TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkExists(t *testing.T)
defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job)
environment := &NomadEnvironment{nil, "", job, nil}
networkResource := &nomadApi.NetworkResource{Mode: "bridge"}
networkResource := &nomadApi.NetworkResource{Mode: "cni/secure-bridge"}
defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource}
if assert.Equal(t, 1, len(defaultTaskGroup.Networks)) {
@ -80,7 +80,7 @@ func TestConfigureNetworkSetsCorrectValues(t *testing.T) {
require.Equal(t, 1, len(testTaskGroup.Networks))
networkResource := testTaskGroup.Networks[0]
assert.Equal(t, "bridge", networkResource.Mode)
assert.Equal(t, "cni/secure-bridge", networkResource.Mode)
require.Equal(t, len(ports), len(networkResource.DynamicPorts))
assertExpectedPorts(t, ports, networkResource)