htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implemented pooling for Docker containers

Browse Source
This commit is contained in:
Hauke Klement
2015-02-06 15:59:51 +01:00
parent a22a5af711
commit 5f0815b140
20 changed files with 453 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
spec/lib/docker_client_spec.rb
View File

@ -4,28 +4,11 @@ require 'seeds_helper'
describe DockerClient, docker: true do
let(:command) { 'whoami' }
let(:docker_client) { DockerClient.new(execution_environment: FactoryGirl.build(:ruby), user: FactoryGirl.build(:admin)) }
let(:execution_environment) { FactoryGirl.build(:ruby) }
let(:image) { double }
let(:submission) { FactoryGirl.create(:submission) }
let(:workspace_path) { '/tmp' }
describe '#bound_folders' do
context 'when executing a submission' do
before(:each) { docker_client.instance_variable_set(:@submission, submission) }
it 'returns a submission-specific mapping' do
mapping = docker_client.send(:bound_folders).first
expect(mapping).to include(submission.id.to_s)
expect(mapping).to end_with(DockerClient::CONTAINER_WORKSPACE_PATH)
end
end
context 'when executing a single command' do
it 'returns an empty mapping' do
expect(docker_client.send(:bound_folders)).to eq([])
end
end
end
describe '.check_availability!' do
context 'when a socket error occurs' do
it 'raises an error' do
@ -42,111 +25,126 @@ describe DockerClient, docker: true do
end
end
describe '#clean_workspace' do
it 'removes the submission-specific directory' do
expect(docker_client).to receive(:local_workspace_path).and_return(workspace_path)
expect(FileUtils).to receive(:rm_rf).with(workspace_path)
docker_client.send(:clean_workspace)
describe '.create_container' do
after(:each) { DockerClient.create_container(execution_environment) }
it 'uses the correct Docker image' do
expect(DockerClient).to receive(:find_image_by_tag).with(execution_environment.docker_image).and_call_original
end
end
describe '#create_container' do
let(:image_tag) { 'tag' }
before(:each) { docker_client.instance_variable_set(:@image, image) }
it 'creates a container waiting for input' do
expect(Docker::Container).to receive(:create).with('Image' => kind_of(String), 'OpenStdin' => true, 'StdinOnce' => true).and_call_original
end
it 'creates a container' do
expect(image).to receive(:info).and_return({'RepoTags' => [image_tag]})
expect(Docker::Container).to receive(:create).with('Cmd' => command, 'Image' => image_tag)
docker_client.send(:create_container, command: command)
it 'starts the container' do
expect_any_instance_of(Docker::Container).to receive(:start)
end
it 'configures mapped directories' do
expect(DockerClient).to receive(:mapped_directories).and_call_original
expect_any_instance_of(Docker::Container).to receive(:start).with(hash_including('Binds' => kind_of(Array)))
end
it 'configures mapped ports' do
expect(DockerClient).to receive(:mapped_ports).with(execution_environment).and_call_original
expect_any_instance_of(Docker::Container).to receive(:start).with(hash_including('PortBindings' => kind_of(Hash)))
end
end
describe '#create_workspace' do
before(:each) { docker_client.instance_variable_set(:@submission, submission) }
let(:container) { double }
before(:each) do
docker_client.instance_variable_set(:@submission, submission)
expect(container).to receive(:binds).at_least(:once).and_return(["#{workspace_path}:#{DockerClient::CONTAINER_WORKSPACE_PATH}"])
end
after(:each) { docker_client.send(:create_workspace, container) }
it 'creates submission-specific directories' do
expect(docker_client).to receive(:local_workspace_path).at_least(:once).and_return(workspace_path)
expect(Dir).to receive(:mkdir).at_least(:once)
docker_client.send(:create_workspace)
end
it 'copies binary files' do
submission.collect_files.select { |file| file.file_type.binary? }.each do |file|
expect(docker_client).to receive(:copy_file_to_workspace).with(container: container, file: file)
end
end
it 'creates non-binary files' do
submission.collect_files.reject { |file| file.file_type.binary? }.each do |file|
expect(docker_client).to receive(:create_workspace_file).with(container: container, file: file)
end
end
end
describe '#create_workspace_file' do
let(:file) { FactoryGirl.build(:file, content: 'puts 42') }
let(:file_path) { File.join(workspace_path, file.name_with_extension) }
after(:each) { File.delete(file_path) }
it 'creates a file' do
expect(docker_client).to receive(:local_workspace_path).and_return(workspace_path)
docker_client.send(:create_workspace_file, file: file)
expect(DockerClient).to receive(:local_workspace_path).and_return(workspace_path)
docker_client.send(:create_workspace_file, container: CONTAINER, file: file)
expect(File.exist?(file_path)).to be true
expect(File.new(file_path, 'r').read).to eq(file.content)
File.delete(file_path)
end
end
describe '.destroy_container' do
let(:container) { docker_client.send(:create_container, {command: command}) }
let(:container) { DockerClient.send(:create_container, execution_environment) }
after(:each) { DockerClient.destroy_container(container) }
it 'stops the container' do
expect(container).to receive(:stop).and_return(container)
end
it 'kills the container' do
it 'kills running processes' do
expect(container).to receive(:kill)
end
it 'releases allocated ports' do
expect(container).to receive(:json).at_least(:once).and_return({'HostConfig' => {'PortBindings' => {foo: [{'HostPort' => '42'}]}}})
expect(container).to receive(:port_bindings).at_least(:once).and_return(foo: [{'HostPort' => '42'}])
expect(PortPool).to receive(:release)
end
it 'removes the mapped directory' do
expect(DockerClient).to receive(:local_workspace_path).and_return(workspace_path)
expect(FileUtils).to receive(:rm_rf).with(workspace_path)
end
it 'deletes the container' do
expect(container).to receive(:delete)
expect(container).to receive(:delete).with(force: true)
end
end
describe '#execute_command' do
after(:each) { docker_client.send(:execute_command, command) }
describe '#execute_arbitrary_command' do
after(:each) { docker_client.execute_arbitrary_command(command) }
it 'creates a container' do
expect(docker_client).to receive(:create_container).with(command: ['bash', '-c', command]).and_call_original
it 'takes a container from the pool' do
expect(DockerContainerPool).to receive(:get_container).and_call_original
end
it 'starts the container' do
expect(docker_client).to receive(:start_container)
it 'sends the command' do
expect(docker_client).to receive(:send_command).with(command, kind_of(Docker::Container))
end
end
describe '#execute_in_workspace' do
let(:block) { Proc.new do; end }
let(:execute_in_workspace) { docker_client.send(:execute_in_workspace, submission, &block) }
after(:each) { execute_in_workspace }
describe '#execute_run_command' do
let(:filename) { submission.exercise.files.detect { |file| file.role == 'main_file' }.name_with_extension }
after(:each) { docker_client.send(:execute_run_command, submission, filename) }
it 'takes a container from the pool' do
expect(DockerContainerPool).to receive(:get_container).with(submission.execution_environment).and_call_original
end
it 'creates the workspace' do
expect(docker_client).to receive(:create_workspace)
end
it 'calls the block' do
expect(block).to receive(:call)
end
it 'cleans the workspace' do
expect(docker_client).to receive(:clean_workspace)
end
end
describe '#execute_run_command' do
let(:block) { Proc.new {} }
let(:filename) { submission.exercise.files.detect { |file| file.role == 'main_file' }.name_with_extension }
after(:each) { docker_client.send(:execute_run_command, submission, filename, &block) }
it 'is executed in the workspace' do
expect(docker_client).to receive(:execute_in_workspace)
end
it 'executes the run command' do
expect(docker_client).to receive(:execute_command).with(kind_of(String), &block)
expect(submission.execution_environment).to receive(:run_command).and_call_original
expect(docker_client).to receive(:send_command).with(kind_of(String), kind_of(Docker::Container))
end
end
@ -154,23 +152,36 @@ describe DockerClient, docker: true do
let(:filename) { submission.exercise.files.detect { |file| file.role == 'teacher_defined_test' }.name_with_extension }
after(:each) { docker_client.send(:execute_test_command, submission, filename) }
it 'is executed in the workspace' do
expect(docker_client).to receive(:execute_in_workspace)
it 'takes a container from the pool' do
expect(DockerContainerPool).to receive(:get_container).with(submission.execution_environment).and_call_original
end
it 'creates the workspace' do
expect(docker_client).to receive(:create_workspace)
end
it 'executes the test command' do
expect(docker_client).to receive(:execute_command).with(kind_of(String))
expect(submission.execution_environment).to receive(:test_command).and_call_original
expect(docker_client).to receive(:send_command).with(kind_of(String), kind_of(Docker::Container))
end
end
describe '.generate_remote_workspace_path' do
it 'includes the correct workspace root' do
expect(DockerClient.generate_remote_workspace_path).to start_with(DockerClient.config[:workspace_root])
end
it 'includes a UUID' do
expect(SecureRandom).to receive(:uuid).and_call_original
DockerClient.generate_remote_workspace_path
end
end
describe '.initialize_environment' do
let(:config) { {connection_timeout: 3, host: 'tcp://8.8.8.8:2375', workspace_root: '/'} }
context 'with complete configuration' do
before(:each) { expect(DockerClient).to receive(:config).at_least(:once).and_return(config) }
it 'does not raise an error' do
expect { DockerClient.initialize_environment }.not_to raise_error
it 'creates the file directory' do
expect(FileUtils).to receive(:mkdir_p).with(DockerClient::LOCAL_WORKSPACE_ROOT)
DockerClient.initialize_environment
end
end
@ -183,75 +194,92 @@ describe DockerClient, docker: true do
end
end
describe '#local_workspace_path' do
before(:each) { docker_client.instance_variable_set(:@submission, submission) }
describe '.local_workspace_path' do
let(:container) { DockerClient.create_container(execution_environment) }
let(:local_workspace_path) { DockerClient.local_workspace_path(container) }
it 'includes the correct workspace root' do
expect(docker_client.send(:local_workspace_path)).to start_with(DockerClient::LOCAL_WORKSPACE_ROOT.to_s)
it 'returns a path' do
expect(local_workspace_path).to be_a(Pathname)
end
it 'is submission-specific' do
expect(docker_client.send(:local_workspace_path)).to end_with(submission.id.to_s)
it 'includes the correct workspace root' do
expect(local_workspace_path.to_s).to start_with(DockerClient::LOCAL_WORKSPACE_ROOT.to_s)
end
end
describe '#remote_workspace_path' do
before(:each) { docker_client.instance_variable_set(:@submission, submission) }
it 'includes the correct workspace root' do
expect(docker_client.send(:remote_workspace_path)).to start_with(DockerClient.config[:workspace_root])
end
it 'is submission-specific' do
expect(docker_client.send(:remote_workspace_path)).to end_with(submission.id.to_s)
describe '.mapped_directories' do
it 'returns a unique mapping' do
expect(DockerClient).to receive(:generate_remote_workspace_path).and_return(workspace_path)
mapping = DockerClient.send(:mapped_directories).first
expect(mapping).to start_with(workspace_path)
expect(mapping).to end_with(DockerClient::CONTAINER_WORKSPACE_PATH)
end
end
describe '#start_container' do
let(:container) { docker_client.send(:create_container, command: command) }
let(:start_container) { docker_client.send(:start_container, container) }
describe '.mapped_ports' do
context 'with exposed ports' do
before(:each) { execution_environment.exposed_ports = '3000' }
it 'configures bound folders' do
expect(container).to receive(:start).with(hash_including('Binds' => kind_of(Array))).and_call_original
start_container
it 'returns a mapping' do
expect(DockerClient.mapped_ports(execution_environment)).to be_a(Hash)
expect(DockerClient.mapped_ports(execution_environment).length).to eq(1)
end
it 'retrieves available ports' do
expect(PortPool).to receive(:available_port)
DockerClient.mapped_ports(execution_environment)
end
end
it 'configures bound ports' do
expect(container).to receive(:start).with(hash_including('PortBindings' => kind_of(Hash))).and_call_original
start_container
context 'without exposed ports' do
it 'returns an empty mapping' do
expect(DockerClient.mapped_ports(execution_environment)).to eq({})
end
end
end
describe '#send_command' do
let(:block) { Proc.new {} }
let(:container) { DockerClient.create_container(execution_environment) }
let(:send_command) { docker_client.send(:send_command, command, container, &block) }
after(:each) { send_command }
it 'limits the execution time' do
expect(Timeout).to receive(:timeout).at_least(:once).with(kind_of(Numeric)).and_call_original
end
it 'starts the container' do
expect(container).to receive(:start).and_call_original
start_container
it 'provides the command to be executed as input' do
expect(container).to receive(:attach).with(stdin: kind_of(StringIO))
end
it 'waits for the container to terminate' do
expect(container).to receive(:wait).with(kind_of(Numeric)).and_call_original
start_container
it 'calls the block' do
expect(block).to receive(:call)
end
context 'when a timeout occurs' do
before(:each) { expect(container).to receive(:wait).and_raise(Docker::Error::TimeoutError) }
before(:each) { expect(container).to receive(:attach).and_raise(Timeout::Error) }
it 'kills the container' do
expect(container).to receive(:kill)
start_container
it 'destroys the container asynchronously' do
expect(Concurrent::Future).to receive(:execute)
end
it 'returns a corresponding status' do
expect(start_container[:status]).to eq(:timeout)
expect(send_command[:status]).to eq(:timeout)
end
end
context 'when the container terminates timely' do
it 'destroys the container asynchronously' do
expect(Concurrent::Future).to receive(:execute)
end
it "returns the container's output" do
expect(start_container[:stderr]).to be_blank
expect(start_container[:stdout]).to start_with('root')
expect(send_command[:stderr]).to be_blank
expect(send_command[:stdout]).to start_with('root')
end
it 'returns a corresponding status' do
expect(start_container[:status]).to eq(:ok)
expect(send_command[:status]).to eq(:ok)
end
end
end

121
spec/lib/docker_container_pool_spec.rb Normal file
View File

@ -0,0 +1,121 @@
require 'rails_helper'
describe DockerContainerPool do
let(:container) { double }
def reload_class
load('docker_container_pool.rb')
end
private :reload_class
before(:each) do
@execution_environment = FactoryGirl.create(:ruby)
reload_class
end
it 'uses thread-safe data structures' do
expect(DockerContainerPool.instance_variable_get(:@containers)).to be_a(ThreadSafe::Hash)
expect(DockerContainerPool.instance_variable_get(:@containers)[@execution_environment.id]).to be_a(ThreadSafe::Array)
end
describe '.clean_up' do
before(:each) { DockerContainerPool.instance_variable_set(:@refill_task, double) }
after(:each) { DockerContainerPool.clean_up }
it 'stops the refill task' do
expect(DockerContainerPool.instance_variable_get(:@refill_task)).to receive(:shutdown)
end
it 'destroys all containers' do
DockerContainerPool.instance_variable_get(:@containers).each do |key, value|
value.each do |container|
expect(DockerClient).to receive(:destroy_container).with(container)
end
end
end
end
describe '.get_container' do
context 'when active' do
before(:each) do
expect(DockerContainerPool).to receive(:config).and_return(active: true)
end
context 'with an available container' do
before(:each) { DockerContainerPool.instance_variable_get(:@containers)[@execution_environment.id].push(container) }
it 'takes a container from the pool' do
expect(DockerContainerPool).not_to receive(:create_container).with(@execution_environment)
expect(DockerContainerPool.get_container(@execution_environment)).to eq(container)
end
end
context 'without an available container' do
before(:each) do
expect(DockerContainerPool.instance_variable_get(:@containers)[@execution_environment.id]).to be_empty
end
it 'creates a new container' do
expect(DockerContainerPool).to receive(:create_container).with(@execution_environment)
DockerContainerPool.get_container(@execution_environment)
end
end
end
context 'when inactive' do
before(:each) do
expect(DockerContainerPool).to receive(:config).and_return(active: false)
end
it 'creates a new container' do
expect(DockerContainerPool).to receive(:create_container).with(@execution_environment)
DockerContainerPool.get_container(@execution_environment)
end
end
end
describe '.quantities' do
it 'maps execution environments to quantities of available containers' do
expect(DockerContainerPool.quantities.keys).to eq(ExecutionEnvironment.all.map(&:id))
expect(DockerContainerPool.quantities.values.uniq).to eq([0])
end
end
describe '.refill' do
after(:each) { DockerContainerPool.refill }
it 'regards all execution environments' do
ExecutionEnvironment.all.each do |execution_environment|
expect(DockerContainerPool.instance_variable_get(:@containers)).to receive(:[]).with(execution_environment.id).and_call_original
end
end
context 'with something to refill' do
before(:each) { @execution_environment.update(pool_size: 1) }
it 'works asynchronously' do
expect(Concurrent::Future).to receive(:execute)
end
end
context 'with nothing to refill' do
before(:each) { @execution_environment.update(pool_size: 0) }
it 'does nothing' do
expect(Concurrent::Future).not_to receive(:execute)
end
end
end
describe '.start_refill_task' do
after(:each) { DockerContainerPool.start_refill_task }
it 'creates an asynchronous task' do
expect(Concurrent::TimerTask).to receive(:new).and_call_original
end
it 'executes the task' do
expect_any_instance_of(Concurrent::TimerTask).to receive(:execute)
end
end
end