htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce strategy for runner behavior

Browse Source 
The runner model is only a class responsible for storing information
now. Based on the configuration it picks a strategy for the runner
management. The Poseidon strategy is already implemented and tested.
The Docker strategy will follow.
This commit is contained in:
Felix Auringer
2021-06-09 09:39:56 +02:00
committed by Sebastian Serth
parent cf58be97ee
commit d0d1b1bffd
13 changed files with 541 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
spec/concerns/submission_scoring_spec.rb
View File

@ -11,7 +11,7 @@ describe SubmissionScoring do
before do
allow(Runner).to receive(:for).and_return(runner)
allow(runner).to receive(:copy_files)
allow(runner).to receive(:execute_interactively).and_return(1.0)
allow(runner).to receive(:attach_to_execution).and_return(1.0)
end
after { submission.calculate_score }

2
spec/factories/runner.rb
View File

@ -3,7 +3,7 @@
# This factory does not request the runner management as the id is already provided.
FactoryBot.define do
factory :runner do
runner_id { 'test-runner-id' }
sequence(:runner_id) {|n| "test-runner-id-#{n}" }
association :execution_environment, factory: :ruby
association :user, factory: :external_user
waiting_time { 1.0 }

15
spec/lib/runner/strategy/docker_spec.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
require 'rails_helper'
describe Runner::Strategy::Docker do
let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] }
let(:execution_environment) { FactoryBot.create :ruby }
let(:docker) { described_class.new(runner_id, execution_environment) }
# TODO: add tests for these methods when implemented
it 'defines all methods all runner management strategies must define' do
expect(docker.public_methods).to include(*Runner::DELEGATED_STRATEGY_METHODS)
expect(described_class.public_methods).to include(:request_from_management)
end
end

273
spec/lib/runner/strategy/poseidon_spec.rb Normal file
View File

@ -0,0 +1,273 @@
# frozen_string_literal: true
require 'rails_helper'
describe Runner::Strategy::Poseidon do
let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] }
let(:execution_environment) { FactoryBot.create :ruby }
let(:poseidon) { described_class.new(runner_id, execution_environment) }
let(:error_message) { 'test error message' }
let(:response_body) { nil }
# All requests handle a BadRequest (400) response the same way.
shared_examples 'BadRequest (400) error handling' do
context 'when Poseidon returns BadRequest (400)' do
let(:response_body) { {message: error_message}.to_json }
let(:response_status) { 400 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/)
end
end
end
# All requests handle a Unauthorized (401) response the same way.
shared_examples 'Unauthorized (401) error handling' do
context 'when Poseidon returns Unauthorized (401)' do
let(:response_status) { 401 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unauthorized)
end
end
end
# All requests except creation handle a NotFound (404) response the same way.
shared_examples 'NotFound (404) error handling' do
context 'when Poseidon returns NotFound (404)' do
let(:response_status) { 404 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/)
end
end
end
# All requests handle an InternalServerError (500) response the same way.
shared_examples 'InternalServerError (500) error handling' do
context 'when Poseidon returns InternalServerError (500)' do
shared_examples 'InternalServerError (500) with error code' do |error_code, error_class|
let(:response_status) { 500 }
let(:response_body) { {message: error_message, errorCode: error_code}.to_json }
it 'raises an error' do
expect { action.call }.to raise_error(error_class) do |error|
expect(error.message).to match(/#{error_message}/)
expect(error.message).to match(/#{error_code}/)
end
end
end
context 'when error code is nomad overload' do
include_examples(
'InternalServerError (500) with error code',
described_class.error_nomad_overload, Runner::Error::NotAvailable
)
end
context 'when error code is not nomad overload' do
include_examples(
'InternalServerError (500) with error code',
described_class.error_unknown, Runner::Error::InternalServerError
)
end
end
end
# All requests handle an unknown response status the same way.
shared_examples 'unknown response status error handling' do
context 'when Poseidon returns an unknown response status' do
let(:response_status) { 1337 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown, /#{response_status}/)
end
end
end
describe '::request_from_management' do
let(:action) { -> { described_class.request_from_management(execution_environment) } }
let!(:request_runner_stub) do
WebMock
.stub_request(:post, "#{Runner::BASE_URL}/runners")
.with(
body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
context 'when Poseidon returns Ok (200) with an id' do
let(:response_body) { {runnerId: runner_id}.to_json }
let(:response_status) { 200 }
it 'successfully requests Poseidon' do
action.call
expect(request_runner_stub).to have_been_requested.once
end
it 'returns the received runner id' do
id = action.call
expect(id).to eq(runner_id)
end
end
context 'when Poseidon returns Ok (200) without an id' do
let(:response_body) { {}.to_json }
let(:response_status) { 200 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
end
context 'when Poseidon returns Ok (200) with invalid JSON' do
let(:response_body) { '{hello}' }
let(:response_status) { 200 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
end
include_examples 'BadRequest (400) error handling'
include_examples 'Unauthorized (401) error handling'
context 'when Poseidon returns NotFound (404)' do
let(:response_status) { 404 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/)
end
end
include_examples 'InternalServerError (500) error handling'
include_examples 'unknown response status error handling'
end
describe '#execute_command' do
let(:command) { 'ls' }
let(:action) { -> { poseidon.send(:execute_command, command) } }
let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' }
let!(:execute_command_stub) do
WebMock
.stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute")
.with(
body: {command: command, timeLimit: execution_environment.permitted_execution_time},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
context 'when Poseidon returns Ok (200) with a websocket url' do
let(:response_status) { 200 }
let(:response_body) { {websocketUrl: websocket_url}.to_json }
it 'schedules an execution in Poseidon' do
action.call
expect(execute_command_stub).to have_been_requested.once
end
it 'returns the url' do
url = action.call
expect(url).to eq(websocket_url)
end
end
context 'when Poseidon returns Ok (200) without a websocket url' do
let(:response_body) { {}.to_json }
let(:response_status) { 200 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
end
context 'when Poseidon returns Ok (200) with invalid JSON' do
let(:response_body) { '{hello}' }
let(:response_status) { 200 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
end
include_examples 'BadRequest (400) error handling'
include_examples 'Unauthorized (401) error handling'
include_examples 'NotFound (404) error handling'
include_examples 'InternalServerError (500) error handling'
include_examples 'unknown response status error handling'
end
describe '#destroy_at_management' do
let(:action) { -> { poseidon.destroy_at_management } }
let!(:destroy_stub) do
WebMock
.stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}")
.to_return(body: response_body, status: response_status)
end
context 'when Poseidon returns NoContent (204)' do
let(:response_status) { 204 }
it 'deletes the runner from Poseidon' do
action.call
expect(destroy_stub).to have_been_requested.once
end
end
include_examples 'Unauthorized (401) error handling'
include_examples 'NotFound (404) error handling'
include_examples 'InternalServerError (500) error handling'
include_examples 'unknown response status error handling'
end
describe '#copy_files' do
let(:filename) { 'main.py' }
let(:file_content) { 'print("Hello World!")' }
let(:action) { -> { poseidon.copy_files({filename => file_content}) } }
let(:encoded_file_content) { Base64.strict_encode64(file_content) }
let!(:copy_files_stub) do
WebMock
.stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files")
.with(
body: {copy: [{path: filename, content: encoded_file_content}]},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
context 'when Poseidon returns NoContent (204)' do
let(:response_status) { 204 }
it 'sends the files to Poseidon' do
action.call
expect(copy_files_stub).to have_been_requested.once
end
end
include_examples 'BadRequest (400) error handling'
include_examples 'Unauthorized (401) error handling'
include_examples 'NotFound (404) error handling'
include_examples 'InternalServerError (500) error handling'
include_examples 'unknown response status error handling'
end
describe '#attach_to_execution' do
# TODO: add more tests here
let(:command) { 'ls' }
let(:action) { -> { poseidon.attach_to_execution command } }
let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' }
it 'returns the execution time' do
allow(poseidon).to receive(:execute_command).with(command).and_return(websocket_url)
allow(EventMachine).to receive(:run)
starting_time = Time.zone.now
execution_time = action.call
test_time = Time.zone.now - starting_time
expect(execution_time).to be_between(0.0, test_time)
end
end
end

367
spec/models/runner_spec.rb
View File

@ -3,92 +3,17 @@
require 'rails_helper'
describe Runner do
let(:runner) { FactoryBot.create :runner }
let(:runner_id) { runner.runner_id }
let(:error_message) { 'test error message' }
let(:response_body) { nil }
let(:user) { FactoryBot.build :external_user }
let(:execution_environment) { FactoryBot.create :ruby }
# All requests handle a BadRequest (400) response the same way.
shared_examples 'BadRequest (400) error handling' do
let(:response_body) { {message: error_message}.to_json }
let(:response_status) { 400 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/)
end
end
# All requests handle a Unauthorized (401) response the same way.
shared_examples 'Unauthorized (401) error handling' do
let(:response_status) { 401 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unauthorized)
end
end
# All requests except creation and destruction handle a NotFound (404) response the same way.
shared_examples 'NotFound (404) error handling' do
let(:response_status) { 404 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/)
end
it 'destroys the runner locally' do
expect { action.call }.to change(described_class, :count).by(-1)
.and raise_error(Runner::Error::NotFound)
end
end
# All requests handle an InternalServerError (500) response the same way.
shared_examples 'InternalServerError (500) error handling' do
shared_examples 'InternalServerError (500) with error code' do |error_code, error_class|
let(:response_status) { 500 }
let(:response_body) { {message: error_message, errorCode: error_code}.to_json }
it 'raises an error' do
expect { action.call }.to raise_error(error_class) do |error|
expect(error.message).to match(/#{error_message}/)
expect(error.message).to match(/#{error_code}/)
end
end
end
context 'when error code is nomad overload' do
include_examples(
'InternalServerError (500) with error code',
described_class.error_nomad_overload, Runner::Error::NotAvailable
)
end
context 'when error code is not nomad overload' do
include_examples(
'InternalServerError (500) with error code',
described_class.error_unknown, Runner::Error::InternalServerError
)
end
end
# All requests handle an unknown response status the same way.
shared_examples 'unknown response status error handling' do
let(:response_status) { 1337 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
end
let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] }
let(:strategy_class) { described_class.strategy_class }
describe 'attribute validation' do
let(:runner) { FactoryBot.create :runner }
it 'validates the presence of the runner id' do
described_class.skip_callback(:validation, :before, :request_remotely)
described_class.skip_callback(:validation, :before, :request_id)
runner.update(runner_id: nil)
expect(runner.errors[:runner_id]).to be_present
described_class.set_callback(:validation, :before, :request_remotely)
described_class.set_callback(:validation, :before, :request_id)
end
it 'validates the presence of an execution environment' do
@ -102,256 +27,98 @@ describe Runner do
end
end
describe 'creation' do
let(:action) { -> { described_class.create(user: user, execution_environment: execution_environment) } }
let!(:create_stub) do
WebMock
.stub_request(:post, "#{Runner::BASE_URL}/runners")
.with(
body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
context 'when a runner is created' do
let(:response_body) { {runnerId: runner_id}.to_json }
let(:response_status) { 200 }
it 'requests a runner from the runner management' do
action.call
expect(create_stub).to have_been_requested.once
end
it 'does not call the runner management again when updating' do
runner = action.call
runner.runner_id = 'another_id'
successfully_saved = runner.save
expect(successfully_saved).to be_truthy
expect(create_stub).to have_been_requested.once
describe '::strategy_class' do
shared_examples 'uses the strategy defined in the constant' do |strategy, strategy_class|
it "uses #{strategy_class} as strategy class for constant #{strategy}" do
stub_const('Runner::STRATEGY_NAME', strategy)
expect(described_class.strategy_class).to eq(strategy_class)
end
end
context 'when the runner management returns Ok (200) with an id' do
let(:response_body) { {runnerId: runner_id}.to_json }
let(:response_status) { 200 }
{poseidon: Runner::Strategy::Poseidon, docker: Runner::Strategy::Docker}.each do |strategy, strategy_class|
include_examples 'uses the strategy defined in the constant', strategy, strategy_class
end
it 'sets the runner id according to the response' do
runner = action.call
expect(runner.runner_id).to eq(runner_id)
expect(runner).to be_persisted
shared_examples 'delegates method sends to its strategy' do |method, *args|
context "when sending #{method}" do
let(:strategy) { instance_double(strategy_class) }
let(:runner) { described_class.create }
before do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
allow(strategy_class).to receive(:new).and_return(strategy)
end
it "delegates the method #{method}" do
expect(strategy).to receive(method)
runner.send(method, *args)
end
end
end
context 'when the runner management returns Ok (200) without an id' do
let(:response_body) { {}.to_json }
let(:response_status) { 200 }
include_examples 'delegates method sends to its strategy', :destroy_at_management
include_examples 'delegates method sends to its strategy', :copy_files, nil
include_examples 'delegates method sends to its strategy', :attach_to_execution, nil
end
it 'does not save the runner' do
runner = action.call
expect(runner).not_to be_persisted
end
describe '#request_id' do
it 'requests a runner from the runner management when created' do
expect(strategy_class).to receive(:request_from_management)
described_class.create
end
context 'when the runner management returns Ok (200) with invalid JSON' do
let(:response_body) { '{hello}' }
let(:response_status) { 200 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
end
it 'sets the runner id when created' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
runner = described_class.create
expect(runner.runner_id).to eq(runner_id)
end
context 'when the runner management returns BadRequest (400)' do
include_examples 'BadRequest (400) error handling'
it 'sets the strategy when created' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
runner = described_class.create
expect(runner.strategy).to be_present
end
context 'when the runner management returns Unauthorized (401)' do
include_examples 'Unauthorized (401) error handling'
end
context 'when the runner management returns NotFound (404)' do
let(:response_status) { 404 }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/)
end
end
context 'when the runner management returns InternalServerError (500)' do
include_examples 'InternalServerError (500) error handling'
end
context 'when the runner management returns an unknown response status' do
include_examples 'unknown response status error handling'
it 'does not call the runner management again when validating the model' do
expect(strategy_class).to receive(:request_from_management).and_return(runner_id).once
runner = described_class.create
runner.valid?
end
end
describe 'execute command' do
let(:command) { 'ls' }
let(:action) { -> { runner.execute_command(command) } }
let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' }
let!(:execute_command_stub) do
WebMock
.stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute")
.with(
body: {command: command, timeLimit: execution_environment.permitted_execution_time},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
describe '::for' do
let(:user) { FactoryBot.create :external_user }
let(:exercise) { FactoryBot.create :fibonacci }
context 'when #execute_command is called' do
let(:response_status) { 200 }
let(:response_body) { {websocketUrl: websocket_url}.to_json }
it 'schedules an execution in the runner management' do
action.call
expect(execute_command_stub).to have_been_requested.once
end
end
context 'when the runner management returns Ok (200) with a websocket url' do
let(:response_status) { 200 }
let(:response_body) { {websocketUrl: websocket_url}.to_json }
it 'returns the url' do
url = action.call
expect(url).to eq(websocket_url)
end
end
context 'when the runner management returns Ok (200) without a websocket url' do
let(:response_body) { {}.to_json }
let(:response_status) { 200 }
context 'when the runner could not be saved' do
before { allow(strategy_class).to receive(:request_from_management).and_return(nil) }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
expect { described_class.for(user, exercise) }.to raise_error(Runner::Error::Unknown, /could not be saved/)
end
end
context 'when the runner management returns Ok (200) with invalid JSON' do
let(:response_body) { '{hello}' }
let(:response_status) { 200 }
context 'when a runner already exists' do
let!(:existing_runner) { FactoryBot.create(:runner, user: user, execution_environment: exercise.execution_environment) }
it 'raises an error' do
expect { action.call }.to raise_error(Runner::Error::Unknown)
it 'returns the existing runner' do
new_runner = described_class.for(user, exercise)
expect(new_runner).to eq(existing_runner)
end
it 'sets the strategy' do
runner = described_class.for(user, exercise)
expect(runner.strategy).to be_present
end
end
context 'when the runner management returns BadRequest (400)' do
include_examples 'BadRequest (400) error handling'
end
context 'when no runner exists' do
before { allow(strategy_class).to receive(:request_from_management).and_return(runner_id) }
context 'when the runner management returns Unauthorized (401)' do
include_examples 'Unauthorized (401) error handling'
end
context 'when the runner management returns NotFound (404)' do
include_examples 'NotFound (404) error handling'
end
context 'when the runner management returns InternalServerError (500)' do
include_examples 'InternalServerError (500) error handling'
end
context 'when the runner management returns an unknown response status' do
include_examples 'unknown response status error handling'
end
end
describe 'destruction' do
let(:action) { -> { runner.destroy_remotely } }
let(:response_status) { 204 }
let!(:destroy_stub) do
WebMock
.stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}")
.to_return(body: response_body, status: response_status)
end
it 'deletes the runner from the runner management' do
action.call
expect(destroy_stub).to have_been_requested.once
end
it 'does not destroy the runner locally' do
expect { action.call }.not_to change(described_class, :count)
end
context 'when the runner management returns NoContent (204)' do
it 'does not raise an error' do
expect { action.call }.not_to raise_error
it 'returns a new runner' do
runner = described_class.for(user, exercise)
expect(runner).to be_valid
end
end
context 'when the runner management returns Unauthorized (401)' do
include_examples 'Unauthorized (401) error handling'
end
context 'when the runner management returns NotFound (404)' do
let(:response_status) { 404 }
it 'raises an exception' do
expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/)
end
end
context 'when the runner management returns InternalServerError (500)' do
include_examples 'InternalServerError (500) error handling'
end
context 'when the runner management returns an unknown response status' do
include_examples 'unknown response status error handling'
end
end
describe 'copy files' do
let(:filename) { 'main.py' }
let(:file_content) { 'print("Hello World!")' }
let(:action) { -> { runner.copy_files({filename => file_content}) } }
let(:encoded_file_content) { Base64.strict_encode64(file_content) }
let(:response_status) { 204 }
let!(:copy_files_stub) do
WebMock
.stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files")
.with(
body: {copy: [{path: filename, content: encoded_file_content}]},
headers: {'Content-Type' => 'application/json'}
)
.to_return(body: response_body, status: response_status)
end
it 'sends the files to the runner management' do
action.call
expect(copy_files_stub).to have_been_requested.once
end
context 'when the runner management returns NoContent (204)' do
let(:response_status) { 204 }
it 'does not raise an error' do
expect { action.call }.not_to raise_error
end
end
context 'when the runner management returns BadRequest (400)' do
include_examples 'BadRequest (400) error handling'
end
context 'when the runner management returns Unauthorized (401)' do
include_examples 'Unauthorized (401) error handling'
end
context 'when the runner management returns NotFound (404)' do
include_examples 'NotFound (404) error handling'
end
context 'when the runner management returns InternalServerError (500)' do
include_examples 'InternalServerError (500) error handling'
end
context 'when the runner management returns an unknown response status' do
include_examples 'unknown response status error handling'
end
end
end