Improve error resilience and handling

Timeouts are now handled correctly and the Runner automatically
creates the execution environment if it could not be found in
Poseidon. The runner is deleted locally if Poseidon returns
a bad request error.
This commit is contained in:
Felix Auringer
2021-06-14 09:22:29 +02:00
committed by Sebastian Serth
parent b6bc578aea
commit 413f9b2705
7 changed files with 136 additions and 29 deletions

View File

@ -16,11 +16,26 @@ describe Runner::Strategy::Poseidon do
let(:response_status) { 400 }
it 'raises an error' do
allow(Runner).to receive(:destroy).with(runner_id)
expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/)
end
end
end
# Only #copy_files and #execute_command destroy the runner locally in case
# of a BadRequest (400) response.
shared_examples 'BadRequest (400) destroys local runner' do
context 'when Poseidon returns BadRequest (400)' do
let(:response_body) { {message: error_message}.to_json }
let(:response_status) { 400 }
it 'destroys the runner locally' do
expect(Runner).to receive(:destroy).with(runner_id)
expect { action.call }.to raise_error(Runner::Error::BadRequest)
end
end
end
# All requests handle a Unauthorized (401) response the same way.
shared_examples 'Unauthorized (401) error handling' do
context 'when Poseidon returns Unauthorized (401)' do
@ -193,6 +208,7 @@ describe Runner::Strategy::Poseidon do
end
include_examples 'BadRequest (400) error handling'
include_examples 'BadRequest (400) destroys local runner'
include_examples 'Unauthorized (401) error handling'
include_examples 'NotFound (404) error handling'
include_examples 'InternalServerError (500) error handling'
@ -247,6 +263,7 @@ describe Runner::Strategy::Poseidon do
end
include_examples 'BadRequest (400) error handling'
include_examples 'BadRequest (400) destroys local runner'
include_examples 'Unauthorized (401) error handling'
include_examples 'NotFound (404) error handling'
include_examples 'InternalServerError (500) error handling'

View File

@ -61,28 +61,81 @@ describe Runner do
include_examples 'delegates method sends to its strategy', :attach_to_execution, nil
end
describe '#request_id' do
it 'requests a runner from the runner management when created' do
describe 'creation' do
let(:user) { FactoryBot.create :external_user }
let(:execution_environment) { FactoryBot.create :ruby }
let(:create_action) { -> { described_class.create(user: user, execution_environment: execution_environment) } }
it 'requests a runner id from the runner management' do
expect(strategy_class).to receive(:request_from_management)
described_class.create
create_action.call
end
it 'sets the runner id when created' do
it 'returns a valid runner' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
runner = described_class.create
expect(runner.runner_id).to eq(runner_id)
expect(create_action.call).to be_valid
end
it 'sets the strategy when created' do
it 'sets the strategy' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
runner = described_class.create
expect(runner.strategy).to be_present
strategy = strategy_class.new(runner_id, execution_environment)
allow(strategy_class).to receive(:new).with(runner_id, execution_environment).and_return(strategy)
runner = create_action.call
expect(runner.strategy).to eq(strategy)
end
it 'does not call the runner management again when validating the model' do
it 'does not call the runner management again while a runner id is set' do
expect(strategy_class).to receive(:request_from_management).and_return(runner_id).once
runner = described_class.create
runner.valid?
runner = create_action.call
runner.update(user: FactoryBot.create(:external_user))
end
end
describe '#request_new_id' do
let(:runner) { FactoryBot.create :runner }
context 'when the environment is available in the runner management' do
it 'requests the runner management' do
expect(strategy_class).to receive(:request_from_management)
runner.send(:request_new_id)
end
it 'updates the runner id' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
runner.send(:request_new_id)
expect(runner.runner_id).to eq(runner_id)
end
it 'updates the strategy' do
allow(strategy_class).to receive(:request_from_management).and_return(runner_id)
strategy = strategy_class.new(runner_id, runner.execution_environment)
allow(strategy_class).to receive(:new).with(runner_id, runner.execution_environment).and_return(strategy)
runner.send(:request_new_id)
expect(runner.strategy).to eq(strategy)
end
end
context 'when the environment could not be found in the runner management' do
let(:environment_id) { runner.execution_environment.id }
before { allow(strategy_class).to receive(:request_from_management).and_raise(Runner::Error::NotFound) }
it 'syncs the execution environment' do
expect(strategy_class).to receive(:sync_environment).with(runner.execution_environment)
runner.send(:request_new_id)
rescue Runner::Error::NotFound
# Ignored because this error is expected (see tests below).
end
it 'raises an error when the environment could be synced' do
allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(true)
expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*successfully synced/)
end
it 'raises an error when the environment could not be synced' do
allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(false)
expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*could not be synced/)
end
end
end