htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Exclusively lock Runners during code executions

Browse Source 
Previously, the same runner could be used multiple times with different submissions simultaneously. This, however, yielded errors, for example when one submission time oud (causing the running to be deleted) while another submission was still executed.

Admin actions, such as the shell, can be still executed regardless of any other code execution.

Fixes CODEOCEAN-HG
Fixes openHPI/poseidon#423
This commit is contained in:
Sebastian Serth
2023-10-29 22:17:31 +01:00
committed by Sebastian Serth
parent 427b54d306
commit 8fc5123bae
16 changed files with 115 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/controllers/execution_environments_controller.rb
View File

@ -33,7 +33,7 @@ class ExecutionEnvironmentsController < ApplicationController
def execute_command
runner = Runner.for(current_user, @execution_environment)
@privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
output = runner.execute_command(params[:command], privileged_execution: @privileged_execution, raise_exception: false)
output = runner.execute_command(params[:command], privileged_execution: @privileged_execution, raise_exception: false, exclusive: false)
render json: output.except(:messages)
end
@ -41,11 +41,11 @@ class ExecutionEnvironmentsController < ApplicationController
runner = Runner.for(current_user, @execution_environment)
@privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
begin
files = runner.retrieve_files(path: params[:path], recursive: false, privileged_execution: @privileged_execution)
files = runner.retrieve_files(path: params[:path], recursive: false, privileged_execution: @privileged_execution, exclusive: false)
downloadable_files, additional_directories = convert_files_json_to_files files
js_tree = FileTree.new(downloadable_files, additional_directories, force_closed: true).to_js_tree
render json: js_tree[:core][:data]
rescue Runner::Error::RunnerNotFound, Runner::Error::WorkspaceError
rescue Runner::Error::RunnerNotFound, Runner::Error::WorkspaceError, Runner::Error::RunnerInUse
render json: []
end
end

6
app/controllers/live_streams_controller.rb
View File

@ -26,15 +26,15 @@ class LiveStreamsController < ApplicationController
runner = Runner.for(current_user, @execution_environment)
fallback_location = shell_execution_environment_path(@execution_environment)
privileged = params[:sudo] || @execution_environment.privileged_execution?
send_runner_file(runner, desired_file, fallback_location, privileged:)
send_runner_file(runner, desired_file, fallback_location, privileged:, exclusive: false)
end
private
def send_runner_file(runner, desired_file, redirect_fallback = root_path, privileged: false)
def send_runner_file(runner, desired_file, redirect_fallback = root_path, privileged: false, exclusive: true)
filename = File.basename(desired_file)
send_stream(filename:, type: 'application/octet-stream', disposition: 'attachment') do |stream|
runner.download_file(desired_file, privileged_execution: privileged) do |chunk, overall_size, _content_type|
runner.download_file(desired_file, privileged_execution: privileged, exclusive:) do |chunk, overall_size, _content_type|
unless response.committed?
# Disable Rack::ETag, which would otherwise cause the response to be cached
# See https://github.com/rack/rack/issues/1619#issuecomment-848460528

3
app/controllers/remote_evaluation_controller.rb
View File

@ -72,6 +72,9 @@ class RemoteEvaluationController < ApplicationController
# TODO: check token expired?
{message: 'No exercise found for this validation_token! Please keep out!', status: 401}
end
rescue Runner::Error::RunnerInUse => e
Rails.logger.debug { "Scoring a submission failed because the runner was already in use: #{e.message}" }
{message: I18n.t('exercises.editor.runner_in_use'), status: 409}
rescue Runner::Error => e
Rails.logger.debug { "Runner error while scoring submission #{@submission.id}: #{e.message}" }
{message: I18n.t('exercises.editor.depleted'), status: 503}

19
app/controllers/request_for_comments_controller.rb
View File

@ -167,11 +167,20 @@ class RequestForCommentsController < ApplicationController
respond_to do |format|
if @request_for_comment.save
# execute the tests here and wait until they finished.
# As the same runner is used for the score and test run, no parallelization is possible
# A run is triggered from the frontend and does not need to be handled here.
@request_for_comment.submission.calculate_score(current_user)
format.json { render :show, status: :created, location: @request_for_comment }
begin
# execute the tests here and wait until they finished.
# As the same runner is used for the score and test run, no parallelization is possible
# A run is triggered from the frontend and does not need to be handled here.
@request_for_comment.submission.calculate_score(current_user)
rescue Runner::Error::RunnerInUse => e
Rails.logger.debug { "Scoring a submission failed because the runner was already in use: #{e.message}" }
format.json { render json: {error: t('exercises.editor.runner_in_use'), status: :runner_in_use}, status: :conflict }
rescue Runner::Error => e
Rails.logger.debug { "Runner error while requesting comments: #{e.message}" }
format.json { render json: {danger: t('exercises.editor.depleted'), status: :container_depleted}, status: :service_unavailable }
else
format.json { render :show, status: :created, location: @request_for_comment }
end
else
format.html { render :new }
format.json { render json: @request_for_comment.errors, status: :unprocessable_entity }

23
app/controllers/submissions_controller.rb
View File

@ -234,6 +234,12 @@ class SubmissionsController < ApplicationController
@testrun[:exit_code] ||= 137
@testrun[:output] = "out_of_memory: #{@testrun[:output]}"
extract_durations(e)
rescue Runner::Error::RunnerInUse => e
send_and_store client_socket, {cmd: :status, status: :runner_in_use}
Rails.logger.debug { "Running a submission failed because the runner was already in use: #{e.message}" }
@testrun[:status] ||= :runner_in_use
@testrun[:output] = "runner_in_use: #{@testrun[:output]}"
extract_durations(e)
rescue Runner::Error => e
# Regardless of the specific error cause, we send a `container_depleted` status to the client.
send_and_store client_socket, {cmd: :status, status: :container_depleted}
@ -266,6 +272,14 @@ class SubmissionsController < ApplicationController
client_socket&.send_data(JSON.dump(@submission.calculate_score(current_user)))
# To enable hints when scoring a submission, uncomment the next line:
# send_hints(client_socket, StructuredError.where(submission: @submission))
rescue Runner::Error::RunnerInUse => e
extract_durations(e)
send_and_store client_socket, {cmd: :status, status: :runner_in_use}
Rails.logger.debug { "Scoring a submission failed because the runner was already in use: #{e.message}" }
@testrun[:passed] = false
@testrun[:status] ||= :runner_in_use
@testrun[:output] = "runner_in_use: #{@testrun[:output]}"
save_testrun_output 'assess'
rescue Runner::Error => e
extract_durations(e)
send_and_store client_socket, {cmd: :status, status: :container_depleted}
@ -301,10 +315,17 @@ class SubmissionsController < ApplicationController
# The score is stored separately, we can forward it to the client immediately
client_socket&.send_data(JSON.dump(@submission.test(@file, current_user)))
rescue Runner::Error::RunnerInUse => e
extract_durations(e)
send_and_store client_socket, {cmd: :status, status: :runner_in_use}
Rails.logger.debug { "Scoring a submission failed because the runner was already in use: #{e.message}" }
@testrun[:passed] = false
@testrun[:status] ||= :runner_in_use
@testrun[:output] = "runner_in_use: #{@testrun[:output]}"
save_testrun_output 'assess'
rescue Runner::Error => e
extract_durations(e)
send_and_store client_socket, {cmd: :status, status: :container_depleted}
kill_client_socket(client_socket)
Rails.logger.debug { "Runner error while testing submission #{@submission.id}: #{e.message}" }
Sentry.capture_exception(e)
@testrun[:passed] = false