From 2e2cd1855eba56c03f09283ce9090b8874302f46 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Mon, 29 Mar 2021 16:05:05 +0200 Subject: [PATCH 001/156] Add Container abstration with new API calls and adapt running a submission Co-authored-by: Felix Auringer --- app/controllers/submissions_controller.rb | 76 +++++++++++------------ app/models/submission.rb | 34 ++++++++++ lib/container.rb | 54 ++++++++++++++++ 3 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 lib/container.rb diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index fc70f5a8..84a320ae 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -10,7 +10,7 @@ class SubmissionsController < ApplicationController before_action :set_submission, only: %i[download download_file render_file run score extract_errors show statistics test] - before_action :set_docker_client, only: %i[run test] + # before_action :set_docker_client, only: %i[run test] before_action :set_files, only: %i[download download_file render_file show run] before_action :set_file, only: %i[download_file render_file run] before_action :set_mime_type, only: %i[download_file render_file] @@ -168,54 +168,48 @@ class SubmissionsController < ApplicationController # socket is the socket into the container, tubesock is the socket to the client # give the docker_client the tubesock object, so that it can send messages (timeout) - @docker_client.tubesock = tubesock + # @docker_client.tubesock = tubesock container_request_time = Time.zone.now - result = @docker_client.execute_run_command(@submission, sanitize_filename) - tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => result[:status]}) + # result = @docker_client.execute_run_command(@submission, sanitize_filename) + container = @submission.run(sanitize_filename) + tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) @waiting_for_container_time = Time.zone.now - container_request_time - if result[:status] == :container_running - socket = result[:socket] - command = result[:command] - + socket = container.socket socket.on :message do |event| Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") - handle_message(event.data, tubesock, result[:container]) + handle_message(event.data, tubesock, container) end socket.on :close do |_event| kill_socket(tubesock) end - tubesock.onmessage do |data| - Rails.logger.info("#{Time.zone.now.getutc}: Client sending: #{data}") - # Check whether the client send a JSON command and kill container - # if the command is 'client_kill', send it to docker otherwise. - begin - parsed = JSON.parse(data) unless data == "\n" - if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' - Rails.logger.debug('Client exited container.') - @docker_client.kill_container(result[:container]) - else - socket.send data - Rails.logger.debug { "Sent the received client data to docker:#{data}" } - end - rescue JSON::ParserError - socket.send data - Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } - Sentry.set_extras(data: data) - end - end + tubesock.onmessage do |data| + Rails.logger.info("#{Time.zone.now.getutc}: Client sending: #{data}") + # Check whether the client send a JSON command and kill container + # if the command is 'client_kill', send it to docker otherwise. + begin - # Send command after all listeners are attached. - # Newline required to flush - @execution_request_time = Time.zone.now - socket.send "#{command}\n" - Rails.logger.info("Sent command: #{command}") - else - kill_socket(tubesock) + parsed = JSON.parse(data) unless data == "\n" + if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' + Rails.logger.debug("Client exited container.") + container.destroy + else + socket.send data + Rails.logger.debug { "Sent the received client data to docker:#{data}" } + end + rescue JSON::ParserError => error + socket.send data + Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } + Sentry.set_extras(data: data) + end end + + # Send command after all listeners are attached. + # Newline required to flush + @execution_request_time = Time.zone.now end end @@ -251,7 +245,7 @@ class SubmissionsController < ApplicationController # Do not call kill_socket for the websocket to the client here. # @docker_client.exit_container closes the socket to the container, # kill_socket is called in the "on close handler" of the websocket to the container - @docker_client.exit_container(container) + container.destroy when /^#timeout/ @run_output = "timeout: #{@run_output}" # add information that this run timed out to the buffer else @@ -275,7 +269,7 @@ class SubmissionsController < ApplicationController if parsed.instance_of?(Hash) && parsed.key?('cmd') socket.send_data message Rails.logger.info("parse_message sent: #{message}") - @docker_client.exit_container(container) if container && parsed['cmd'] == 'exit' + container.destroy if container && parsed['cmd'] == 'exit' else parsed = {'cmd' => 'write', 'stream' => output_stream, 'data' => message} socket.send_data JSON.dump(parsed) @@ -388,10 +382,10 @@ class SubmissionsController < ApplicationController end end - def set_docker_client - @docker_client = DockerClient.new(execution_environment: @submission.execution_environment) - end - private :set_docker_client + # def set_docker_client + # @docker_client = DockerClient.new(execution_environment: @submission.execution_environment) + # end + # private :set_docker_client def set_file @file = @files.detect {|file| file.name_with_extension == sanitize_filename } diff --git a/app/models/submission.rb b/app/models/submission.rb index ca062fa2..362c5588 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -135,4 +135,38 @@ class Submission < ApplicationRecord ((rfc_element.comments_count < MAX_COMMENTS_ON_RECOMMENDED_RFC) && !rfc_element.question.empty?) end end + + def test(file) + score_command = command_for execution_environment.test_command, file + container = run_command_with_self score_command + container + end + + def run(file) + run_command = command_for execution_environment.run_command, file + container = run_command_with_self run_command + container + end + + def run_command_with_self(command) + container = Container.new(execution_environment) + container.copy_submission_files self + container.execute_command_interactively(command) + container + end + + private + + def command_for(template, file) + filepath = collect_files.find { |f| f.name_with_extension == file }.filepath + template % command_substitutions(filepath) + end + + def command_substitutions(filename) + { + class_name: File.basename(filename, File.extname(filename)).camelize, + filename: filename, + module_name: File.basename(filename, File.extname(filename)).underscore + } + end end diff --git a/lib/container.rb b/lib/container.rb new file mode 100644 index 00000000..c2bc77ab --- /dev/null +++ b/lib/container.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class Container + BASE_URL = "http://192.168.178.53:5000" + + attr_accessor :socket + + def initialize(execution_environment) + url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" + response = Faraday.post url + response = parse response + @container_id = response[:id] + end + + def copy_files(files) + url = container_url + "/files" + payload = files.map{ |filename, content| { filename => content } } + Faraday.post(url, payload.to_json) + end + + def copy_submission_files(submission) + files = {} + submission.collect_files.each do |file| + files[file.name] = file.content + end + copy_files(files) + end + + def execute_command(command) + url = container_url + "/execute" + response = Faraday.patch(url, {command: command}.to_json, "Content-Type" => "application/json") + response = parse response + response + end + + def execute_command_interactively(command) + websocket_url = execute_command(command)[:websocket_url] + @socket = Faye::WebSocket::Client.new websocket_url + end + + def destroy + Faraday.delete container_url + end + + private + + def container_url + "#{BASE_URL}/containers/#{@container_id}" + end + + def parse(response) + JSON.parse(response.body).deep_symbolize_keys + end +end From 3cf70a33d857fab5ba02a214a6a6a9ae3a76ef7c Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 30 Mar 2021 14:15:00 +0200 Subject: [PATCH 002/156] Integrate new API with websocket (run only) Co-authored-by: Felix Auringer --- app/controllers/submissions_controller.rb | 119 +++++++++------------- app/models/submission.rb | 4 +- lib/container.rb | 15 ++- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 84a320ae..f5a59054 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -138,79 +138,60 @@ class SubmissionsController < ApplicationController end def run - # TODO: reimplement SSEs with websocket commands - # with_server_sent_events do |server_sent_event| - # output = @docker_client.execute_run_command(@submission, sanitize_filename) - - # server_sent_event.write({stdout: output[:stdout]}, event: 'output') if output[:stdout] - # server_sent_event.write({stderr: output[:stderr]}, event: 'output') if output[:stderr] - # end - - hijack do |tubesock| - if @embed_options[:disable_run] - kill_socket(tubesock) - return - end - - # probably add: - # ensure - # #guarantee that the thread is releasing the DB connection after it is done - # ApplicationRecord.connectionpool.releaseconnection - # end - unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - Thread.new do - EventMachine.run - ensure - ActiveRecord::Base.connection_pool.release_connection - end - end - - # socket is the socket into the container, tubesock is the socket to the client - - # give the docker_client the tubesock object, so that it can send messages (timeout) - # @docker_client.tubesock = tubesock - - container_request_time = Time.zone.now - # result = @docker_client.execute_run_command(@submission, sanitize_filename) - container = @submission.run(sanitize_filename) - tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) - @waiting_for_container_time = Time.zone.now - container_request_time - - socket = container.socket - socket.on :message do |event| - Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") - handle_message(event.data, tubesock, container) - end - - socket.on :close do |_event| + Thread.new do + hijack do |tubesock| + if @embed_options[:disable_run] kill_socket(tubesock) + return end + EventMachine.run do + container_request_time = Time.zone.now + @submission.run(sanitize_filename) do |socket| + tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) + @waiting_for_container_time = Time.zone.now - container_request_time + @execution_request_time = Time.zone.now - tubesock.onmessage do |data| - Rails.logger.info("#{Time.zone.now.getutc}: Client sending: #{data}") - # Check whether the client send a JSON command and kill container - # if the command is 'client_kill', send it to docker otherwise. - begin + socket.on :message do |event| + Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") + handle_message(event.data, tubesock) + end - parsed = JSON.parse(data) unless data == "\n" - if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' - Rails.logger.debug("Client exited container.") - container.destroy - else - socket.send data - Rails.logger.debug { "Sent the received client data to docker:#{data}" } + socket.on :close do |_event| + EventMachine.stop_event_loop + kill_socket(tubesock) + end + + tubesock.onmessage do |data| + Rails.logger.info(Time.now.getutc.to_s + ": Client sending: " + data) + # Check whether the client send a JSON command and kill container + # if the command is 'client_kill', send it to docker otherwise. + begin + + parsed = JSON.parse(data) unless data == "\n" + if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' + Rails.logger.debug("Client exited container.") + container.destroy + else + socket.send data + Rails.logger.debug { "Sent the received client data to docker:#{data}" } + end + rescue JSON::ParserError => error + socket.send data + Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } + Sentry.set_extras(data: data) + end + end end - rescue JSON::ParserError => error - socket.send data - Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } - Sentry.set_extras(data: data) end end - - # Send command after all listeners are attached. - # Newline required to flush - @execution_request_time = Time.zone.now end + # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? + # Thread.new do + # EventMachine.run + # ensure + # ActiveRecord::Base.connection_pool.release_connection + # end + # end end def kill_socket(tubesock) @@ -235,7 +216,7 @@ class SubmissionsController < ApplicationController tubesock.close end - def handle_message(message, tubesock, container) + def handle_message(message, tubesock) @raw_output ||= '' @run_output ||= '' # Handle special commands first @@ -245,7 +226,7 @@ class SubmissionsController < ApplicationController # Do not call kill_socket for the websocket to the client here. # @docker_client.exit_container closes the socket to the container, # kill_socket is called in the "on close handler" of the websocket to the container - container.destroy + # container.destroy when /^#timeout/ @run_output = "timeout: #{@run_output}" # add information that this run timed out to the buffer else @@ -257,7 +238,7 @@ class SubmissionsController < ApplicationController test_command = run_command end unless %r{root@|:/workspace|#{run_command}|#{test_command}|bash: cmd:canvasevent: command not found}.match?(message) - parse_message(message, 'stdout', tubesock, container) + parse_message(message, 'stdout', tubesock) end end end @@ -269,7 +250,7 @@ class SubmissionsController < ApplicationController if parsed.instance_of?(Hash) && parsed.key?('cmd') socket.send_data message Rails.logger.info("parse_message sent: #{message}") - container.destroy if container && parsed['cmd'] == 'exit' + # container.destroy if container && parsed['cmd'] == 'exit' else parsed = {'cmd' => 'write', 'stream' => output_stream, 'data' => message} socket.send_data JSON.dump(parsed) diff --git a/app/models/submission.rb b/app/models/submission.rb index 362c5588..fc05be7e 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -146,10 +146,12 @@ class Submission < ApplicationRecord run_command = command_for execution_environment.run_command, file container = run_command_with_self run_command container + yield(container.socket) if block_given? + container.destroy end def run_command_with_self(command) - container = Container.new(execution_environment) + container = Container.new(execution_environment, execution_environment.permitted_execution_time) container.copy_submission_files self container.execute_command_interactively(command) container diff --git a/lib/container.rb b/lib/container.rb index c2bc77ab..140a50ad 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -5,17 +5,21 @@ class Container attr_accessor :socket - def initialize(execution_environment) + def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" - response = Faraday.post url + body = {} + if time_limit + body[:time_limit] = time_limit + end + response = Faraday.post(url, body.to_json, "Content-Type" => "application/json") response = parse response @container_id = response[:id] end def copy_files(files) url = container_url + "/files" - payload = files.map{ |filename, content| { filename => content } } - Faraday.post(url, payload.to_json) + body = files.map{ |filename, content| { filename => content } } + Faraday.post(url, body.to_json, "Content-Type" => "application/json") end def copy_submission_files(submission) @@ -35,7 +39,8 @@ class Container def execute_command_interactively(command) websocket_url = execute_command(command)[:websocket_url] - @socket = Faye::WebSocket::Client.new websocket_url + @socket = Faye::WebSocket::Client.new(websocket_url, [], ping: 0.1) + # Faye::WebSocket::Client.new(socket_url, [], headers: headers, ping: 0.1) end def destroy From 1546f7081882ba5da9ada19c6e081fe522e55995 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 30 Mar 2021 16:10:19 +0200 Subject: [PATCH 003/156] Begin to refactor websocket handling and implement test Co-authored-by: Felix Auringer --- app/controllers/submissions_controller.rb | 105 +++++++++++----------- app/models/submission.rb | 24 +++-- lib/container.rb | 7 +- 3 files changed, 78 insertions(+), 58 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index f5a59054..947d7c1c 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -137,6 +137,45 @@ class SubmissionsController < ApplicationController end end + def handle_websockets(tubesock, container) + socket = container.socket + tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) + @waiting_for_container_time = Time.zone.now - @container_request_time + @execution_request_time = Time.zone.now + + socket.on :message do |event| + Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") + handle_message(event.data, tubesock) + end + + socket.on :close do |_event| + EventMachine.stop_event_loop + tubesock.send_data JSON.dump({'cmd' => 'timeout'}) if container.status == 'timeouted' + kill_socket(tubesock) + end + + tubesock.onmessage do |data| + Rails.logger.info("#{Time.now.getutc.to_s}: Client sending: #{data}") + # Check whether the client send a JSON command and kill container + # if the command is 'client_kill', send it to docker otherwise. + begin + + parsed = JSON.parse(data) unless data == "\n" + if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' + Rails.logger.debug("Client exited container.") + container.destroy + else + socket.send data + Rails.logger.debug { "Sent the received client data to docker:#{data}" } + end + rescue JSON::ParserError => error + socket.send data + Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } + Sentry.set_extras(data: data) + end + end + end + def run Thread.new do hijack do |tubesock| @@ -144,46 +183,13 @@ class SubmissionsController < ApplicationController kill_socket(tubesock) return end - EventMachine.run do - container_request_time = Time.zone.now - @submission.run(sanitize_filename) do |socket| - tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) - @waiting_for_container_time = Time.zone.now - container_request_time - @execution_request_time = Time.zone.now - - socket.on :message do |event| - Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") - handle_message(event.data, tubesock) - end - - socket.on :close do |_event| - EventMachine.stop_event_loop - kill_socket(tubesock) - end - - tubesock.onmessage do |data| - Rails.logger.info(Time.now.getutc.to_s + ": Client sending: " + data) - # Check whether the client send a JSON command and kill container - # if the command is 'client_kill', send it to docker otherwise. - begin - - parsed = JSON.parse(data) unless data == "\n" - if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' - Rails.logger.debug("Client exited container.") - container.destroy - else - socket.send data - Rails.logger.debug { "Sent the received client data to docker:#{data}" } - end - rescue JSON::ParserError => error - socket.send data - Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } - Sentry.set_extras(data: data) - end - end - end + @container_request_time = Time.zone.now + @submission.run(sanitize_filename) do |container| + handle_websockets(tubesock, container) end end + ensure + ActiveRecord::Base.connection_pool.release_connection end # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? # Thread.new do @@ -396,20 +402,19 @@ class SubmissionsController < ApplicationController def statistics; end def test - hijack do |tubesock| - unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - Thread.new do - EventMachine.run - ensure - ActiveRecord::Base.connection_pool.release_connection + Thread.new do + hijack do |tubesock| + if @embed_options[:disable_run] + kill_socket(tubesock) + return + end + @container_request_time = Time.now + @submission.run_tests(sanitize_filename) do |container| + handle_websockets(tubesock, container) end end - - output = @docker_client.execute_test_command(@submission, sanitize_filename) - - # tubesock is the socket to the client - tubesock.send_data JSON.dump(output) - tubesock.send_data JSON.dump('cmd' => 'exit') + ensure + ActiveRecord::Base.connection_pool.release_connection end end diff --git a/app/models/submission.rb b/app/models/submission.rb index fc05be7e..757c4786 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -136,24 +136,36 @@ class Submission < ApplicationRecord end end - def test(file) + def score(file) score_command = command_for execution_environment.test_command, file container = run_command_with_self score_command container + # Todo receive websocket data and pass it to some score function end - def run(file) + def run(file, &block) run_command = command_for execution_environment.run_command, file - container = run_command_with_self run_command - container - yield(container.socket) if block_given? + execute_interactively(run_command, &block) + end + + def run_tests(file, &block) + test_command = command_for execution_environment.test_command, file + execute_interactively(test_command, &block) + end + + def execute_interactively(command) + container = nil + EventMachine.run do + container = run_command_with_self command + yield(container) if block_given? + end container.destroy end def run_command_with_self(command) container = Container.new(execution_environment, execution_environment.permitted_execution_time) container.copy_submission_files self - container.execute_command_interactively(command) + container.execute_interactively(command) container end diff --git a/lib/container.rb b/lib/container.rb index 140a50ad..ee1add52 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -37,16 +37,19 @@ class Container response end - def execute_command_interactively(command) + def execute_interactively(command) websocket_url = execute_command(command)[:websocket_url] @socket = Faye::WebSocket::Client.new(websocket_url, [], ping: 0.1) - # Faye::WebSocket::Client.new(socket_url, [], headers: headers, ping: 0.1) end def destroy Faraday.delete container_url end + def status + parse(Faraday.get(container_url))[:status] + end + private def container_url From 92b249e7b30142b01797e235edee2cacef4d7d74 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Wed, 31 Mar 2021 16:18:21 +0200 Subject: [PATCH 004/156] Reimplement scoring and create connection abstraction Co-authored-by: Felix Auringer --- .../concerns/submission_scoring.rb | 97 +++++++++---------- app/controllers/submissions_controller.rb | 43 +++----- app/models/submission.rb | 75 +++++++++----- lib/container.rb | 10 +- lib/container_connection.rb | 58 +++++++++++ 5 files changed, 172 insertions(+), 111 deletions(-) create mode 100644 lib/container_connection.rb diff --git a/app/controllers/concerns/submission_scoring.rb b/app/controllers/concerns/submission_scoring.rb index a0847373..997ebae5 100644 --- a/app/controllers/concerns/submission_scoring.rb +++ b/app/controllers/concerns/submission_scoring.rb @@ -1,57 +1,48 @@ # frozen_string_literal: true -require 'concurrent/future' - module SubmissionScoring - def collect_test_results(submission) + def test_result(output, file) + submission = self # Mnemosyne.trace 'custom.codeocean.collect_test_results', meta: { submission: submission.id } do - futures = submission.collect_files.select(&:teacher_defined_assessment?).map do |file| - Concurrent::Future.execute do - # Mnemosyne.trace 'custom.codeocean.collect_test_results_block', meta: { file: file.id, submission: submission.id } do - assessor = Assessor.new(execution_environment: submission.execution_environment) - output = execute_test_file(file, submission) - assessment = assessor.assess(output) - passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score]).positive?) - testrun_output = passed ? nil : "status: #{output[:status]}\n stdout: #{output[:stdout]}\n stderr: #{output[:stderr]}" - if testrun_output.present? - submission.exercise.execution_environment.error_templates.each do |template| - pattern = Regexp.new(template.signature).freeze - StructuredError.create_from_template(template, testrun_output, submission) if pattern.match(testrun_output) - end - end - testrun = Testrun.create( - submission: submission, - cause: 'assess', # Required to differ run and assess for RfC show - file: file, # Test file that was executed - passed: passed, - output: testrun_output, - container_execution_time: output[:container_execution_time], - waiting_for_container_time: output[:waiting_for_container_time] - ) - - filename = file.name_with_extension - - if file.teacher_defined_linter? - LinterCheckRun.create_from(testrun, assessment) - switch_locale do - assessment = assessor.translate_linter(assessment, I18n.locale) - - # replace file name with hint if linter is not used for grading. Refactor! - filename = t('exercises.implement.not_graded') if file.weight.zero? - end - end - - output.merge!(assessment) - output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight) - # end + # Mnemosyne.trace 'custom.codeocean.collect_test_results_block', meta: { file: file.id, submission: submission.id } do + assessor = Assessor.new(execution_environment: submission.execution_environment) + assessment = assessor.assess(output) + passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score]).positive?) + testrun_output = passed ? nil : "status: #{output[:status]}\n stdout: #{output[:stdout]}\n stderr: #{output[:stderr]}" + if testrun_output.present? + submission.exercise.execution_environment.error_templates.each do |template| + pattern = Regexp.new(template.signature).freeze + StructuredError.create_from_template(template, testrun_output, submission) if pattern.match(testrun_output) end end - futures.map(&:value!) + testrun = Testrun.create( + submission: submission, + cause: 'assess', # Required to differ run and assess for RfC show + file: file, # Test file that was executed + passed: passed, + output: testrun_output, + container_execution_time: output[:container_execution_time], + waiting_for_container_time: output[:waiting_for_container_time] + ) + + filename = file.name_with_extension + + if file.teacher_defined_linter? + LinterCheckRun.create_from(testrun, assessment) + assessment = assessor.translate_linter(assessment, I18n.locale) + + # replace file name with hint if linter is not used for grading. Refactor! + filename = t('exercises.implement.not_graded') if file.weight.zero? + end + + output.merge!(assessment) + output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight) end private :collect_test_results def execute_test_file(file, submission) + # TODO: replace DockerClient here DockerClient.new(execution_environment: file.context.execution_environment).execute_test_command(submission, file.name_with_extension) end @@ -59,19 +50,19 @@ module SubmissionScoring private :execute_test_file def feedback_message(file, output) - switch_locale do - if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' - I18n.t('exercises.implement.default_test_feedback') - elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' - I18n.t('exercises.implement.default_linter_feedback') - else - render_markdown(file.feedback_message) - end + # set_locale + if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' + I18n.t('exercises.implement.default_test_feedback') + elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' + I18n.t('exercises.implement.default_linter_feedback') + else + # render_markdown(file.feedback_message) end end - def score_submission(submission) - outputs = collect_test_results(submission) + def score_submission(outputs) + # outputs = collect_test_results(submission) + submission = self score = 0.0 if outputs.present? outputs.each do |output| diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 947d7c1c..f26a1b2f 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -137,18 +137,17 @@ class SubmissionsController < ApplicationController end end - def handle_websockets(tubesock, container) - socket = container.socket + def handle_websockets(tubesock, container, socket) tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) @waiting_for_container_time = Time.zone.now - @container_request_time @execution_request_time = Time.zone.now - socket.on :message do |event| - Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{event.data}") - handle_message(event.data, tubesock) + socket.on :message do |data| + Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{data}") + handle_message(data, tubesock) end - socket.on :close do |_event| + socket.on :exit do |_exit_code| EventMachine.stop_event_loop tubesock.send_data JSON.dump({'cmd' => 'timeout'}) if container.status == 'timeouted' kill_socket(tubesock) @@ -184,8 +183,8 @@ class SubmissionsController < ApplicationController return end @container_request_time = Time.zone.now - @submission.run(sanitize_filename) do |container| - handle_websockets(tubesock, container) + @submission.run(sanitize_filename) do |container, socket| + handle_websockets(tubesock, container, socket) end end ensure @@ -330,33 +329,19 @@ class SubmissionsController < ApplicationController end def score - hijack do |tubesock| - if @embed_options[:disable_score] - kill_socket(tubesock) - return - end - - unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - Thread.new do - EventMachine.run - ensure - ActiveRecord::Base.connection_pool.release_connection + Thread.new do + hijack do |tubesock| + if @embed_options[:disable_run] + return kill_socket(tubesock) end - end - # tubesock is the socket to the client - - # the score_submission call will end up calling docker exec, which is blocking. - # to ensure responsiveness, we therefore open a thread here. - Thread.new do - tubesock.send_data JSON.dump(score_submission(@submission)) - + tubesock.send_data(@submission.calculate_score) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) tubesock.send_data JSON.dump({'cmd' => 'exit'}) - ensure - ActiveRecord::Base.connection_pool.release_connection end + ensure + ActiveRecord::Base.connection_pool.release_connection end end diff --git a/app/models/submission.rb b/app/models/submission.rb index 757c4786..eb9b7c47 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -4,6 +4,9 @@ class Submission < ApplicationRecord include Context include Creation include ActionCableHelper + include SubmissionScoring + + require 'concurrent/future' CAUSES = %w[assess download file render run save submit test autosave requestComments remoteAssess remoteSubmit].freeze @@ -136,41 +139,61 @@ class Submission < ApplicationRecord end end - def score(file) - score_command = command_for execution_environment.test_command, file - container = run_command_with_self score_command - container - # Todo receive websocket data and pass it to some score function + def calculate_score + score = nil + prepared_container do |container| + scores = collect_files.select(&:teacher_defined_assessment?).map do |file| + score_command = command_for execution_environment.test_command, file.name_with_extension + stdout = "" + stderr = "" + exit_code = 0 + container.execute_interactively(score_command) do |container, socket| + socket.on :stderr do + |data| stderr << data + end + socket.on :stdout do + |data| stdout << data + end + socket.on :close do |_exit_code| + exit_code = _exit_code + EventMachine.stop_event_loop + end + end + output = { + file_role: file.role, + waiting_for_container_time: 1.second, # TODO + container_execution_time: 1.second, # TODO + status: (exit_code == 0) ? :ok : :failed, + stdout: stdout, + stderr: stderr, + } + test_result(output, file) + end + score = score_submission(scores) + end + JSON.dump(score) end def run(file, &block) run_command = command_for execution_environment.run_command, file - execute_interactively(run_command, &block) - end - - def run_tests(file, &block) - test_command = command_for execution_environment.test_command, file - execute_interactively(test_command, &block) - end - - def execute_interactively(command) - container = nil - EventMachine.run do - container = run_command_with_self command - yield(container) if block_given? + prepared_container do |container| + container.execute_interactively(run_command, &block) end - container.destroy - end - - def run_command_with_self(command) - container = Container.new(execution_environment, execution_environment.permitted_execution_time) - container.copy_submission_files self - container.execute_interactively(command) - container end private + def prepared_container + request_time = Time.now + container = Container.new(execution_environment, execution_environment.permitted_execution_time) + container.copy_submission_files self + container_time = Time.now + waiting_for_container_time = Time.now - request_time + yield(container) if block_given? + execution_time = Time.now - container_time + container.destroy + end + def command_for(template, file) filepath = collect_files.find { |f| f.name_with_extension == file }.filepath template % command_substitutions(filepath) diff --git a/lib/container.rb b/lib/container.rb index ee1add52..fee25f41 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true +require 'container_connection' + class Container BASE_URL = "http://192.168.178.53:5000" - attr_accessor :socket - def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" body = {} @@ -39,7 +39,11 @@ class Container def execute_interactively(command) websocket_url = execute_command(command)[:websocket_url] - @socket = Faye::WebSocket::Client.new(websocket_url, [], ping: 0.1) + EventMachine.run do + #socket = Faye::WebSocket::Client.new(websocket_url, [], ping: 0.1) + socket = ContainerConnection.new(websocket_url) + yield(self, socket) if block_given? + end end def destroy diff --git a/lib/container_connection.rb b/lib/container_connection.rb new file mode 100644 index 00000000..fd51775c --- /dev/null +++ b/lib/container_connection.rb @@ -0,0 +1,58 @@ +require 'faye/websocket/client' + +class ContainerConnection + EVENTS = %i[start message exit stdout stderr].freeze + + def initialize(url) + @socket = Faye::WebSocket::Client.new(url, [], ping: 0.1) + + %i[open message error close].each do |event_type| + @socket.on event_type, &:"on_#{event_type}" + end + + EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", lambda {}) } + end + + def on(event, &block) + return unless EVENTS.include? event + + instance_variable_set(:"@#{event}_callback", block) + end + + def send(data) + @socket.send(data) + end + + private + + def parse(event) + JSON.parse(event.data).deep_symbolize_keys + end + + def on_message(event) + event = parse(event) + case event[:type] + when :exit_code + @exit_code = event[:data] + when :stderr + @stderr_callback.call event[:data] + @message_callback.call event[:data] + when :stdout + @stdout_callback.call event[:data] + @message_callback.call event[:data] + else + :error + end + end + + def on_open(event) + @start_callback.call + end + + def on_error(event) + end + + def on_close(event) + @exit_callback.call @exit_code + end +end \ No newline at end of file From 6a4e302f4ea13a5103355c76055fe4f5161258f9 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 1 Apr 2021 10:32:47 +0200 Subject: [PATCH 005/156] Fix socket handling and add configuration option Co-authored-by: Felix Auringer --- app/controllers/submissions_controller.rb | 2 +- app/models/submission.rb | 16 +++++++------- lib/container.rb | 3 +-- lib/container_connection.rb | 26 ++++++++++++++--------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index f26a1b2f..ac8f6560 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -142,7 +142,7 @@ class SubmissionsController < ApplicationController @waiting_for_container_time = Time.zone.now - @container_request_time @execution_request_time = Time.zone.now - socket.on :message do |data| + socket.on :output do |data| Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{data}") handle_message(data, tubesock) end diff --git a/app/models/submission.rb b/app/models/submission.rb index eb9b7c47..3c590389 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -146,23 +146,23 @@ class Submission < ApplicationRecord score_command = command_for execution_environment.test_command, file.name_with_extension stdout = "" stderr = "" - exit_code = 0 + exit_code = 1 # default to error container.execute_interactively(score_command) do |container, socket| - socket.on :stderr do - |data| stderr << data + socket.on :stderr do |data| + stderr << data end - socket.on :stdout do - |data| stdout << data + socket.on :stdout do |data| + stdout << data end - socket.on :close do |_exit_code| + socket.on :exit do |_exit_code| exit_code = _exit_code EventMachine.stop_event_loop end end output = { file_role: file.role, - waiting_for_container_time: 1.second, # TODO - container_execution_time: 1.second, # TODO + waiting_for_container_time: 1, # TODO + container_execution_time: 1, # TODO status: (exit_code == 0) ? :ok : :failed, stdout: stdout, stderr: stderr, diff --git a/lib/container.rb b/lib/container.rb index fee25f41..a34cc53f 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -3,7 +3,7 @@ require 'container_connection' class Container - BASE_URL = "http://192.168.178.53:5000" + BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" @@ -40,7 +40,6 @@ class Container def execute_interactively(command) websocket_url = execute_command(command)[:websocket_url] EventMachine.run do - #socket = Faye::WebSocket::Client.new(websocket_url, [], ping: 0.1) socket = ContainerConnection.new(websocket_url) yield(self, socket) if block_given? end diff --git a/lib/container_connection.rb b/lib/container_connection.rb index fd51775c..f48c1b28 100644 --- a/lib/container_connection.rb +++ b/lib/container_connection.rb @@ -1,16 +1,18 @@ require 'faye/websocket/client' class ContainerConnection - EVENTS = %i[start message exit stdout stderr].freeze + EVENTS = %i[start output exit stdout stderr].freeze def initialize(url) @socket = Faye::WebSocket::Client.new(url, [], ping: 0.1) %i[open message error close].each do |event_type| - @socket.on event_type, &:"on_#{event_type}" + @socket.on event_type do |event| __send__(:"on_#{event_type}", event) end end - EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", lambda {}) } + EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", lambda {|e|}) } + @start_callback = lambda {} + @exit_code = 0 end def on(event, &block) @@ -20,26 +22,30 @@ class ContainerConnection end def send(data) - @socket.send(data) + @socket.send(encode(data)) end private - def parse(event) - JSON.parse(event.data).deep_symbolize_keys + def decode(event) + JSON.parse(event).deep_symbolize_keys + end + + def encode(data) + data end def on_message(event) - event = parse(event) - case event[:type] + event = decode(event.data) + case event[:type].to_sym when :exit_code @exit_code = event[:data] when :stderr @stderr_callback.call event[:data] - @message_callback.call event[:data] + @output_callback.call event[:data] when :stdout @stdout_callback.call event[:data] - @message_callback.call event[:data] + @output_callback.call event[:data] else :error end From 347e4728a029a3f907965eebb66358344aa46820 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 1 Apr 2021 11:33:57 +0200 Subject: [PATCH 006/156] Rework protocol inside websocket Co-authored-by Felix Auringer --- app/controllers/submissions_controller.rb | 215 ++++++++-------------- app/models/submission.rb | 14 +- config/locales/de.yml | 1 + config/locales/en.yml | 1 + lib/container.rb | 13 +- 5 files changed, 90 insertions(+), 154 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index ac8f6560..04f0bfd8 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -16,7 +16,7 @@ class SubmissionsController < ApplicationController before_action :set_mime_type, only: %i[download_file render_file] skip_before_action :verify_authenticity_token, only: %i[download_file render_file] - def max_run_output_buffer_size + def max_output_buffer_size if @submission.cause == 'requestComments' 5000 else @@ -139,37 +139,53 @@ class SubmissionsController < ApplicationController def handle_websockets(tubesock, container, socket) tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) - @waiting_for_container_time = Time.zone.now - @container_request_time - @execution_request_time = Time.zone.now + @output = '' socket.on :output do |data| Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{data}") - handle_message(data, tubesock) + @output << data if @output.size + data.size <= max_output_buffer_size end - socket.on :exit do |_exit_code| + socket.on :stdout do |data| + tubesock.send_data(JSON.dump({cmd: :write, stream: :stdout, data: data})) + end + + socket.on :stderr do |data| + tubesock.send_data(JSON.dump({cmd: :write, stream: :stderr, data: data})) + end + + socket.on :exit do |exit_code| EventMachine.stop_event_loop - tubesock.send_data JSON.dump({'cmd' => 'timeout'}) if container.status == 'timeouted' + status = container.status + if status == :timeouted + tubesock.send_data JSON.dump({cmd: :timeout}) + @output = "timeout: #{@output}" + elsif @output.empty? + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.no_output', timestamp: l(Time.now, format: :short))}) + end + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.exit', exit_code: exit_code)}) unless status == :timeouted kill_socket(tubesock) end - tubesock.onmessage do |data| - Rails.logger.info("#{Time.now.getutc.to_s}: Client sending: #{data}") - # Check whether the client send a JSON command and kill container - # if the command is 'client_kill', send it to docker otherwise. + tubesock.onmessage do |event| begin - - parsed = JSON.parse(data) unless data == "\n" - if parsed.instance_of?(Hash) && parsed['cmd'] == 'client_kill' - Rails.logger.debug("Client exited container.") + event = JSON.parse(event).deep_symbolize_keys + case event[:cmd].to_sym + when :client_kill + EventMachine.stop_event_loop + kill_socket(tubesock) container.destroy + Rails.logger.debug('Client exited container.') + when :result + socket.send event[:data] else - socket.send data - Rails.logger.debug { "Sent the received client data to docker:#{data}" } + Rails.logger.info("Unknown command from client: #{event[:cmd]}") end rescue JSON::ParserError => error - socket.send data - Rails.logger.debug { "Rescued parsing error, sent the received client data to docker:#{data}" } + Rails.logger.debug { "Data received from client is not valid json: #{data}" } + Sentry.set_extras(data: data) + rescue TypeError => error + Rails.logger.debug { "JSON data received from client cannot be parsed to hash: #{data}" } Sentry.set_extras(data: data) end end @@ -182,14 +198,17 @@ class SubmissionsController < ApplicationController kill_socket(tubesock) return end - @container_request_time = Time.zone.now - @submission.run(sanitize_filename) do |container, socket| + @container_execution_time = @submission.run(sanitize_filename) do |container, socket| + @waiting_for_container_time = container.waiting_time handle_websockets(tubesock, container, socket) end end + # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb) + save_run_output ensure ActiveRecord::Base.connection_pool.release_connection end + # TODO determine if this is necessary # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? # Thread.new do # EventMachine.run @@ -200,128 +219,36 @@ class SubmissionsController < ApplicationController end def kill_socket(tubesock) - @container_execution_time = Time.zone.now - @execution_request_time if @execution_request_time.present? # search for errors and save them as StructuredError (for scoring runs see submission_scoring.rb) errors = extract_errors send_hints(tubesock, errors) - # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb) - save_run_output - - # For Python containers, the @run_output is '{"cmd":"exit"}' as a string. - # If this is the case, we should consider it as blank - if @run_output.blank? || @run_output&.strip == '{"cmd":"exit"}' || @run_output&.strip == 'timeout:' - @raw_output ||= '' - @run_output ||= '' - parse_message t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short)), 'stdout', tubesock - end - # Hijacked connection needs to be notified correctly - tubesock.send_data JSON.dump({'cmd' => 'exit'}) + tubesock.send_data JSON.dump({cmd: :exit}) tubesock.close end - def handle_message(message, tubesock) - @raw_output ||= '' - @run_output ||= '' - # Handle special commands first - case message - when /^#exit|{"cmd": "exit"}/ - # Just call exit_container on the docker_client. - # Do not call kill_socket for the websocket to the client here. - # @docker_client.exit_container closes the socket to the container, - # kill_socket is called in the "on close handler" of the websocket to the container - # container.destroy - when /^#timeout/ - @run_output = "timeout: #{@run_output}" # add information that this run timed out to the buffer - else - # Filter out information about run_command, test_command, user or working directory - run_command = @submission.execution_environment.run_command % command_substitutions(sanitize_filename) - test_command = @submission.execution_environment.test_command % command_substitutions(sanitize_filename) - if test_command.blank? - # If no test command is set, use the run_command for the RegEx below. Otherwise, no output will be displayed! - test_command = run_command - end - unless %r{root@|:/workspace|#{run_command}|#{test_command}|bash: cmd:canvasevent: command not found}.match?(message) - parse_message(message, 'stdout', tubesock) - end - end - end - - def parse_message(message, output_stream, socket, container = nil, recursive: true) - parsed = '' - begin - parsed = JSON.parse(message) - if parsed.instance_of?(Hash) && parsed.key?('cmd') - socket.send_data message - Rails.logger.info("parse_message sent: #{message}") - # container.destroy if container && parsed['cmd'] == 'exit' - else - parsed = {'cmd' => 'write', 'stream' => output_stream, 'data' => message} - socket.send_data JSON.dump(parsed) - Rails.logger.info("parse_message sent: #{JSON.dump(parsed)}") - end - rescue JSON::ParserError - # Check wether the message contains multiple lines, if true try to parse each line - if recursive && message.include?("\n") - message.split("\n").each do |part| - parse_message(part, output_stream, socket, container, recursive: false) - end - elsif message.include?('') - @buffer += message - parsed = {'cmd' => 'write', 'stream' => output_stream, 'data' => @buffer} - socket.send_data JSON.dump(parsed) - # socket.send_data @buffer - @buffering = false - # Rails.logger.info('Sent complete buffer') - elsif @buffering && message.end_with?("}\r") - @buffer += message - socket.send_data @buffer - @buffering = false - # Rails.logger.info('Sent complete buffer') - elsif @buffering - @buffer += message - # Rails.logger.info('Appending to buffer') - else - # Rails.logger.info('else') - parsed = {'cmd' => 'write', 'stream' => output_stream, 'data' => message} - socket.send_data JSON.dump(parsed) - Rails.logger.info("parse_message sent: #{JSON.dump(parsed)}") - end - ensure - @raw_output += parsed['data'].to_s if parsed.instance_of?(Hash) && parsed.key?('data') - # save the data that was send to the run_output if there is enough space left. this will be persisted as a testrun with cause "run" - @run_output += JSON.dump(parsed).to_s if @run_output.size <= max_run_output_buffer_size - end - end - def save_run_output - if @run_output.present? - @run_output = @run_output[(0..max_run_output_buffer_size - 1)] # trim the string to max_message_buffer_size chars - Testrun.create( - file: @file, - cause: 'run', - submission: @submission, - output: @run_output, - container_execution_time: @container_execution_time, - waiting_for_container_time: @waiting_for_container_time - ) - end + return if @output.blank? + + @output = @output[0, max_output_buffer_size] # trim the string to max_output_buffer_size chars + Testrun.create( + file: @file, + cause: 'run', + submission: @submission, + output: @output, + container_execution_time: @container_execution_time, + waiting_for_container_time: @waiting_for_container_time + ) end def extract_errors results = [] - if @raw_output.present? + if @output.present? @submission.exercise.execution_environment.error_templates.each do |template| pattern = Regexp.new(template.signature).freeze - if pattern.match(@raw_output) - results << StructuredError.create_from_template(template, @raw_output, @submission) + if pattern.match(@output) + results << StructuredError.create_from_template(template, @output, @submission) end end end @@ -386,22 +313,24 @@ class SubmissionsController < ApplicationController def statistics; end - def test - Thread.new do - hijack do |tubesock| - if @embed_options[:disable_run] - kill_socket(tubesock) - return - end - @container_request_time = Time.now - @submission.run_tests(sanitize_filename) do |container| - handle_websockets(tubesock, container) - end - end - ensure - ActiveRecord::Base.connection_pool.release_connection - end - end + # TODO is this needed? + # def test + # Thread.new do + # hijack do |tubesock| + # if @embed_options[:disable_run] + # kill_socket(tubesock) + # return + # end + # @container_request_time = Time.now + # @submission.run_tests(sanitize_filename) do |container| + # handle_websockets(tubesock, container) + # end + # end + # ensure + # ActiveRecord::Base.connection_pool.release_connection + # end + # end + def with_server_sent_events response.headers['Content-Type'] = 'text/event-stream' diff --git a/app/models/submission.rb b/app/models/submission.rb index 3c590389..7d94f400 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -147,7 +147,7 @@ class Submission < ApplicationRecord stdout = "" stderr = "" exit_code = 1 # default to error - container.execute_interactively(score_command) do |container, socket| + execution_time = container.execute_interactively(score_command) do |container, socket| socket.on :stderr do |data| stderr << data end @@ -161,8 +161,8 @@ class Submission < ApplicationRecord end output = { file_role: file.role, - waiting_for_container_time: 1, # TODO - container_execution_time: 1, # TODO + waiting_for_container_time: container.waiting_time, + container_execution_time: execution_time, status: (exit_code == 0) ? :ok : :failed, stdout: stdout, stderr: stderr, @@ -176,9 +176,11 @@ class Submission < ApplicationRecord def run(file, &block) run_command = command_for execution_environment.run_command, file + execution_time = 0 prepared_container do |container| - container.execute_interactively(run_command, &block) + execution_time = container.execute_interactively(run_command, &block) end + execution_time end private @@ -187,10 +189,8 @@ class Submission < ApplicationRecord request_time = Time.now container = Container.new(execution_environment, execution_environment.permitted_execution_time) container.copy_submission_files self - container_time = Time.now - waiting_for_container_time = Time.now - request_time + container.waiting_time = Time.now - request_time yield(container) if block_given? - execution_time = Time.now - container_time container.destroy end diff --git a/config/locales/de.yml b/config/locales/de.yml index d0131e12..29cf2eb4 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -397,6 +397,7 @@ de: hint: Hinweis no_files: Die Aufgabe umfasst noch keine sichtbaren Dateien. no_output: Die letzte Code-Ausführung terminierte am %{timestamp} ohne Ausgabe. + exit: Der Exit-Status war %{exit_code}. no_output_yet: Bisher existiert noch keine Ausgabe. output: Programm-Ausgabe passed_tests: Erfolgreiche Tests diff --git a/config/locales/en.yml b/config/locales/en.yml index 60a87cc5..41c37dfc 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -397,6 +397,7 @@ en: hint: Hint no_files: The exercise does not comprise visible files yet. no_output: The last code run finished on %{timestamp} without any output. + exit: The exit status was %{exit_code}. no_output_yet: There is no output yet. output: Program Output passed_tests: Passed Tests diff --git a/lib/container.rb b/lib/container.rb index a34cc53f..004f8653 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -4,6 +4,9 @@ require 'container_connection' class Container BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] + HEADERS = {"Content-Type" => "application/json"} + + attr_accessor :waiting_time def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" @@ -11,7 +14,7 @@ class Container if time_limit body[:time_limit] = time_limit end - response = Faraday.post(url, body.to_json, "Content-Type" => "application/json") + response = Faraday.post(url, body.to_json, HEADERS) response = parse response @container_id = response[:id] end @@ -19,7 +22,7 @@ class Container def copy_files(files) url = container_url + "/files" body = files.map{ |filename, content| { filename => content } } - Faraday.post(url, body.to_json, "Content-Type" => "application/json") + Faraday.post(url, body.to_json, HEADERS) end def copy_submission_files(submission) @@ -32,17 +35,19 @@ class Container def execute_command(command) url = container_url + "/execute" - response = Faraday.patch(url, {command: command}.to_json, "Content-Type" => "application/json") + response = Faraday.patch(url, {command: command}.to_json, HEADERS) response = parse response response end def execute_interactively(command) + starting_time = Time.now websocket_url = execute_command(command)[:websocket_url] EventMachine.run do socket = ContainerConnection.new(websocket_url) yield(self, socket) if block_given? end + Time.now - starting_time # execution time end def destroy @@ -50,7 +55,7 @@ class Container end def status - parse(Faraday.get(container_url))[:status] + parse(Faraday.get(container_url))[:status].to_sym end private From 28f8de1a930a2d7b05010bd78b5504b41e4be19f Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 1 Apr 2021 11:56:30 +0200 Subject: [PATCH 007/156] Implement workaroud for double render error Co-authored-by: Felix Auringer --- app/controllers/submissions_controller.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 04f0bfd8..358d353c 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -192,22 +192,22 @@ class SubmissionsController < ApplicationController end def run + # TODO do we need this thread? If so, how to fix double render? (to reproduce: remove .join and run) Thread.new do hijack do |tubesock| if @embed_options[:disable_run] kill_socket(tubesock) - return - end - @container_execution_time = @submission.run(sanitize_filename) do |container, socket| - @waiting_for_container_time = container.waiting_time - handle_websockets(tubesock, container, socket) + else + @container_execution_time = @submission.run(sanitize_filename) do |container, socket| + @waiting_for_container_time = container.waiting_time + handle_websockets(tubesock, container, socket) + end + save_run_output end end - # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb) - save_run_output ensure ActiveRecord::Base.connection_pool.release_connection - end + end.join # TODO determine if this is necessary # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? # Thread.new do @@ -228,6 +228,7 @@ class SubmissionsController < ApplicationController tubesock.close end + # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb) def save_run_output return if @output.blank? From c36ec447ff95183c02d63f149b7a91b5ab5e4d40 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 1 Apr 2021 13:18:29 +0200 Subject: [PATCH 008/156] Fix faulty API data format --- lib/container.rb | 4 ++-- lib/container_connection.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/container.rb b/lib/container.rb index 004f8653..68e443c4 100644 --- a/lib/container.rb +++ b/lib/container.rb @@ -21,14 +21,14 @@ class Container def copy_files(files) url = container_url + "/files" - body = files.map{ |filename, content| { filename => content } } + body = { files: files.map{ |filename, content| { filename: filename, content: content } } } Faraday.post(url, body.to_json, HEADERS) end def copy_submission_files(submission) files = {} submission.collect_files.each do |file| - files[file.name] = file.content + files[file.name_with_extension] = file.content end copy_files(files) end diff --git a/lib/container_connection.rb b/lib/container_connection.rb index f48c1b28..ec8fb23f 100644 --- a/lib/container_connection.rb +++ b/lib/container_connection.rb @@ -4,7 +4,7 @@ class ContainerConnection EVENTS = %i[start output exit stdout stderr].freeze def initialize(url) - @socket = Faye::WebSocket::Client.new(url, [], ping: 0.1) + @socket = Faye::WebSocket::Client.new(url, [], ping: 5) %i[open message error close].each do |event_type| @socket.on event_type do |event| __send__(:"on_#{event_type}", event) end From cf1e4d6edff30e211f1ad6aa0f2886766a539f8d Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 1 Apr 2021 16:11:56 +0200 Subject: [PATCH 009/156] Rename API routes --- app/controllers/submissions_controller.rb | 4 +-- app/models/submission.rb | 2 +- lib/{container.rb => runner.rb} | 26 +++++++++---------- ...ner_connection.rb => runner_connection.rb} | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) rename lib/{container.rb => runner.rb} (69%) rename lib/{container_connection.rb => runner_connection.rb} (98%) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 358d353c..a3e2add7 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -161,9 +161,9 @@ class SubmissionsController < ApplicationController tubesock.send_data JSON.dump({cmd: :timeout}) @output = "timeout: #{@output}" elsif @output.empty? - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.no_output', timestamp: l(Time.now, format: :short))}) + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.no_output', timestamp: l(Time.now, format: :short)) + "\n"}) end - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.exit', exit_code: exit_code)}) unless status == :timeouted + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.exit', exit_code: exit_code) + "\n"}) unless status == :timeouted kill_socket(tubesock) end diff --git a/app/models/submission.rb b/app/models/submission.rb index 7d94f400..5a5ed14e 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -187,7 +187,7 @@ class Submission < ApplicationRecord def prepared_container request_time = Time.now - container = Container.new(execution_environment, execution_environment.permitted_execution_time) + container = Runner.new(execution_environment, execution_environment.permitted_execution_time) container.copy_submission_files self container.waiting_time = Time.now - request_time yield(container) if block_given? diff --git a/lib/container.rb b/lib/runner.rb similarity index 69% rename from lib/container.rb rename to lib/runner.rb index 68e443c4..4ef77a8a 100644 --- a/lib/container.rb +++ b/lib/runner.rb @@ -1,26 +1,26 @@ # frozen_string_literal: true -require 'container_connection' +require 'runner_connection' -class Container +class Runner BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] HEADERS = {"Content-Type" => "application/json"} attr_accessor :waiting_time def initialize(execution_environment, time_limit = nil) - url = "#{BASE_URL}/execution-environments/#{execution_environment.id}/containers/create" - body = {} + url = "#{BASE_URL}/runners" + body = {execution_environment_id: execution_environment.id} if time_limit body[:time_limit] = time_limit end response = Faraday.post(url, body.to_json, HEADERS) response = parse response - @container_id = response[:id] + @id = response[:id] end def copy_files(files) - url = container_url + "/files" + url = runner_url + "/files" body = { files: files.map{ |filename, content| { filename: filename, content: content } } } Faraday.post(url, body.to_json, HEADERS) end @@ -34,8 +34,8 @@ class Container end def execute_command(command) - url = container_url + "/execute" - response = Faraday.patch(url, {command: command}.to_json, HEADERS) + url = runner_url + "/execute" + response = Faraday.post(url, {command: command}.to_json, HEADERS) response = parse response response end @@ -44,24 +44,24 @@ class Container starting_time = Time.now websocket_url = execute_command(command)[:websocket_url] EventMachine.run do - socket = ContainerConnection.new(websocket_url) + socket = RunnerConnection.new(websocket_url) yield(self, socket) if block_given? end Time.now - starting_time # execution time end def destroy - Faraday.delete container_url + Faraday.delete runner_url end def status - parse(Faraday.get(container_url))[:status].to_sym + parse(Faraday.get(runner_url))[:status].to_sym end private - def container_url - "#{BASE_URL}/containers/#{@container_id}" + def runner_url + "#{BASE_URL}/runners/#{@id}" end def parse(response) diff --git a/lib/container_connection.rb b/lib/runner_connection.rb similarity index 98% rename from lib/container_connection.rb rename to lib/runner_connection.rb index ec8fb23f..25008d66 100644 --- a/lib/container_connection.rb +++ b/lib/runner_connection.rb @@ -1,6 +1,6 @@ require 'faye/websocket/client' -class ContainerConnection +class RunnerConnection EVENTS = %i[start output exit stdout stderr].freeze def initialize(url) From 6e9562c9e145a2b9de393ea03ff3d25a1506614a Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 6 Apr 2021 09:43:33 +0200 Subject: [PATCH 010/156] Validate json --- Gemfile | 1 + Gemfile.lock | 10 ++++++ lib/runner/backend-output.schema.json | 44 +++++++++++++++++++++++++++ lib/{ => runner}/runner.rb | 18 +++++------ lib/{ => runner}/runner_connection.rb | 6 ++++ 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 lib/runner/backend-output.schema.json rename lib/{ => runner}/runner.rb (74%) rename lib/{ => runner}/runner_connection.rb (84%) diff --git a/Gemfile b/Gemfile index 92afc60f..073f6d8a 100644 --- a/Gemfile +++ b/Gemfile @@ -17,6 +17,7 @@ gem 'i18n-js' gem 'ims-lti', '< 2.0.0' gem 'jbuilder' gem 'js-routes' +gem 'json_schemer' gem 'kramdown' gem 'mimemagic' gem 'nokogiri' diff --git a/Gemfile.lock b/Gemfile.lock index d9309fff..f0efcde2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -157,6 +157,8 @@ GEM multi_json domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + ecma-re-validator (0.3.0) + regexp_parser (~> 2.0) erubi (1.10.0) eventmachine (1.2.7) excon (0.87.0) @@ -194,6 +196,7 @@ GEM haml (5.2.2) temple (>= 0.8.0) tilt + hana (1.3.7) hashdiff (1.0.1) headless (2.3.1) highline (2.0.3) @@ -222,6 +225,11 @@ GEM js-routes (2.1.1) railties (>= 4) json (2.3.1) + json_schemer (0.2.18) + ecma-re-validator (~> 0.3) + hana (~> 1.3) + regexp_parser (~> 2.0) + uri_template (~> 0.7) jwt (2.3.0) kaminari (1.2.1) activesupport (>= 4.1.0) @@ -507,6 +515,7 @@ GEM unf_ext unf_ext (0.0.8) unicode-display_width (2.1.0) + uri_template (0.7.0) web-console (4.1.0) actionview (>= 6.0.0) activemodel (>= 6.0.0) @@ -561,6 +570,7 @@ DEPENDENCIES ims-lti (< 2.0.0) jbuilder js-routes + json_schemer kramdown listen mimemagic diff --git a/lib/runner/backend-output.schema.json b/lib/runner/backend-output.schema.json new file mode 100644 index 00000000..a256846e --- /dev/null +++ b/lib/runner/backend-output.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/schema#", + "title": "event", + "type": "object", + "oneOf": [ + { + "properties": { + "type": { + "const": "exit", + "required": true + }, + "data": { + "type": "integer", + "required": true, + "minimum": 0, + "maximum": 255 + } + }, + "additionalProperties": false + }, + { + "properties": { + "type": { + "enum": [ "stdout", "stderr", "error" ], + "required": true + }, + "data": { + "type": "string", + "required": true + } + }, + "additionalProperties": false + }, + { + "properties": { + "type": { + "enum": [ "start", "timeout" ], + "required": true + } + }, + "additionalProperties": false + } + ] +} \ No newline at end of file diff --git a/lib/runner.rb b/lib/runner/runner.rb similarity index 74% rename from lib/runner.rb rename to lib/runner/runner.rb index 4ef77a8a..aa5fa7bc 100644 --- a/lib/runner.rb +++ b/lib/runner/runner.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'runner_connection' - class Runner BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] HEADERS = {"Content-Type" => "application/json"} @@ -10,19 +8,19 @@ class Runner def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/runners" - body = {execution_environment_id: execution_environment.id} + body = {executionEnvironmentId: execution_environment.id} if time_limit - body[:time_limit] = time_limit + body[:timeLimit] = time_limit end response = Faraday.post(url, body.to_json, HEADERS) response = parse response - @id = response[:id] + @id = response[:runnerId] end def copy_files(files) url = runner_url + "/files" - body = { files: files.map{ |filename, content| { filename: filename, content: content } } } - Faraday.post(url, body.to_json, HEADERS) + body = { files: files.map { |filename, content| { filepath: filename, content: content } } } + Faraday.patch(url, body.to_json, HEADERS) end def copy_submission_files(submission) @@ -42,7 +40,7 @@ class Runner def execute_interactively(command) starting_time = Time.now - websocket_url = execute_command(command)[:websocket_url] + websocket_url = execute_command(command)[:websocketUrl] EventMachine.run do socket = RunnerConnection.new(websocket_url) yield(self, socket) if block_given? @@ -55,7 +53,9 @@ class Runner end def status - parse(Faraday.get(runner_url))[:status].to_sym + # parse(Faraday.get(runner_url))[:status].to_sym + # TODO return actual state retrieved via websocket + :timeouted end private diff --git a/lib/runner_connection.rb b/lib/runner/runner_connection.rb similarity index 84% rename from lib/runner_connection.rb rename to lib/runner/runner_connection.rb index 25008d66..415c282b 100644 --- a/lib/runner_connection.rb +++ b/lib/runner/runner_connection.rb @@ -1,7 +1,9 @@ require 'faye/websocket/client' +require 'json_schemer' class RunnerConnection EVENTS = %i[start output exit stdout stderr].freeze + BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read("lib/runner/backend-output.schema.json"))) def initialize(url) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) @@ -36,7 +38,11 @@ class RunnerConnection end def on_message(event) + return unless BACKEND_OUTPUT_SCHEMA.valid?(JSON.parse(event.data)) + event = decode(event.data) + + # TODO handle other events like timeout case event[:type].to_sym when :exit_code @exit_code = event[:data] From 80932c0c4070afbf730802a7ff058a40fabf14e7 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 6 Apr 2021 13:51:11 +0200 Subject: [PATCH 011/156] Auto-correct linting issues --- app/controllers/submissions_controller.rb | 63 ++++++++++------------- lib/runner/runner.rb | 19 +++---- lib/runner/runner_connection.rb | 21 ++++---- 3 files changed, 46 insertions(+), 57 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index a3e2add7..27d7a5d6 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -68,9 +68,7 @@ class SubmissionsController < ApplicationController end def download - if @embed_options[:disable_download] - raise Pundit::NotAuthorizedError - end + raise Pundit::NotAuthorizedError if @embed_options[:disable_download] # files = @submission.files.map{ } # zipline( files, 'submission.zip') @@ -112,9 +110,7 @@ class SubmissionsController < ApplicationController end def download_file - if @embed_options[:disable_download] - raise Pundit::NotAuthorizedError - end + raise Pundit::NotAuthorizedError if @embed_options[:disable_download] if @file.native_file? send_file(@file.native_file.path) @@ -142,7 +138,7 @@ class SubmissionsController < ApplicationController @output = '' socket.on :output do |data| - Rails.logger.info("#{Time.zone.now.getutc}: Docker sending: #{data}") + Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data}") @output << data if @output.size + data.size <= max_output_buffer_size end @@ -161,38 +157,36 @@ class SubmissionsController < ApplicationController tubesock.send_data JSON.dump({cmd: :timeout}) @output = "timeout: #{@output}" elsif @output.empty? - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.no_output', timestamp: l(Time.now, format: :short)) + "\n"}) + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) end - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: t('exercises.implement.exit', exit_code: exit_code) + "\n"}) unless status == :timeouted + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) unless status == :timeouted kill_socket(tubesock) end tubesock.onmessage do |event| - begin - event = JSON.parse(event).deep_symbolize_keys - case event[:cmd].to_sym - when :client_kill - EventMachine.stop_event_loop - kill_socket(tubesock) - container.destroy - Rails.logger.debug('Client exited container.') - when :result - socket.send event[:data] - else - Rails.logger.info("Unknown command from client: #{event[:cmd]}") - end - rescue JSON::ParserError => error - Rails.logger.debug { "Data received from client is not valid json: #{data}" } - Sentry.set_extras(data: data) - rescue TypeError => error - Rails.logger.debug { "JSON data received from client cannot be parsed to hash: #{data}" } - Sentry.set_extras(data: data) + event = JSON.parse(event).deep_symbolize_keys + case event[:cmd].to_sym + when :client_kill + EventMachine.stop_event_loop + kill_socket(tubesock) + container.destroy + Rails.logger.debug('Client exited container.') + when :result + socket.send event[:data] + else + Rails.logger.info("Unknown command from client: #{event[:cmd]}") end + rescue JSON::ParserError => e + ails.logger.debug { "Data received from client is not valid json: #{data}" } + Sentry.set_extras(data: data) + rescue TypeError => e + Rails.logger.debug { "JSON data received from client cannot be parsed to hash: #{data}" } + Sentry.set_extras(data: data) end end def run - # TODO do we need this thread? If so, how to fix double render? (to reproduce: remove .join and run) + # TODO: do we need this thread? If so, how to fix double render? (to reproduce: remove .join and run) Thread.new do hijack do |tubesock| if @embed_options[:disable_run] @@ -208,7 +202,7 @@ class SubmissionsController < ApplicationController ensure ActiveRecord::Base.connection_pool.release_connection end.join - # TODO determine if this is necessary + # TODO: determine if this is necessary # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? # Thread.new do # EventMachine.run @@ -248,9 +242,7 @@ class SubmissionsController < ApplicationController if @output.present? @submission.exercise.execution_environment.error_templates.each do |template| pattern = Regexp.new(template.signature).freeze - if pattern.match(@output) - results << StructuredError.create_from_template(template, @output, @submission) - end + results << StructuredError.create_from_template(template, @output, @submission) if pattern.match(@output) end end results @@ -288,7 +280,7 @@ class SubmissionsController < ApplicationController # private :set_docker_client def set_file - @file = @files.detect {|file| file.name_with_extension == sanitize_filename } + @file = @files.detect { |file| file.name_with_extension == sanitize_filename } head :not_found unless @file end private :set_file @@ -314,7 +306,7 @@ class SubmissionsController < ApplicationController def statistics; end - # TODO is this needed? + # TODO: is this needed? # def test # Thread.new do # hijack do |tubesock| @@ -332,7 +324,6 @@ class SubmissionsController < ApplicationController # end # end - def with_server_sent_events response.headers['Content-Type'] = 'text/event-stream' server_sent_event = SSE.new(response.stream) diff --git a/lib/runner/runner.rb b/lib/runner/runner.rb index aa5fa7bc..dbaa8a5c 100644 --- a/lib/runner/runner.rb +++ b/lib/runner/runner.rb @@ -2,24 +2,22 @@ class Runner BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] - HEADERS = {"Content-Type" => "application/json"} + HEADERS = {'Content-Type' => 'application/json'}.freeze attr_accessor :waiting_time def initialize(execution_environment, time_limit = nil) url = "#{BASE_URL}/runners" body = {executionEnvironmentId: execution_environment.id} - if time_limit - body[:timeLimit] = time_limit - end + body[:timeLimit] = time_limit if time_limit response = Faraday.post(url, body.to_json, HEADERS) response = parse response @id = response[:runnerId] end def copy_files(files) - url = runner_url + "/files" - body = { files: files.map { |filename, content| { filepath: filename, content: content } } } + url = "#{runner_url}/files" + body = {files: files.map { |filename, content| {filepath: filename, content: content} }} Faraday.patch(url, body.to_json, HEADERS) end @@ -32,20 +30,19 @@ class Runner end def execute_command(command) - url = runner_url + "/execute" + url = "#{runner_url}/execute" response = Faraday.post(url, {command: command}.to_json, HEADERS) - response = parse response - response + parse response end def execute_interactively(command) - starting_time = Time.now + starting_time = Time.zone.now websocket_url = execute_command(command)[:websocketUrl] EventMachine.run do socket = RunnerConnection.new(websocket_url) yield(self, socket) if block_given? end - Time.now - starting_time # execution time + Time.zone.now - starting_time # execution time end def destroy diff --git a/lib/runner/runner_connection.rb b/lib/runner/runner_connection.rb index 415c282b..ae654dd1 100644 --- a/lib/runner/runner_connection.rb +++ b/lib/runner/runner_connection.rb @@ -1,19 +1,21 @@ +# frozen_string_literal: true + require 'faye/websocket/client' require 'json_schemer' class RunnerConnection EVENTS = %i[start output exit stdout stderr].freeze - BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read("lib/runner/backend-output.schema.json"))) + BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) def initialize(url) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) %i[open message error close].each do |event_type| - @socket.on event_type do |event| __send__(:"on_#{event_type}", event) end + @socket.on(event_type) { |event| __send__(:"on_#{event_type}", event) } end - EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", lambda {|e|}) } - @start_callback = lambda {} + EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", ->(e) {}) } + @start_callback = -> {} @exit_code = 0 end @@ -42,7 +44,7 @@ class RunnerConnection event = decode(event.data) - # TODO handle other events like timeout + # TODO: handle other events like timeout case event[:type].to_sym when :exit_code @exit_code = event[:data] @@ -57,14 +59,13 @@ class RunnerConnection end end - def on_open(event) + def on_open(_event) @start_callback.call end - def on_error(event) - end + def on_error(event); end - def on_close(event) + def on_close(_event) @exit_callback.call @exit_code end -end \ No newline at end of file +end From 575057acd3dcc323d0023adfd2609c00fe3b723f Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Wed, 7 Apr 2021 14:43:34 +0200 Subject: [PATCH 012/156] Fix some non-autocorrectable linting issues Many functions in submission_controller.rb still are very long and have a high complexity. Because the logic for handling execution of submissions will probably move elsewhere (when switching to ActionCable), this is fine for now. --- app/controllers/submissions_controller.rb | 63 +++++++++-------------- lib/runner/runner.rb | 3 +- lib/runner/runner_connection.rb | 37 ++++++++----- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 27d7a5d6..9f5b7a9d 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -10,7 +10,6 @@ class SubmissionsController < ApplicationController before_action :set_submission, only: %i[download download_file render_file run score extract_errors show statistics test] - # before_action :set_docker_client, only: %i[run test] before_action :set_files, only: %i[download download_file render_file show run] before_action :set_file, only: %i[download_file render_file run] before_action :set_mime_type, only: %i[download_file render_file] @@ -135,7 +134,7 @@ class SubmissionsController < ApplicationController def handle_websockets(tubesock, container, socket) tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) - @output = '' + @output = String.new socket.on :output do |data| Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data}") @@ -176,40 +175,27 @@ class SubmissionsController < ApplicationController else Rails.logger.info("Unknown command from client: #{event[:cmd]}") end - rescue JSON::ParserError => e - ails.logger.debug { "Data received from client is not valid json: #{data}" } + rescue JSON::ParserError + Rails.logger.debug { "Data received from client is not valid json: #{data}" } Sentry.set_extras(data: data) - rescue TypeError => e + rescue TypeError Rails.logger.debug { "JSON data received from client cannot be parsed to hash: #{data}" } Sentry.set_extras(data: data) end end def run - # TODO: do we need this thread? If so, how to fix double render? (to reproduce: remove .join and run) - Thread.new do - hijack do |tubesock| - if @embed_options[:disable_run] - kill_socket(tubesock) - else - @container_execution_time = @submission.run(sanitize_filename) do |container, socket| - @waiting_for_container_time = container.waiting_time - handle_websockets(tubesock, container, socket) - end - save_run_output + hijack do |tubesock| + if @embed_options[:disable_run] + kill_socket(tubesock) + else + @container_execution_time = @submission.run(sanitize_filename) do |container, socket| + @waiting_for_container_time = container.waiting_time + handle_websockets(tubesock, container, socket) end + save_run_output end - ensure - ActiveRecord::Base.connection_pool.release_connection - end.join - # TODO: determine if this is necessary - # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - # Thread.new do - # EventMachine.run - # ensure - # ActiveRecord::Base.connection_pool.release_connection - # end - # end + end end def kill_socket(tubesock) @@ -306,21 +292,20 @@ class SubmissionsController < ApplicationController def statistics; end - # TODO: is this needed? + # TODO: make this run, but with the test command # def test - # Thread.new do - # hijack do |tubesock| - # if @embed_options[:disable_run] - # kill_socket(tubesock) - # return - # end - # @container_request_time = Time.now - # @submission.run_tests(sanitize_filename) do |container| - # handle_websockets(tubesock, container) + # hijack do |tubesock| + # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? + # Thread.new do + # EventMachine.run + # ensure + # ActiveRecord::Base.connection_pool.release_connection # end # end - # ensure - # ActiveRecord::Base.connection_pool.release_connection + # output = @docker_client.execute_test_command(@submission, sanitize_filename) + # # tubesock is the socket to the client + # tubesock.send_data JSON.dump(output) + # tubesock.send_data JSON.dump('cmd' => 'exit') # end # end diff --git a/lib/runner/runner.rb b/lib/runner/runner.rb index dbaa8a5c..5e850e9e 100644 --- a/lib/runner/runner.rb +++ b/lib/runner/runner.rb @@ -50,8 +50,7 @@ class Runner end def status - # parse(Faraday.get(runner_url))[:status].to_sym - # TODO return actual state retrieved via websocket + # TODO: return actual state retrieved via websocket :timeouted end diff --git a/lib/runner/runner_connection.rb b/lib/runner/runner_connection.rb index ae654dd1..4f692edd 100644 --- a/lib/runner/runner_connection.rb +++ b/lib/runner/runner_connection.rb @@ -43,20 +43,7 @@ class RunnerConnection return unless BACKEND_OUTPUT_SCHEMA.valid?(JSON.parse(event.data)) event = decode(event.data) - - # TODO: handle other events like timeout - case event[:type].to_sym - when :exit_code - @exit_code = event[:data] - when :stderr - @stderr_callback.call event[:data] - @output_callback.call event[:data] - when :stdout - @stdout_callback.call event[:data] - @output_callback.call event[:data] - else - :error - end + __send__("handle_#{event[:type]}", event) end def on_open(_event) @@ -68,4 +55,26 @@ class RunnerConnection def on_close(_event) @exit_callback.call @exit_code end + + def handle_exit(event) + @exit_code = event[:data] + end + + def handle_stdout(event) + @stdout_callback.call event[:data] + @output_callback.call event[:data] + end + + def handle_stderr(event) + @stderr_callback.call event[:data] + @output_callback.call event[:data] + end + + def handle_error(event) end + + def handle_start(event) end + + def handle_timeout(event) + # TODO: set the runner state + end end From 3e6534567dee4897e08051b2f15a8a35d6bbd86e Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Wed, 7 Apr 2021 17:07:29 +0200 Subject: [PATCH 013/156] Move copy_submission_files from runner to submission --- app/models/submission.rb | 10 +++++++++- lib/runner/runner.rb | 8 -------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/models/submission.rb b/app/models/submission.rb index 5a5ed14e..6256b446 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -185,10 +185,18 @@ class Submission < ApplicationRecord private + def copy_files_to(container) + files = {} + collect_files.each do |file| + files[file.name_with_extension] = file.content + end + container.copy_files(files) + end + def prepared_container request_time = Time.now container = Runner.new(execution_environment, execution_environment.permitted_execution_time) - container.copy_submission_files self + copy_files_to container container.waiting_time = Time.now - request_time yield(container) if block_given? container.destroy diff --git a/lib/runner/runner.rb b/lib/runner/runner.rb index 5e850e9e..d8ae60b8 100644 --- a/lib/runner/runner.rb +++ b/lib/runner/runner.rb @@ -21,14 +21,6 @@ class Runner Faraday.patch(url, body.to_json, HEADERS) end - def copy_submission_files(submission) - files = {} - submission.collect_files.each do |file| - files[file.name_with_extension] = file.content - end - copy_files(files) - end - def execute_command(command) url = "#{runner_url}/execute" response = Faraday.post(url, {command: command}.to_json, HEADERS) From 2404c1c36c8ab3615d03ca1d76ecd53964c1d887 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Wed, 7 Apr 2021 17:12:26 +0200 Subject: [PATCH 014/156] Rename variables from container to runner --- app/controllers/submissions_controller.rb | 12 +++++------ app/models/submission.rb | 26 +++++++++++------------ config/code_ocean.yml.example | 2 ++ lib/runner/runner.rb | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 9f5b7a9d..34ad6f53 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -132,7 +132,7 @@ class SubmissionsController < ApplicationController end end - def handle_websockets(tubesock, container, socket) + def handle_websockets(tubesock, runner, socket) tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) @output = String.new @@ -151,7 +151,7 @@ class SubmissionsController < ApplicationController socket.on :exit do |exit_code| EventMachine.stop_event_loop - status = container.status + status = runner.status if status == :timeouted tubesock.send_data JSON.dump({cmd: :timeout}) @output = "timeout: #{@output}" @@ -168,7 +168,7 @@ class SubmissionsController < ApplicationController when :client_kill EventMachine.stop_event_loop kill_socket(tubesock) - container.destroy + runner.destroy Rails.logger.debug('Client exited container.') when :result socket.send event[:data] @@ -189,9 +189,9 @@ class SubmissionsController < ApplicationController if @embed_options[:disable_run] kill_socket(tubesock) else - @container_execution_time = @submission.run(sanitize_filename) do |container, socket| - @waiting_for_container_time = container.waiting_time - handle_websockets(tubesock, container, socket) + @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| + @waiting_for_container_time = runner.waiting_time + handle_websockets(tubesock, runner, socket) end save_run_output end diff --git a/app/models/submission.rb b/app/models/submission.rb index 6256b446..90addbf8 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -141,13 +141,13 @@ class Submission < ApplicationRecord def calculate_score score = nil - prepared_container do |container| + prepared_runner do |runner| scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension stdout = "" stderr = "" exit_code = 1 # default to error - execution_time = container.execute_interactively(score_command) do |container, socket| + execution_time = runner.execute_interactively(score_command) do |runner, socket| socket.on :stderr do |data| stderr << data end @@ -161,7 +161,7 @@ class Submission < ApplicationRecord end output = { file_role: file.role, - waiting_for_container_time: container.waiting_time, + waiting_for_container_time: runner.waiting_time, container_execution_time: execution_time, status: (exit_code == 0) ? :ok : :failed, stdout: stdout, @@ -177,29 +177,29 @@ class Submission < ApplicationRecord def run(file, &block) run_command = command_for execution_environment.run_command, file execution_time = 0 - prepared_container do |container| - execution_time = container.execute_interactively(run_command, &block) + prepared_runner do |runner| + execution_time = runner.execute_interactively(run_command, &block) end execution_time end private - def copy_files_to(container) + def copy_files_to(runner) files = {} collect_files.each do |file| files[file.name_with_extension] = file.content end - container.copy_files(files) + runner.copy_files(files) end - def prepared_container + def prepared_runner request_time = Time.now - container = Runner.new(execution_environment, execution_environment.permitted_execution_time) - copy_files_to container - container.waiting_time = Time.now - request_time - yield(container) if block_given? - container.destroy + runner = Runner.new(execution_environment, execution_environment.permitted_execution_time) + copy_files_to runner + runner.waiting_time = Time.now - request_time + yield(runner) if block_given? + runner.destroy end def command_for(template, file) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 72f7ed84..eb152f8f 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -21,6 +21,8 @@ development: url: https://codeharbor.openhpi.de prometheus_exporter: enabled: false + runner_management: + url: https://runners.example.org production: <<: *default diff --git a/lib/runner/runner.rb b/lib/runner/runner.rb index d8ae60b8..e304a7df 100644 --- a/lib/runner/runner.rb +++ b/lib/runner/runner.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Runner - BASE_URL = CodeOcean::Config.new(:code_ocean).read[:container_management][:url] + BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] HEADERS = {'Content-Type' => 'application/json'}.freeze attr_accessor :waiting_time From 3017e4600648eb02b1db2e7002cd737d7742f9b7 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 8 Apr 2021 08:32:48 +0000 Subject: [PATCH 015/156] Add newline to end of json schema --- lib/runner/backend-output.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/backend-output.schema.json b/lib/runner/backend-output.schema.json index a256846e..0bae4019 100644 --- a/lib/runner/backend-output.schema.json +++ b/lib/runner/backend-output.schema.json @@ -41,4 +41,4 @@ "additionalProperties": false } ] -} \ No newline at end of file +} From 17bd2d8726a6608694442fc04afdecb4e3c9c831 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Mon, 19 Apr 2021 08:54:05 +0200 Subject: [PATCH 016/156] Reuse runners per user and execution environment Co-authored-by: Jan-Eric Hellenberg Co-authored-by: Maximilian Pass --- app/controllers/submissions_controller.rb | 20 +++-- app/errors/application_error.rb | 2 + app/errors/runner_not_available_error.rb | 2 + app/jobs/application_job.rb | 3 + app/jobs/runner_cleanup_job.rb | 16 ++++ app/models/runner.rb | 96 +++++++++++++++++++++ app/models/submission.rb | 3 +- config/code_ocean.yml.example | 6 +- config/initializers/jobs.rb | 1 + db/migrate/20210415064948_create_runners.rb | 14 +++ db/schema.rb | 14 +++ lib/runner/runner.rb | 58 ------------- 12 files changed, 164 insertions(+), 71 deletions(-) create mode 100644 app/errors/application_error.rb create mode 100644 app/errors/runner_not_available_error.rb create mode 100644 app/jobs/application_job.rb create mode 100644 app/jobs/runner_cleanup_job.rb create mode 100644 app/models/runner.rb create mode 100644 config/initializers/jobs.rb create mode 100644 db/migrate/20210415064948_create_runners.rb delete mode 100644 lib/runner/runner.rb diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 34ad6f53..fe281b29 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -152,10 +152,7 @@ class SubmissionsController < ApplicationController socket.on :exit do |exit_code| EventMachine.stop_event_loop status = runner.status - if status == :timeouted - tubesock.send_data JSON.dump({cmd: :timeout}) - @output = "timeout: #{@output}" - elsif @output.empty? + if @output.empty? tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) end tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) unless status == :timeouted @@ -168,7 +165,6 @@ class SubmissionsController < ApplicationController when :client_kill EventMachine.stop_event_loop kill_socket(tubesock) - runner.destroy Rails.logger.debug('Client exited container.') when :result socket.send event[:data] @@ -189,11 +185,17 @@ class SubmissionsController < ApplicationController if @embed_options[:disable_run] kill_socket(tubesock) else - @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| - @waiting_for_container_time = runner.waiting_time - handle_websockets(tubesock, runner, socket) + begin + @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| + @waiting_for_container_time = runner.waiting_time + handle_websockets(tubesock, runner, socket) + end + save_run_output + rescue RunnerNotAvailableError + tubesock.send_data JSON.dump({cmd: :timeout}) + kill_socket(tubesock) + Rails.logger.debug('Runner not available') end - save_run_output end end end diff --git a/app/errors/application_error.rb b/app/errors/application_error.rb new file mode 100644 index 00000000..4adca849 --- /dev/null +++ b/app/errors/application_error.rb @@ -0,0 +1,2 @@ +class ApplicationError < StandardError +end \ No newline at end of file diff --git a/app/errors/runner_not_available_error.rb b/app/errors/runner_not_available_error.rb new file mode 100644 index 00000000..ac595539 --- /dev/null +++ b/app/errors/runner_not_available_error.rb @@ -0,0 +1,2 @@ +class RunnerNotAvailableError < ApplicationError +end \ No newline at end of file diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb new file mode 100644 index 00000000..06752606 --- /dev/null +++ b/app/jobs/application_job.rb @@ -0,0 +1,3 @@ +class ApplicationJob < ActiveJob::Base + queue_as :default +end \ No newline at end of file diff --git a/app/jobs/runner_cleanup_job.rb b/app/jobs/runner_cleanup_job.rb new file mode 100644 index 00000000..4a5c172f --- /dev/null +++ b/app/jobs/runner_cleanup_job.rb @@ -0,0 +1,16 @@ +class RunnerCleanupJob < ApplicationJob + CLEANUP_INTERVAL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:cleanup_interval].seconds + + after_perform do |job| + # re-schedule job + self.class.set(wait: CLEANUP_INTERVAL).perform_later + end + + def perform + Rails.logger.debug(Time.now) + Runner.inactive_runners.each do |runner| + Rails.logger.debug("Destroying runner #{runner.runner_id}, unused since #{runner.last_used}") + runner.destroy + end + end +end diff --git a/app/models/runner.rb b/app/models/runner.rb new file mode 100644 index 00000000..22e8bd85 --- /dev/null +++ b/app/models/runner.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'runner/runner_connection' + +class Runner < ApplicationRecord + BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] + HEADERS = {'Content-Type' => 'application/json'}.freeze + UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds + + belongs_to :execution_environment + belongs_to :user, polymorphic: true + + before_create :get_runner + before_destroy :destroy_runner + + validates :execution_environment_id, presence: true + validates :user, presence: true + validates :time_limit, presence: true + + scope :inactive_runners, -> { where('last_used < ?', Time.now - UNUSED_EXPIRATION_TIME) } + + def self.for(user, exercise, time_limit = 0) + execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) + runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment, time_limit: time_limit) + + return runner if runner.save + raise(RunnerNotAvailableError, 'No runner available') + end + + def copy_files(files) + url = "#{runner_url}/files" + body = {files: files.map { |filename, content| {filepath: filename, content: content} }} + response = Faraday.patch(url, body.to_json, HEADERS) + if response.status == 404 + # runner has disappeared for some reason + self.destroy + raise(RunnerNotAvailableError, "Runner unavailable") + end + end + + def execute_command(command) + url = "#{runner_url}/execute" + response = Faraday.post(url, {command: command}.to_json, HEADERS) + if response.status == 404 + # runner has disappeared for some reason + self.destroy + raise(RunnerNotAvailableError, "Runner unavailable") + end + used_now + parse response + end + + def execute_interactively(command) + starting_time = Time.zone.now + websocket_url = execute_command(command)[:websocketUrl] + EventMachine.run do + socket = RunnerConnection.new(websocket_url) + yield(self, socket) if block_given? + end + Time.zone.now - starting_time # execution time + end + + def destroy_runner + Faraday.delete runner_url + end + + def status + # TODO: return actual state retrieved via websocket + :timeouted + end + + private + + def get_runner + url = "#{BASE_URL}/runners" + body = {executionEnvironmentId: execution_environment.id} + body[:timeLimit] = time_limit + response = Faraday.post(url, body.to_json, HEADERS) + response_body = parse response + self.runner_id = response_body[:runnerId] + throw :abort unless response.status == 200 + end + + def runner_url + "#{BASE_URL}/runners/#{runner_id}" + end + + def parse(response) + JSON.parse(response.body).deep_symbolize_keys + end + + def used_now + self.last_used = Time.now + save + end +end diff --git a/app/models/submission.rb b/app/models/submission.rb index 90addbf8..7d14e87d 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -195,11 +195,10 @@ class Submission < ApplicationRecord def prepared_runner request_time = Time.now - runner = Runner.new(execution_environment, execution_environment.permitted_execution_time) + runner = Runner.for(user, exercise, execution_environment.permitted_execution_time) copy_files_to runner runner.waiting_time = Time.now - request_time yield(runner) if block_given? - runner.destroy end def command_for(template, file) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index eb152f8f..b2413754 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -8,6 +8,10 @@ default: &default enabled: false codeocean_events: enabled: false + runner_management: + url: https://runners.example.org + cleanup_interval: 60 + unused_runner_expiration_time: 180 development: flowr: @@ -21,8 +25,6 @@ development: url: https://codeharbor.openhpi.de prometheus_exporter: enabled: false - runner_management: - url: https://runners.example.org production: <<: *default diff --git a/config/initializers/jobs.rb b/config/initializers/jobs.rb new file mode 100644 index 00000000..40fe6ab9 --- /dev/null +++ b/config/initializers/jobs.rb @@ -0,0 +1 @@ +RunnerCleanupJob.perform_now unless Rake.application.top_level_tasks.to_s.include?('db:') diff --git a/db/migrate/20210415064948_create_runners.rb b/db/migrate/20210415064948_create_runners.rb new file mode 100644 index 00000000..3ebebb63 --- /dev/null +++ b/db/migrate/20210415064948_create_runners.rb @@ -0,0 +1,14 @@ +class CreateRunners < ActiveRecord::Migration[5.2] + def change + create_table :runners do |t| + t.string :runner_id + t.references :execution_environment + t.references :user, polymorphic: true + t.integer :time_limit + t.float :waiting_time + t.datetime :last_used + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index d6e6d60e..b899981c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -339,6 +339,20 @@ ActiveRecord::Schema.define(version: 2021_05_12_133612) do t.index ["user_id", "user_type", "created_at"], name: "index_rfc_on_user_and_created_at", order: { created_at: :desc } end + create_table "runners", force: :cascade do |t| + t.string "runner_id" + t.bigint "execution_environment_id" + t.string "user_type" + t.bigint "user_id" + t.integer "time_limit" + t.float "waiting_time" + t.datetime "last_used" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["execution_environment_id"], name: "index_runners_on_execution_environment_id" + t.index ["user_type", "user_id"], name: "index_runners_on_user_type_and_user_id" + end + create_table "searches", id: :serial, force: :cascade do |t| t.integer "exercise_id", null: false t.integer "user_id", null: false diff --git a/lib/runner/runner.rb b/lib/runner/runner.rb deleted file mode 100644 index e304a7df..00000000 --- a/lib/runner/runner.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -class Runner - BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] - HEADERS = {'Content-Type' => 'application/json'}.freeze - - attr_accessor :waiting_time - - def initialize(execution_environment, time_limit = nil) - url = "#{BASE_URL}/runners" - body = {executionEnvironmentId: execution_environment.id} - body[:timeLimit] = time_limit if time_limit - response = Faraday.post(url, body.to_json, HEADERS) - response = parse response - @id = response[:runnerId] - end - - def copy_files(files) - url = "#{runner_url}/files" - body = {files: files.map { |filename, content| {filepath: filename, content: content} }} - Faraday.patch(url, body.to_json, HEADERS) - end - - def execute_command(command) - url = "#{runner_url}/execute" - response = Faraday.post(url, {command: command}.to_json, HEADERS) - parse response - end - - def execute_interactively(command) - starting_time = Time.zone.now - websocket_url = execute_command(command)[:websocketUrl] - EventMachine.run do - socket = RunnerConnection.new(websocket_url) - yield(self, socket) if block_given? - end - Time.zone.now - starting_time # execution time - end - - def destroy - Faraday.delete runner_url - end - - def status - # TODO: return actual state retrieved via websocket - :timeouted - end - - private - - def runner_url - "#{BASE_URL}/runners/#{@id}" - end - - def parse(response) - JSON.parse(response.body).deep_symbolize_keys - end -end From c14cf99a9605f3c0953f6988a6de48c8b4e54f38 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Mon, 19 Apr 2021 10:05:14 +0200 Subject: [PATCH 017/156] Don't cleanup runners during precompile --- config/initializers/jobs.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/jobs.rb b/config/initializers/jobs.rb index 40fe6ab9..68eb2e92 100644 --- a/config/initializers/jobs.rb +++ b/config/initializers/jobs.rb @@ -1 +1 @@ -RunnerCleanupJob.perform_now unless Rake.application.top_level_tasks.to_s.include?('db:') +RunnerCleanupJob.perform_now unless Rake.application.top_level_tasks.to_s.match?(/db:|assets:/) From b29bc5e70f79c6da9afeb4bd0f8ed7cbe1d6fb93 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Wed, 21 Apr 2021 06:01:55 +0000 Subject: [PATCH 018/156] Apply 3 suggestion(s) to 3 file(s) --- app/errors/application_error.rb | 2 +- app/errors/runner_not_available_error.rb | 2 +- app/jobs/application_job.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/errors/application_error.rb b/app/errors/application_error.rb index 4adca849..abcd35ed 100644 --- a/app/errors/application_error.rb +++ b/app/errors/application_error.rb @@ -1,2 +1,2 @@ class ApplicationError < StandardError -end \ No newline at end of file +end diff --git a/app/errors/runner_not_available_error.rb b/app/errors/runner_not_available_error.rb index ac595539..08d651d8 100644 --- a/app/errors/runner_not_available_error.rb +++ b/app/errors/runner_not_available_error.rb @@ -1,2 +1,2 @@ class RunnerNotAvailableError < ApplicationError -end \ No newline at end of file +end diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 06752606..9b7f20f2 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,3 +1,3 @@ class ApplicationJob < ActiveJob::Base queue_as :default -end \ No newline at end of file +end From 286a3f394d6dc545bc1c3760384a566d49fb2dca Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Thu, 22 Apr 2021 08:58:50 +0200 Subject: [PATCH 019/156] Fix autocorrectable rubocop offences and implement suggestions --- app/errors/application_error.rb | 2 ++ app/errors/runner_not_available_error.rb | 2 ++ app/jobs/application_job.rb | 2 ++ app/jobs/runner_cleanup_job.rb | 6 ++-- app/models/runner.rb | 37 ++++++++++++------------ app/models/submission.rb | 22 +++++++------- config/initializers/jobs.rb | 2 ++ 7 files changed, 42 insertions(+), 31 deletions(-) diff --git a/app/errors/application_error.rb b/app/errors/application_error.rb index abcd35ed..6056921c 100644 --- a/app/errors/application_error.rb +++ b/app/errors/application_error.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class ApplicationError < StandardError end diff --git a/app/errors/runner_not_available_error.rb b/app/errors/runner_not_available_error.rb index 08d651d8..4390fad4 100644 --- a/app/errors/runner_not_available_error.rb +++ b/app/errors/runner_not_available_error.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class RunnerNotAvailableError < ApplicationError end diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 9b7f20f2..97fc265f 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ApplicationJob < ActiveJob::Base queue_as :default end diff --git a/app/jobs/runner_cleanup_job.rb b/app/jobs/runner_cleanup_job.rb index 4a5c172f..f578e9d5 100644 --- a/app/jobs/runner_cleanup_job.rb +++ b/app/jobs/runner_cleanup_job.rb @@ -1,13 +1,15 @@ +# frozen_string_literal: true + class RunnerCleanupJob < ApplicationJob CLEANUP_INTERVAL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:cleanup_interval].seconds - after_perform do |job| + after_perform do |_job| # re-schedule job self.class.set(wait: CLEANUP_INTERVAL).perform_later end def perform - Rails.logger.debug(Time.now) + Rails.logger.debug(Time.zone.now) Runner.inactive_runners.each do |runner| Rails.logger.debug("Destroying runner #{runner.runner_id}, unused since #{runner.last_used}") runner.destroy diff --git a/app/models/runner.rb b/app/models/runner.rb index 22e8bd85..653bc886 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -10,20 +10,22 @@ class Runner < ApplicationRecord belongs_to :execution_environment belongs_to :user, polymorphic: true - before_create :get_runner + before_create :new_runner before_destroy :destroy_runner - validates :execution_environment_id, presence: true + validates :execution_environment, presence: true validates :user, presence: true validates :time_limit, presence: true - scope :inactive_runners, -> { where('last_used < ?', Time.now - UNUSED_EXPIRATION_TIME) } - - def self.for(user, exercise, time_limit = 0) + scope :inactive_runners, -> { where('last_used < ?', Time.zone.now - UNUSED_EXPIRATION_TIME) } + + def self.for(user, exercise) execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) - runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment, time_limit: time_limit) + runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment, + time_limit: execution_environment.permitted_execution_time) return runner if runner.save + raise(RunnerNotAvailableError, 'No runner available') end @@ -31,22 +33,22 @@ class Runner < ApplicationRecord url = "#{runner_url}/files" body = {files: files.map { |filename, content| {filepath: filename, content: content} }} response = Faraday.patch(url, body.to_json, HEADERS) - if response.status == 404 - # runner has disappeared for some reason - self.destroy - raise(RunnerNotAvailableError, "Runner unavailable") - end + return unless response.status == 404 + + # runner has disappeared for some reason + destroy + raise(RunnerNotAvailableError, 'Runner unavailable') end def execute_command(command) + used_now url = "#{runner_url}/execute" response = Faraday.post(url, {command: command}.to_json, HEADERS) if response.status == 404 # runner has disappeared for some reason - self.destroy - raise(RunnerNotAvailableError, "Runner unavailable") + destroy + raise(RunnerNotAvailableError, 'Runner unavailable') end - used_now parse response end @@ -71,10 +73,9 @@ class Runner < ApplicationRecord private - def get_runner + def new_runner url = "#{BASE_URL}/runners" - body = {executionEnvironmentId: execution_environment.id} - body[:timeLimit] = time_limit + body = {executionEnvironmentId: execution_environment.id, timeLimit: time_limit} response = Faraday.post(url, body.to_json, HEADERS) response_body = parse response self.runner_id = response_body[:runnerId] @@ -90,7 +91,7 @@ class Runner < ApplicationRecord end def used_now - self.last_used = Time.now + self.last_used = Time.zone.now save end end diff --git a/app/models/submission.rb b/app/models/submission.rb index 7d14e87d..1a529630 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -72,7 +72,7 @@ class Submission < ApplicationRecord # expects the full file path incl. file extension # Caution: There must be no unnecessary path prefix included. # Use `file.ext` rather than `./file.ext` - collect_files.detect {|file| file.filepath == file_path } + collect_files.detect { |file| file.filepath == file_path } end def normalized_score @@ -144,18 +144,18 @@ class Submission < ApplicationRecord prepared_runner do |runner| scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension - stdout = "" - stderr = "" + stdout = '' + stderr = '' exit_code = 1 # default to error - execution_time = runner.execute_interactively(score_command) do |runner, socket| + execution_time = runner.execute_interactively(score_command) do |_runner, socket| socket.on :stderr do |data| stderr << data end socket.on :stdout do |data| stdout << data end - socket.on :exit do |_exit_code| - exit_code = _exit_code + socket.on :exit do |received_exit_code| + exit_code = received_exit_code EventMachine.stop_event_loop end end @@ -163,9 +163,9 @@ class Submission < ApplicationRecord file_role: file.role, waiting_for_container_time: runner.waiting_time, container_execution_time: execution_time, - status: (exit_code == 0) ? :ok : :failed, + status: exit_code.zero? ? :ok : :failed, stdout: stdout, - stderr: stderr, + stderr: stderr } test_result(output, file) end @@ -194,10 +194,10 @@ class Submission < ApplicationRecord end def prepared_runner - request_time = Time.now - runner = Runner.for(user, exercise, execution_environment.permitted_execution_time) + request_time = Time.zone.now + runner = Runner.for(user, exercise) copy_files_to runner - runner.waiting_time = Time.now - request_time + runner.waiting_time = Time.zone.now - request_time yield(runner) if block_given? end diff --git a/config/initializers/jobs.rb b/config/initializers/jobs.rb index 68eb2e92..f4bf7be2 100644 --- a/config/initializers/jobs.rb +++ b/config/initializers/jobs.rb @@ -1 +1,3 @@ +# frozen_string_literal: true + RunnerCleanupJob.perform_now unless Rake.application.top_level_tasks.to_s.match?(/db:|assets:/) From 7ff65135b59446904cb483179cf4f13397e0435a Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 25 May 2021 11:55:36 +0200 Subject: [PATCH 020/156] Add runner management configuration to ci --- config/code_ocean.yml.ci | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/code_ocean.yml.ci b/config/code_ocean.yml.ci index 5b15067e..8c0e14a8 100644 --- a/config/code_ocean.yml.ci +++ b/config/code_ocean.yml.ci @@ -9,3 +9,7 @@ test: enabled: false prometheus_exporter: enabled: false + runner_management: + url: https://runners.example.org + cleanup_interval: 60 + unused_runner_expiration_time: 180 From fc6aa12b0abe38d28beabff76cebda18756a4d4a Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Wed, 19 May 2021 16:02:32 +0200 Subject: [PATCH 021/156] Remove handling of runner timeouts --- app/jobs/application_job.rb | 5 ----- app/jobs/runner_cleanup_job.rb | 18 ------------------ app/models/runner.rb | 13 ++----------- config/initializers/jobs.rb | 3 --- ...ers.rb => 20210519134938_create_runners.rb} | 4 +--- db/schema.rb | 10 ++++------ 6 files changed, 7 insertions(+), 46 deletions(-) delete mode 100644 app/jobs/application_job.rb delete mode 100644 app/jobs/runner_cleanup_job.rb delete mode 100644 config/initializers/jobs.rb rename db/migrate/{20210415064948_create_runners.rb => 20210519134938_create_runners.rb} (67%) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb deleted file mode 100644 index 97fc265f..00000000 --- a/app/jobs/application_job.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -class ApplicationJob < ActiveJob::Base - queue_as :default -end diff --git a/app/jobs/runner_cleanup_job.rb b/app/jobs/runner_cleanup_job.rb deleted file mode 100644 index f578e9d5..00000000 --- a/app/jobs/runner_cleanup_job.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class RunnerCleanupJob < ApplicationJob - CLEANUP_INTERVAL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:cleanup_interval].seconds - - after_perform do |_job| - # re-schedule job - self.class.set(wait: CLEANUP_INTERVAL).perform_later - end - - def perform - Rails.logger.debug(Time.zone.now) - Runner.inactive_runners.each do |runner| - Rails.logger.debug("Destroying runner #{runner.runner_id}, unused since #{runner.last_used}") - runner.destroy - end - end -end diff --git a/app/models/runner.rb b/app/models/runner.rb index 653bc886..cb566661 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -15,14 +15,10 @@ class Runner < ApplicationRecord validates :execution_environment, presence: true validates :user, presence: true - validates :time_limit, presence: true - - scope :inactive_runners, -> { where('last_used < ?', Time.zone.now - UNUSED_EXPIRATION_TIME) } def self.for(user, exercise) execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) - runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment, - time_limit: execution_environment.permitted_execution_time) + runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment) return runner if runner.save @@ -41,7 +37,6 @@ class Runner < ApplicationRecord end def execute_command(command) - used_now url = "#{runner_url}/execute" response = Faraday.post(url, {command: command}.to_json, HEADERS) if response.status == 404 @@ -75,6 +70,7 @@ class Runner < ApplicationRecord def new_runner url = "#{BASE_URL}/runners" + time_limit = CodeOcean::Config.new(:code_ocean)[:runner_management][:unused_runner_expiration_time] body = {executionEnvironmentId: execution_environment.id, timeLimit: time_limit} response = Faraday.post(url, body.to_json, HEADERS) response_body = parse response @@ -89,9 +85,4 @@ class Runner < ApplicationRecord def parse(response) JSON.parse(response.body).deep_symbolize_keys end - - def used_now - self.last_used = Time.zone.now - save - end end diff --git a/config/initializers/jobs.rb b/config/initializers/jobs.rb deleted file mode 100644 index f4bf7be2..00000000 --- a/config/initializers/jobs.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -RunnerCleanupJob.perform_now unless Rake.application.top_level_tasks.to_s.match?(/db:|assets:/) diff --git a/db/migrate/20210415064948_create_runners.rb b/db/migrate/20210519134938_create_runners.rb similarity index 67% rename from db/migrate/20210415064948_create_runners.rb rename to db/migrate/20210519134938_create_runners.rb index 3ebebb63..e747ed4a 100644 --- a/db/migrate/20210415064948_create_runners.rb +++ b/db/migrate/20210519134938_create_runners.rb @@ -1,12 +1,10 @@ -class CreateRunners < ActiveRecord::Migration[5.2] +class CreateRunners < ActiveRecord::Migration[6.1] def change create_table :runners do |t| t.string :runner_id t.references :execution_environment t.references :user, polymorphic: true - t.integer :time_limit t.float :waiting_time - t.datetime :last_used t.timestamps end diff --git a/db/schema.rb b/db/schema.rb index b899981c..fb4dbc1d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_12_133612) do +ActiveRecord::Schema.define(version: 2021_05_19_134938) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -344,13 +344,11 @@ ActiveRecord::Schema.define(version: 2021_05_12_133612) do t.bigint "execution_environment_id" t.string "user_type" t.bigint "user_id" - t.integer "time_limit" t.float "waiting_time" - t.datetime "last_used" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false t.index ["execution_environment_id"], name: "index_runners_on_execution_environment_id" - t.index ["user_type", "user_id"], name: "index_runners_on_user_type_and_user_id" + t.index ["user_type", "user_id"], name: "index_runners_on_user" end create_table "searches", id: :serial, force: :cascade do |t| From 63d997a7e3112b2f6fa65b59f24b5f1edddbf93b Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 25 May 2021 12:45:38 +0200 Subject: [PATCH 022/156] Fix Rubocop offenses after Rubocop was reconfigured --- .../concerns/submission_scoring.rb | 7 ++--- app/controllers/submissions_controller.rb | 27 +++++++++---------- app/models/runner.rb | 8 +++--- app/models/submission.rb | 8 +++--- db/migrate/20210519134938_create_runners.rb | 2 ++ lib/runner/runner_connection.rb | 4 +-- 6 files changed, 29 insertions(+), 27 deletions(-) diff --git a/app/controllers/concerns/submission_scoring.rb b/app/controllers/concerns/submission_scoring.rb index 997ebae5..85da5106 100644 --- a/app/controllers/concerns/submission_scoring.rb +++ b/app/controllers/concerns/submission_scoring.rb @@ -49,14 +49,15 @@ module SubmissionScoring private :execute_test_file - def feedback_message(file, output) - # set_locale + def feedback_message(_file, output) + # TODO: why did we comment out set_locale and render_markdown? + set_locale if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' I18n.t('exercises.implement.default_test_feedback') elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' I18n.t('exercises.implement.default_linter_feedback') else - # render_markdown(file.feedback_message) + render_markdown(file.feedback_message) end end diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index fe281b29..77cafbb8 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -9,7 +9,7 @@ class SubmissionsController < ApplicationController include Tubesock::Hijack before_action :set_submission, - only: %i[download download_file render_file run score extract_errors show statistics test] + only: %i[download download_file render_file run score extract_errors show statistics] before_action :set_files, only: %i[download download_file render_file show run] before_action :set_file, only: %i[download_file render_file run] before_action :set_mime_type, only: %i[download_file render_file] @@ -134,7 +134,7 @@ class SubmissionsController < ApplicationController def handle_websockets(tubesock, runner, socket) tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) - @output = String.new + @output = +'' socket.on :output do |data| Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data}") @@ -162,14 +162,14 @@ class SubmissionsController < ApplicationController tubesock.onmessage do |event| event = JSON.parse(event).deep_symbolize_keys case event[:cmd].to_sym - when :client_kill - EventMachine.stop_event_loop - kill_socket(tubesock) - Rails.logger.debug('Client exited container.') - when :result - socket.send event[:data] - else - Rails.logger.info("Unknown command from client: #{event[:cmd]}") + when :client_kill + EventMachine.stop_event_loop + kill_socket(tubesock) + Rails.logger.debug('Client exited container.') + when :result + socket.send event[:data] + else + Rails.logger.info("Unknown command from client: #{event[:cmd]}") end rescue JSON::ParserError Rails.logger.debug { "Data received from client is not valid json: #{data}" } @@ -239,9 +239,8 @@ class SubmissionsController < ApplicationController def score Thread.new do hijack do |tubesock| - if @embed_options[:disable_run] - return kill_socket(tubesock) - end + return kill_socket(tubesock) if @embed_options[:disable_run] + tubesock.send_data(@submission.calculate_score) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) @@ -268,7 +267,7 @@ class SubmissionsController < ApplicationController # private :set_docker_client def set_file - @file = @files.detect { |file| file.name_with_extension == sanitize_filename } + @file = @files.detect {|file| file.name_with_extension == sanitize_filename } head :not_found unless @file end private :set_file diff --git a/app/models/runner.rb b/app/models/runner.rb index cb566661..7c812307 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -22,18 +22,18 @@ class Runner < ApplicationRecord return runner if runner.save - raise(RunnerNotAvailableError, 'No runner available') + raise RunnerNotAvailableError.new('No runner available') end def copy_files(files) url = "#{runner_url}/files" - body = {files: files.map { |filename, content| {filepath: filename, content: content} }} + body = {files: files.map {|filename, content| {filepath: filename, content: content} }} response = Faraday.patch(url, body.to_json, HEADERS) return unless response.status == 404 # runner has disappeared for some reason destroy - raise(RunnerNotAvailableError, 'Runner unavailable') + raise RunnerNotAvailableError.new('Runner unavailable') end def execute_command(command) @@ -42,7 +42,7 @@ class Runner < ApplicationRecord if response.status == 404 # runner has disappeared for some reason destroy - raise(RunnerNotAvailableError, 'Runner unavailable') + raise RunnerNotAvailableError.new('Runner unavailable') end parse response end diff --git a/app/models/submission.rb b/app/models/submission.rb index 1a529630..249fdc3f 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -72,7 +72,7 @@ class Submission < ApplicationRecord # expects the full file path incl. file extension # Caution: There must be no unnecessary path prefix included. # Use `file.ext` rather than `./file.ext` - collect_files.detect { |file| file.filepath == file_path } + collect_files.detect {|file| file.filepath == file_path } end def normalized_score @@ -165,7 +165,7 @@ class Submission < ApplicationRecord container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed, stdout: stdout, - stderr: stderr + stderr: stderr, } test_result(output, file) end @@ -202,7 +202,7 @@ class Submission < ApplicationRecord end def command_for(template, file) - filepath = collect_files.find { |f| f.name_with_extension == file }.filepath + filepath = collect_files.find {|f| f.name_with_extension == file }.filepath template % command_substitutions(filepath) end @@ -210,7 +210,7 @@ class Submission < ApplicationRecord { class_name: File.basename(filename, File.extname(filename)).camelize, filename: filename, - module_name: File.basename(filename, File.extname(filename)).underscore + module_name: File.basename(filename, File.extname(filename)).underscore, } end end diff --git a/db/migrate/20210519134938_create_runners.rb b/db/migrate/20210519134938_create_runners.rb index e747ed4a..91ea4814 100644 --- a/db/migrate/20210519134938_create_runners.rb +++ b/db/migrate/20210519134938_create_runners.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateRunners < ActiveRecord::Migration[6.1] def change create_table :runners do |t| diff --git a/lib/runner/runner_connection.rb b/lib/runner/runner_connection.rb index 4f692edd..2aae3b45 100644 --- a/lib/runner/runner_connection.rb +++ b/lib/runner/runner_connection.rb @@ -11,10 +11,10 @@ class RunnerConnection @socket = Faye::WebSocket::Client.new(url, [], ping: 5) %i[open message error close].each do |event_type| - @socket.on(event_type) { |event| __send__(:"on_#{event_type}", event) } + @socket.on(event_type) {|event| __send__(:"on_#{event_type}", event) } end - EVENTS.each { |event_type| instance_variable_set(:"@#{event_type}_callback", ->(e) {}) } + EVENTS.each {|event_type| instance_variable_set(:"@#{event_type}_callback", ->(e) {}) } @start_callback = -> {} @exit_code = 0 end From 8d968e01e6d547a94d07087838ea76e6c65cb8b9 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 25 May 2021 10:47:49 +0200 Subject: [PATCH 023/156] Move RunnerConnection into class `Runner` The old approach was to require the runner connection. This did not work anymore with Zeitwerk in Rails 6. @sebastian.serth and I moved the Connection class in `lib` into the ActiveRecord class `Runner`. This will also work with future changes like specific error classes. Furthermore the config was fixed and simplified. Co-authored-by: Sebastian Serth --- app/models/runner.rb | 8 +++----- config/code_ocean.yml.ci | 1 - config/code_ocean.yml.example | 12 ++++-------- lib/runner/{runner_connection.rb => connection.rb} | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) rename lib/runner/{runner_connection.rb => connection.rb} (98%) diff --git a/app/models/runner.rb b/app/models/runner.rb index 7c812307..08e84642 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'runner/runner_connection' - class Runner < ApplicationRecord BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] HEADERS = {'Content-Type' => 'application/json'}.freeze @@ -18,7 +16,7 @@ class Runner < ApplicationRecord def self.for(user, exercise) execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) - runner = Runner.find_or_create_by(user: user, execution_environment: execution_environment) + runner = find_or_create_by(user: user, execution_environment: execution_environment) return runner if runner.save @@ -51,7 +49,7 @@ class Runner < ApplicationRecord starting_time = Time.zone.now websocket_url = execute_command(command)[:websocketUrl] EventMachine.run do - socket = RunnerConnection.new(websocket_url) + socket = Runner::Connection.new(websocket_url) yield(self, socket) if block_given? end Time.zone.now - starting_time # execution time @@ -70,7 +68,7 @@ class Runner < ApplicationRecord def new_runner url = "#{BASE_URL}/runners" - time_limit = CodeOcean::Config.new(:code_ocean)[:runner_management][:unused_runner_expiration_time] + time_limit = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time] body = {executionEnvironmentId: execution_environment.id, timeLimit: time_limit} response = Faraday.post(url, body.to_json, HEADERS) response_body = parse response diff --git a/config/code_ocean.yml.ci b/config/code_ocean.yml.ci index 8c0e14a8..6865a478 100644 --- a/config/code_ocean.yml.ci +++ b/config/code_ocean.yml.ci @@ -11,5 +11,4 @@ test: enabled: false runner_management: url: https://runners.example.org - cleanup_interval: 60 unused_runner_expiration_time: 180 diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index b2413754..f948ac6a 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -4,27 +4,25 @@ default: &default answers_per_query: 3 code_pilot: enabled: false + url: //localhost:3000 codeharbor: enabled: false codeocean_events: enabled: false + prometheus_exporter: + enabled: false runner_management: url: https://runners.example.org - cleanup_interval: 60 unused_runner_expiration_time: 180 development: + <<: *default flowr: enabled: true answers_per_query: 3 - code_pilot: - enabled: false - url: //localhost:3000 codeharbor: enabled: true url: https://codeharbor.openhpi.de - prometheus_exporter: - enabled: false production: <<: *default @@ -33,5 +31,3 @@ production: test: <<: *default - prometheus_exporter: - enabled: false diff --git a/lib/runner/runner_connection.rb b/lib/runner/connection.rb similarity index 98% rename from lib/runner/runner_connection.rb rename to lib/runner/connection.rb index 2aae3b45..ea3840a6 100644 --- a/lib/runner/runner_connection.rb +++ b/lib/runner/connection.rb @@ -3,7 +3,7 @@ require 'faye/websocket/client' require 'json_schemer' -class RunnerConnection +class Runner::Connection EVENTS = %i[start output exit stdout stderr].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) From b762c73ddd7e47a70a6e0cfb6449d1ba477f43df Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Wed, 2 Jun 2021 10:39:13 +0200 Subject: [PATCH 024/156] Update usage of Poseidon API to newest API version (0.2.2) copy file, create and execute command had to be adapted. --- app/models/runner.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 08e84642..eec2a1e6 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -25,7 +25,7 @@ class Runner < ApplicationRecord def copy_files(files) url = "#{runner_url}/files" - body = {files: files.map {|filename, content| {filepath: filename, content: content} }} + body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} response = Faraday.patch(url, body.to_json, HEADERS) return unless response.status == 404 @@ -36,7 +36,8 @@ class Runner < ApplicationRecord def execute_command(command) url = "#{runner_url}/execute" - response = Faraday.post(url, {command: command}.to_json, HEADERS) + body = {command: command, timeLimit: execution_environment.permitted_execution_time} + response = Faraday.post(url, body.to_json, HEADERS) if response.status == 404 # runner has disappeared for some reason destroy @@ -68,8 +69,7 @@ class Runner < ApplicationRecord def new_runner url = "#{BASE_URL}/runners" - time_limit = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time] - body = {executionEnvironmentId: execution_environment.id, timeLimit: time_limit} + body = {executionEnvironmentId: execution_environment.id, inactivityTimeout: UNUSED_EXPIRATION_TIME} response = Faraday.post(url, body.to_json, HEADERS) response_body = parse response self.runner_id = response_body[:runnerId] From 90fac7b94cebb1fdc8ee639150979207cd03e77c Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 1 Jun 2021 14:07:35 +0200 Subject: [PATCH 025/156] Copy execution environment to Poseidon on create and update When creating or updating an execution environment, an API call to Poseidon is made with the necessary information to create the corresponding Nomad job. If runner management is configured, his will display a warning (currently in the same color as if it were a success) in the UI, if the API call fails. The environment is saved even if it fails. If runner management is not configured, this warning will not be created. --- app/controllers/concerns/common_behavior.rb | 17 +++++++--- .../execution_environments_controller.rb | 19 +++++++++-- app/models/execution_environment.rb | 33 +++++++++++++++++++ .../execution_environments/_form.html.slim | 5 ++- .../execution_environments/show.html.slim | 2 +- config/locales/de.yml | 5 ++- config/locales/en.yml | 5 ++- ..._add_cpu_limit_to_execution_environment.rb | 7 ++++ ..._exposed_ports_in_execution_environment.rb | 17 ++++++++++ db/schema.rb | 3 +- 10 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb create mode 100644 db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb diff --git a/app/controllers/concerns/common_behavior.rb b/app/controllers/concerns/common_behavior.rb index 4c36cf63..8d63246d 100644 --- a/app/controllers/concerns/common_behavior.rb +++ b/app/controllers/concerns/common_behavior.rb @@ -5,10 +5,13 @@ module CommonBehavior @object = options[:object] respond_to do |format| if @object.save - yield if block_given? + notice = t('shared.object_created', model: @object.class.model_name.human) + if block_given? + result = yield + notice = result if result.present? + end path = options[:path].try(:call) || @object - respond_with_valid_object(format, notice: t('shared.object_created', model: @object.class.model_name.human), -path: path, status: :created) + respond_with_valid_object(format, notice: notice, path: path, status: :created) else respond_with_invalid_object(format, template: :new) end @@ -42,9 +45,13 @@ path: path, status: :created) @object = options[:object] respond_to do |format| if @object.update(options[:params]) + notice = t('shared.object_updated', model: @object.class.model_name.human) + if block_given? + result = yield + notice = result if result.present? + end path = options[:path] || @object - respond_with_valid_object(format, notice: t('shared.object_updated', model: @object.class.model_name.human), -path: path, status: :ok) + respond_with_valid_object(format, notice: notice, path: path, status: :ok) else respond_with_invalid_object(format, template: :edit) end diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 4449fa41..95132b09 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -3,6 +3,8 @@ class ExecutionEnvironmentsController < ApplicationController include CommonBehavior + RUNNER_MANAGEMENT_PRESENT = CodeOcean::Config.new(:code_ocean).read[:runner_management].present? + before_action :set_docker_images, only: %i[create edit new update] before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell statistics] before_action :set_testing_framework_adapters, only: %i[create edit new update] @@ -15,7 +17,9 @@ class ExecutionEnvironmentsController < ApplicationController def create @execution_environment = ExecutionEnvironment.new(execution_environment_params) authorize! - create_and_respond(object: @execution_environment) + create_and_respond(object: @execution_environment) do + copy_execution_environment_to_poseidon + end end def destroy @@ -105,7 +109,7 @@ class ExecutionEnvironmentsController < ApplicationController def execution_environment_params if params[:execution_environment].present? - params[:execution_environment].permit(:docker_image, :exposed_ports, :editor_mode, :file_extension, :file_type_id, :help, :indent_size, :memory_limit, :name, :network_enabled, :permitted_execution_time, :pool_size, :run_command, :test_command, :testing_framework).merge( + params[:execution_environment].permit(:docker_image, :exposed_ports, :editor_mode, :file_extension, :file_type_id, :help, :indent_size, :memory_limit, :cpu_limit, :name, :network_enabled, :permitted_execution_time, :pool_size, :run_command, :test_command, :testing_framework).merge( user_id: current_user.id, user_type: current_user.class.name ) end @@ -155,6 +159,15 @@ class ExecutionEnvironmentsController < ApplicationController end def update - update_and_respond(object: @execution_environment, params: execution_environment_params) + update_and_respond(object: @execution_environment, params: execution_environment_params) do + copy_execution_environment_to_poseidon + end end + + def copy_execution_environment_to_poseidon + unless RUNNER_MANAGEMENT_PRESENT && @execution_environment.copy_to_poseidon + t('execution_environments.form.errors.not_synced_to_poseidon') + end + end + private :copy_execution_environment_to_poseidon end diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index e4d39a93..3db35c35 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -7,6 +7,10 @@ class ExecutionEnvironment < ApplicationRecord include DefaultValues VALIDATION_COMMAND = 'whoami' + RUNNER_MANAGEMENT_PRESENT = CodeOcean::Config.new(:code_ocean).read[:runner_management].present? + BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] if RUNNER_MANAGEMENT_PRESENT + HEADERS = {'Content-Type' => 'application/json'}.freeze + DEFAULT_CPU_LIMIT = 20 after_initialize :set_default_values @@ -26,6 +30,8 @@ class ExecutionEnvironment < ApplicationRecord validates :permitted_execution_time, numericality: {only_integer: true}, presence: true validates :pool_size, numericality: {only_integer: true}, presence: true validates :run_command, presence: true + validates :cpu_limit, presence: true, numericality: {greater_than: 0, only_integer: true} + validates :exposed_ports, format: {with: /\A(([[:digit:]]{1,5},)*([[:digit:]]{1,5}))?\z/} def set_default_values set_default_values_if_present(permitted_execution_time: 60, pool_size: 0) @@ -36,6 +42,33 @@ class ExecutionEnvironment < ApplicationRecord name end + def copy_to_poseidon + return false unless RUNNER_MANAGEMENT_PRESENT + + url = "#{BASE_URL}/execution-environments/#{id}" + response = Faraday.put(url, to_json, HEADERS) + return true if [201, 204].include? response.status + + Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") + false + end + + def to_json(*_args) + { + id: id, + image: docker_image, + prewarmingPoolSize: pool_size, + cpuLimit: cpu_limit, + memoryLimit: memory_limit, + networkAccess: network_enabled, + exposedPorts: exposed_ports_list, + }.to_json + end + + def exposed_ports_list + (exposed_ports || '').split(',').map(&:to_i) + end + def valid_test_setup? if test_command? ^ testing_framework? errors.add(:test_command, diff --git a/app/views/execution_environments/_form.html.slim b/app/views/execution_environments/_form.html.slim index 13ee252e..9447d1c6 100644 --- a/app/views/execution_environments/_form.html.slim +++ b/app/views/execution_environments/_form.html.slim @@ -15,11 +15,14 @@ .help-block.form-text == t('.hints.docker_image') .form-group = f.label(:exposed_ports) - = f.text_field(:exposed_ports, class: 'form-control', placeholder: '3000, 4000') + = f.text_field(:exposed_ports, class: 'form-control', placeholder: '3000,4000', pattern: '^((\d{1,5},)*(\d{1,5}))?$') .help-block.form-text == t('.hints.exposed_ports') .form-group = f.label(:memory_limit) = f.number_field(:memory_limit, class: 'form-control', min: DockerClient::MINIMUM_MEMORY_LIMIT, value: f.object.memory_limit || DockerClient::DEFAULT_MEMORY_LIMIT) + .form-group + = f.label(:cpu_limit) + = f.number_field(:cpu_limit, class: 'form-control', min: 1, step: 1, value: ExecutionEnvironment::DEFAULT_CPU_LIMIT) .form-check.mb-3 label.form-check-label = f.check_box(:network_enabled, class: 'form-check-input') diff --git a/app/views/execution_environments/show.html.slim b/app/views/execution_environments/show.html.slim index 21133a71..6d9fb32e 100644 --- a/app/views/execution_environments/show.html.slim +++ b/app/views/execution_environments/show.html.slim @@ -5,7 +5,7 @@ h1 = row(label: 'execution_environment.name', value: @execution_environment.name) = row(label: 'execution_environment.user', value: link_to_if(policy(@execution_environment.author).show?, @execution_environment.author, @execution_environment.author)) = row(label: 'execution_environment.file_type', value: @execution_environment.file_type.present? ? link_to(@execution_environment.file_type, @execution_environment.file_type) : nil) -- [:docker_image, :exposed_ports, :memory_limit, :network_enabled, :permitted_execution_time, :pool_size].each do |attribute| +- [:docker_image, :exposed_ports, :memory_limit, :cpu_limit, :network_enabled, :permitted_execution_time, :pool_size].each do |attribute| = row(label: "execution_environment.#{attribute}", value: @execution_environment.send(attribute)) - [:run_command, :test_command].each do |attribute| = row(label: "execution_environment.#{attribute}") do diff --git a/config/locales/de.yml b/config/locales/de.yml index 29cf2eb4..344f1d58 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -14,6 +14,7 @@ de: file_type_id: Standard-Dateityp help: Hilfetext memory_limit: Speicher-Limit (in MB) + cpu_limit: CPU-Limit (in MHz) network_enabled: Netzwerkzugriff name: Name permitted_execution_time: Erlaubte Ausführungszeit (in Sekunden) @@ -281,7 +282,9 @@ de: hints: command: filename wird automatisch durch den richtigen Dateinamen ersetzt. docker_image: 'Wählen Sie ein Docker-Image aus der Liste oder fügen Sie ein neues hinzu, welches über DockerHub verfügbar ist.' - exposed_ports: Während der Ausführung sind diese Ports für den Nutzer zugänglich. + exposed_ports: Während der Ausführung sind diese Ports für den Nutzer zugänglich. Die Portnummern müssen mit Komma, aber ohne Leerzeichen voneinander getrennt sein. + errors: + not_synced_to_poseidon: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zu Poseidon synchronisiert. index: shell: Shell shell: diff --git a/config/locales/en.yml b/config/locales/en.yml index 41c37dfc..183a1186 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -14,6 +14,7 @@ en: file_type_id: Default File Type help: Help Text memory_limit: Memory Limit (in MB) + cpu_limit: CPU Limit (in MHz) name: Name network_enabled: Network Enabled permitted_execution_time: Permitted Execution Time (in Seconds) @@ -281,7 +282,9 @@ en: hints: command: filename is automatically replaced with the correct filename. docker_image: Pick a Docker image listed above or add a new one which is available via DockerHub. - exposed_ports: During code execution these ports are accessible for the user. + exposed_ports: During code execution these ports are accessible for the user. Port numbers must be separated by a comma but no space. + errors: + not_synced_to_poseidon: The ExecutionEnvironment was created but not synced to Poseidon due to an error. index: shell: Shell shell: diff --git a/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb b/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb new file mode 100644 index 00000000..c74aa69a --- /dev/null +++ b/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddCpuLimitToExecutionEnvironment < ActiveRecord::Migration[6.1] + def change + add_column :execution_environments, :cpu_limit, :integer, default: 20 + end +end diff --git a/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb b/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb new file mode 100644 index 00000000..32db9950 --- /dev/null +++ b/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CleanExposedPortsInExecutionEnvironment < ActiveRecord::Migration[6.1] + def change + ExecutionEnvironment.all.each do |execution_environment| + continue if execution_environment.exposed_ports.nil? + + cleaned = execution_environment.exposed_ports.gsub(/[[:space:]]/, '') + list = cleaned.split(',').map(&:to_i).uniq + if list.empty? + execution_environment.update(exposed_ports: nil) + else + execution_environment.update(exposed_ports: list.join(',')) + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index fb4dbc1d..beefc5b1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_19_134938) do +ActiveRecord::Schema.define(version: 2021_06_02_071834) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -112,6 +112,7 @@ ActiveRecord::Schema.define(version: 2021_05_19_134938) do t.integer "file_type_id" t.integer "memory_limit" t.boolean "network_enabled" + t.integer "cpu_limit" end create_table "exercise_collection_items", id: :serial, force: :cascade do |t| From d22d24df4daa286f921c1d00b586e1eb3aeedc9a Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 1 Jun 2021 14:39:30 +0200 Subject: [PATCH 026/156] Add tests for execution environment copy to Poseidon --- .../execution_environments_controller_spec.rb | 22 +++++- spec/factories/execution_environment.rb | 13 ++++ spec/models/execution_environment_spec.rb | 75 +++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index 6736a18b..3507fc8a 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -6,7 +6,10 @@ describe ExecutionEnvironmentsController do let(:execution_environment) { FactoryBot.create(:ruby) } let(:user) { FactoryBot.create(:admin) } - before { allow(controller).to receive(:current_user).and_return(user) } + before do + allow(controller).to receive(:current_user).and_return(user) + allow(controller).to receive(:copy_execution_environment_to_poseidon).and_return(nil) + end describe 'POST #create' do before { allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) } @@ -23,6 +26,10 @@ describe ExecutionEnvironmentsController do expect { perform_request.call }.to change(ExecutionEnvironment, :count).by(1) end + it 'registers the execution environment with Poseidon' do + expect(controller).to have_received(:copy_execution_environment_to_poseidon) + end + expect_redirect(ExecutionEnvironment.last) end @@ -32,6 +39,10 @@ describe ExecutionEnvironmentsController do expect_assigns(execution_environment: ExecutionEnvironment) expect_status(200) expect_template(:new) + + it 'does not register the execution environment with Poseidon' do + expect(controller).not_to have_received(:copy_execution_environment_to_poseidon) + end end end @@ -156,12 +167,17 @@ describe ExecutionEnvironmentsController do context 'with a valid execution environment' do before do allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) + allow(controller).to receive(:copy_execution_environment_to_poseidon).and_return(nil) put :update, params: {execution_environment: FactoryBot.attributes_for(:ruby), id: execution_environment.id} end expect_assigns(docker_images: Array) expect_assigns(execution_environment: ExecutionEnvironment) expect_redirect(:execution_environment) + + it 'updates the execution environment at Poseidon' do + expect(controller).to have_received(:copy_execution_environment_to_poseidon) + end end context 'with an invalid execution environment' do @@ -170,6 +186,10 @@ describe ExecutionEnvironmentsController do expect_assigns(execution_environment: ExecutionEnvironment) expect_status(200) expect_template(:edit) + + it 'does not update the execution environment at Poseidon' do + expect(controller).not_to have_received(:copy_execution_environment_to_poseidon) + end end end end diff --git a/spec/factories/execution_environment.rb b/spec/factories/execution_environment.rb index 2b671575..9d145031 100644 --- a/spec/factories/execution_environment.rb +++ b/spec/factories/execution_environment.rb @@ -4,6 +4,7 @@ FactoryBot.define do factory :coffee_script, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-coffee:latest' } file_type { association :dot_coffee, user: user } help @@ -18,6 +19,7 @@ FactoryBot.define do factory :html, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-html:latest' } file_type { association :dot_html, user: user } help @@ -34,6 +36,7 @@ FactoryBot.define do factory :java, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'openhpi/co_execenv_java:8' } file_type { association :dot_java, user: user } help @@ -50,6 +53,7 @@ FactoryBot.define do factory :jruby, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-jruby:latest' } file_type { association :dot_rb, user: user } help @@ -66,6 +70,7 @@ FactoryBot.define do factory :node_js, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-node:latest' } file_type { association :dot_js, user: user } help @@ -80,6 +85,7 @@ FactoryBot.define do factory :python, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'openhpi/co_execenv_python:3.4' } file_type { association :dot_py, user: user } help @@ -96,6 +102,7 @@ FactoryBot.define do factory :ruby, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-ruby:latest' } file_type { association :dot_rb, user: user } help @@ -112,6 +119,7 @@ FactoryBot.define do factory :sinatra, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-sinatra:latest' } file_type { association :dot_rb, user: user } exposed_ports { '4567' } @@ -129,6 +137,7 @@ FactoryBot.define do factory :sqlite, class: 'ExecutionEnvironment' do created_by_teacher default_memory_limit + default_cpu_limit docker_image { 'hklement/ubuntu-sqlite:latest' } file_type { association :dot_sql, user: user } help @@ -146,6 +155,10 @@ FactoryBot.define do memory_limit { DockerClient::DEFAULT_MEMORY_LIMIT } end + trait :default_cpu_limit do + cpu_limit { 20 } + end + trait :help do help { Forgery(:lorem_ipsum).words(Forgery(:basic).number(at_least: 50, at_most: 100)) } end diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index b2edd346..54e01111 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -30,6 +30,21 @@ describe ExecutionEnvironment do expect(execution_environment.errors[:memory_limit]).to be_present end + it 'validates the minimum value of the cpu limit' do + execution_environment.update(cpu_limit: 0) + expect(execution_environment.errors[:cpu_limit]).to be_present + end + + it 'validates that cpu limit is an integer' do + execution_environment.update(cpu_limit: Math::PI) + expect(execution_environment.errors[:cpu_limit]).to be_present + end + + it 'validates the presence of a cpu limit' do + execution_environment.update(cpu_limit: nil) + expect(execution_environment.errors[:cpu_limit]).to be_present + end + it 'validates the presence of a name' do expect(execution_environment.errors[:name]).to be_present end @@ -69,6 +84,14 @@ describe ExecutionEnvironment do expect(execution_environment.errors[:user_type]).to be_present end + it 'validates the format of the exposed ports' do + execution_environment.update(exposed_ports: '1,') + expect(execution_environment.errors[:exposed_ports]).to be_present + + execution_environment.update(exposed_ports: '1,a') + expect(execution_environment.errors[:exposed_ports]).to be_present + end + describe '#valid_test_setup?' do context 'with a test command and a testing framework' do before { execution_environment.update(test_command: FactoryBot.attributes_for(:ruby)[:test_command], testing_framework: FactoryBot.attributes_for(:ruby)[:testing_framework]) } @@ -153,4 +176,56 @@ describe ExecutionEnvironment do end end end + + describe '#exposed_ports_list' do + it 'returns an empty array if no ports are exposed' do + execution_environment.exposed_ports = nil + expect(execution_environment.exposed_ports_list).to eq([]) + end + + it 'returns an array of integers representing the exposed ports' do + execution_environment.exposed_ports = '1,2,3' + expect(execution_environment.exposed_ports_list).to eq([1, 2, 3]) + + execution_environment.exposed_ports_list.each do |port| + expect(execution_environment.exposed_ports).to include(port.to_s) + end + end + end + + describe '#copy_to_poseidon' do + let(:execution_environment) { FactoryBot.create(:ruby) } + + it 'makes the correct request to Poseidon' do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: 201)) + execution_environment.copy_to_poseidon + expect(Faraday).to have_received(:put) do |url, body, headers| + expect(url).to match(%r{execution-environments/#{execution_environment.id}\z}) + expect(body).to eq(execution_environment.to_json) + expect(headers).to include({'Content-Type' => 'application/json'}) + end + end + + shared_examples 'returns true when the api request was successful' do |status| + it "returns true on status #{status}" do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + expect(execution_environment.copy_to_poseidon).to be_truthy + end + end + + shared_examples 'returns false when the api request failed' do |status| + it "returns false on status #{status}" do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + expect(execution_environment.copy_to_poseidon).to be_falsey + end + end + + [201, 204].each do |status| + include_examples 'returns true when the api request was successful', status + end + + [400, 500].each do |status| + include_examples 'returns false when the api request failed', status + end + end end From 5e913c8a1a18761c09e393f07b905fae0da5e8eb Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Mon, 7 Jun 2021 08:46:02 +0200 Subject: [PATCH 027/156] Skip failing tests 17 tests are always failing, due to changes introduced when adding the Runner abstraction. To know only these fail, they now get skipped in order to make it apparent if tests that should not fail do fail in the pipeline. --- spec/concerns/submission_scoring_spec.rb | 3 ++- spec/controllers/exercises_controller_spec.rb | 3 ++- spec/features/editor_spec.rb | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/concerns/submission_scoring_spec.rb b/spec/concerns/submission_scoring_spec.rb index ac4e8490..086b2ae7 100644 --- a/spec/concerns/submission_scoring_spec.rb +++ b/spec/concerns/submission_scoring_spec.rb @@ -6,7 +6,8 @@ class Controller < AnonymousController include SubmissionScoring end -describe SubmissionScoring do +# This is broken since the Runner was added. +describe SubmissionScoring, skip: true do let(:controller) { Controller.new } let(:submission) { FactoryBot.create(:submission, cause: 'submit') } diff --git a/spec/controllers/exercises_controller_spec.rb b/spec/controllers/exercises_controller_spec.rb index 0332641f..6d75ef64 100644 --- a/spec/controllers/exercises_controller_spec.rb +++ b/spec/controllers/exercises_controller_spec.rb @@ -236,7 +236,8 @@ describe ExercisesController do expect_template(:statistics) end - describe 'POST #submit' do + # This is broken since the Runner was added. + describe 'POST #submit', skip: true do let(:output) { {} } let(:perform_request) { post :submit, format: :json, params: {id: exercise.id, submission: {cause: 'submit', exercise_id: exercise.id}} } let(:user) { FactoryBot.create(:external_user) } diff --git a/spec/features/editor_spec.rb b/spec/features/editor_spec.rb index 334cc7f0..1a8152ef 100644 --- a/spec/features/editor_spec.rb +++ b/spec/features/editor_spec.rb @@ -94,6 +94,9 @@ describe 'Editor', js: true do end it 'contains a button for submitting the exercise' do + # This is broken since the Runner was added. + skip + allow_any_instance_of(SubmissionsController).to receive(:score_submission).and_return(scoring_response) click_button(I18n.t('exercises.editor.score')) expect(page).not_to have_css('#submit_outdated') From d5b274c9f29a283dfcfcd5f0866cea152cf2c99c Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 27 May 2021 10:21:25 +0200 Subject: [PATCH 028/156] Introduce new error types for runners The errors are raised in the runner model and in the runner connection class. In the submission controller the errors are rescued and, depending on the error, the status timeout / container depleted is sent to the client. --- app/controllers/submissions_controller.rb | 41 ++++--- app/errors/runner/error.rb | 3 + app/errors/runner/error/bad_request.rb | 3 + app/errors/runner/error/execution_timeout.rb | 3 + .../runner/error/internal_server_error.rb | 3 + app/errors/runner/error/not_available.rb | 3 + app/errors/runner/error/not_found.rb | 3 + app/errors/runner/error/unauthorized.rb | 3 + app/errors/runner/error/unknown.rb | 3 + app/errors/runner_not_available_error.rb | 4 - app/models/runner.rb | 106 +++++++++++++----- lib/runner/connection.rb | 15 ++- 12 files changed, 137 insertions(+), 53 deletions(-) create mode 100644 app/errors/runner/error.rb create mode 100644 app/errors/runner/error/bad_request.rb create mode 100644 app/errors/runner/error/execution_timeout.rb create mode 100644 app/errors/runner/error/internal_server_error.rb create mode 100644 app/errors/runner/error/not_available.rb create mode 100644 app/errors/runner/error/not_found.rb create mode 100644 app/errors/runner/error/unauthorized.rb create mode 100644 app/errors/runner/error/unknown.rb delete mode 100644 app/errors/runner_not_available_error.rb diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 77cafbb8..93bdab1d 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -133,7 +133,7 @@ class SubmissionsController < ApplicationController end def handle_websockets(tubesock, runner, socket) - tubesock.send_data JSON.dump({'cmd' => 'status', 'status' => :container_running}) + tubesock.send_data JSON.dump({cmd: :status, status: :container_running}) @output = +'' socket.on :output do |data| @@ -182,21 +182,21 @@ class SubmissionsController < ApplicationController def run hijack do |tubesock| - if @embed_options[:disable_run] - kill_socket(tubesock) - else - begin - @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| - @waiting_for_container_time = runner.waiting_time - handle_websockets(tubesock, runner, socket) - end - save_run_output - rescue RunnerNotAvailableError - tubesock.send_data JSON.dump({cmd: :timeout}) - kill_socket(tubesock) - Rails.logger.debug('Runner not available') - end + return kill_socket(tubesock) if @embed_options[:disable_run] + + @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| + @waiting_for_container_time = runner.waiting_time + handle_websockets(tubesock, runner, socket) end + save_run_output + rescue Runner::Error::ExecutionTimeout => e + tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) + kill_socket(tubesock) + Rails.logger.debug { "Running a submission failed: #{e.message}" } + rescue Runner::Error => e + tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) + kill_socket(tubesock) + Rails.logger.debug { "Runner error while running a submission: #{e.message}" } end end @@ -244,8 +244,15 @@ class SubmissionsController < ApplicationController tubesock.send_data(@submission.calculate_score) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) - - tubesock.send_data JSON.dump({'cmd' => 'exit'}) + rescue Runner::Error::ExecutionTimeout => e + tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) + Rails.logger.debug { "Running a submission failed: #{e.message}" } + rescue Runner::Error => e + tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) + Rails.logger.debug { "Runner error while scoring a submission: #{e.message}" } + ensure + tubesock.send_data JSON.dump({cmd: :exit}) + tubesock.close end ensure ActiveRecord::Base.connection_pool.release_connection diff --git a/app/errors/runner/error.rb b/app/errors/runner/error.rb new file mode 100644 index 00000000..3470f553 --- /dev/null +++ b/app/errors/runner/error.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error < ApplicationError; end diff --git a/app/errors/runner/error/bad_request.rb b/app/errors/runner/error/bad_request.rb new file mode 100644 index 00000000..98b8f5ca --- /dev/null +++ b/app/errors/runner/error/bad_request.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::BadRequest < Runner::Error; end diff --git a/app/errors/runner/error/execution_timeout.rb b/app/errors/runner/error/execution_timeout.rb new file mode 100644 index 00000000..c10806a5 --- /dev/null +++ b/app/errors/runner/error/execution_timeout.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::ExecutionTimeout < Runner::Error; end diff --git a/app/errors/runner/error/internal_server_error.rb b/app/errors/runner/error/internal_server_error.rb new file mode 100644 index 00000000..68f3f0b8 --- /dev/null +++ b/app/errors/runner/error/internal_server_error.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::InternalServerError < Runner::Error; end diff --git a/app/errors/runner/error/not_available.rb b/app/errors/runner/error/not_available.rb new file mode 100644 index 00000000..4785ea59 --- /dev/null +++ b/app/errors/runner/error/not_available.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::NotAvailable < Runner::Error; end diff --git a/app/errors/runner/error/not_found.rb b/app/errors/runner/error/not_found.rb new file mode 100644 index 00000000..329bcc71 --- /dev/null +++ b/app/errors/runner/error/not_found.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::NotFound < Runner::Error; end diff --git a/app/errors/runner/error/unauthorized.rb b/app/errors/runner/error/unauthorized.rb new file mode 100644 index 00000000..8bccf7fa --- /dev/null +++ b/app/errors/runner/error/unauthorized.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::Unauthorized < Runner::Error; end diff --git a/app/errors/runner/error/unknown.rb b/app/errors/runner/error/unknown.rb new file mode 100644 index 00000000..6e1a1fce --- /dev/null +++ b/app/errors/runner/error/unknown.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Error::Unknown < Runner::Error; end diff --git a/app/errors/runner_not_available_error.rb b/app/errors/runner_not_available_error.rb deleted file mode 100644 index 4390fad4..00000000 --- a/app/errors/runner_not_available_error.rb +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true - -class RunnerNotAvailableError < ApplicationError -end diff --git a/app/models/runner.rb b/app/models/runner.rb index eec2a1e6..f8861a2a 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -4,51 +4,60 @@ class Runner < ApplicationRecord BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] HEADERS = {'Content-Type' => 'application/json'}.freeze UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds + ERRORS = %w[NOMAD_UNREACHABLE NOMAD_OVERLOAD NOMAD_INTERNAL_SERVER_ERROR UNKNOWN].freeze + + ERRORS.each do |error| + define_singleton_method :"error_#{error.downcase}" do + error + end + end belongs_to :execution_environment belongs_to :user, polymorphic: true - before_create :new_runner - before_destroy :destroy_runner + before_validation :request_remotely - validates :execution_environment, presence: true - validates :user, presence: true + validates :execution_environment, :user, :runner_id, presence: true def self.for(user, exercise) execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) runner = find_or_create_by(user: user, execution_environment: execution_environment) - return runner if runner.save + unless runner.persisted? + # runner was not saved in the database (was not valid) + raise Runner::Error::InternalServerError.new("Provided runner could not be saved: #{runner.errors.inspect}") + end - raise RunnerNotAvailableError.new('No runner available') + runner end def copy_files(files) url = "#{runner_url}/files" body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} response = Faraday.patch(url, body.to_json, HEADERS) - return unless response.status == 404 - - # runner has disappeared for some reason - destroy - raise RunnerNotAvailableError.new('Runner unavailable') + handle_error response unless response.status == 204 end def execute_command(command) url = "#{runner_url}/execute" body = {command: command, timeLimit: execution_environment.permitted_execution_time} response = Faraday.post(url, body.to_json, HEADERS) - if response.status == 404 - # runner has disappeared for some reason - destroy - raise RunnerNotAvailableError.new('Runner unavailable') + if response.status == 200 + response_body = parse response + websocket_url = response_body[:websocketUrl] + if websocket_url.present? + return websocket_url + else + raise Runner::Error::Unknown.new('Runner management sent unexpected response') + end end - parse response + + handle_error response end def execute_interactively(command) starting_time = Time.zone.now - websocket_url = execute_command(command)[:websocketUrl] + websocket_url = execute_command(command) EventMachine.run do socket = Runner::Connection.new(websocket_url) yield(self, socket) if block_given? @@ -56,24 +65,64 @@ class Runner < ApplicationRecord Time.zone.now - starting_time # execution time end - def destroy_runner - Faraday.delete runner_url - end + # This method is currently not used. + # This does *not* destroy the ActiveRecord model. + def destroy_remotely + response = Faraday.delete runner_url + return if response.status == 204 - def status - # TODO: return actual state retrieved via websocket - :timeouted + if response.status == 404 + raise Runner::Error::NotFound.new('Runner not found') + else + handle_error response + end end private - def new_runner + def request_remotely + return if runner_id.present? + url = "#{BASE_URL}/runners" body = {executionEnvironmentId: execution_environment.id, inactivityTimeout: UNUSED_EXPIRATION_TIME} response = Faraday.post(url, body.to_json, HEADERS) - response_body = parse response - self.runner_id = response_body[:runnerId] - throw :abort unless response.status == 200 + + case response.status + when 200 + response_body = parse response + runner_id = response_body[:runnerId] + throw(:abort) if runner_id.blank? + self.runner_id = response_body[:runnerId] + when 404 + raise Runner::Error::NotFound.new('Execution environment not found') + else + handle_error response + end + end + + def handle_error(response) + case response.status + when 400 + response_body = parse response + raise Runner::Error::BadRequest.new(response_body[:message]) + when 401 + raise Runner::Error::Unauthorized.new('Authentication with runner management failed') + when 404 + # The runner does not exist in the runner management (e.g. due to an inactivity timeout). + # Delete the runner model in this case as it can not be used anymore. + destroy + raise Runner::Error::NotFound.new('Runner not found') + when 500 + response_body = parse response + error_code = response_body[:errorCode] + if error_code == Runner.error_nomad_overload + raise Runner::Error::NotAvailable.new("No runner available (#{error_code}): #{response_body[:message]}") + else + raise Runner::Error::InternalServerError.new("#{response_body[:errorCode]}: #{response_body[:message]}") + end + else + raise Runner::Error::Unknown.new('Runner management sent unexpected response') + end end def runner_url @@ -82,5 +131,8 @@ class Runner < ApplicationRecord def parse(response) JSON.parse(response.body).deep_symbolize_keys + rescue JSON::ParserError => e + # the runner management should not send invalid json + raise Runner::Error::Unknown.new("Error parsing response from runner management: #{e.message}") end end diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index ea3840a6..55378e04 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -4,16 +4,20 @@ require 'faye/websocket/client' require 'json_schemer' class Runner::Connection + # These are events for which callbacks can be registered. EVENTS = %i[start output exit stdout stderr].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) def initialize(url) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) + # For every event type of faye websockets, the corresponding + # RunnerConnection method starting with `on_` is called. %i[open message error close].each do |event_type| @socket.on(event_type) {|event| __send__(:"on_#{event_type}", event) } end + # This registers empty default callbacks. EVENTS.each {|event_type| instance_variable_set(:"@#{event_type}_callback", ->(e) {}) } @start_callback = -> {} @exit_code = 0 @@ -43,6 +47,7 @@ class Runner::Connection return unless BACKEND_OUTPUT_SCHEMA.valid?(JSON.parse(event.data)) event = decode(event.data) + # There is one `handle_` method for every message type defined in the WebSocket schema. __send__("handle_#{event[:type]}", event) end @@ -50,7 +55,7 @@ class Runner::Connection @start_callback.call end - def on_error(event); end + def on_error(_event); end def on_close(_event) @exit_callback.call @exit_code @@ -70,11 +75,11 @@ class Runner::Connection @output_callback.call event[:data] end - def handle_error(event) end + def handle_error(_event); end - def handle_start(event) end + def handle_start(_event); end - def handle_timeout(event) - # TODO: set the runner state + def handle_timeout(_event) + raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') end end From 0978a3be83e00952ab661a04741dc8b8d45be1fe Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 1 Jun 2021 18:38:52 +0200 Subject: [PATCH 029/156] Add tests for the different runner errors --- spec/factories/runner.rb | 11 ++ spec/models/runner_spec.rb | 357 +++++++++++++++++++++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 spec/factories/runner.rb create mode 100644 spec/models/runner_spec.rb diff --git a/spec/factories/runner.rb b/spec/factories/runner.rb new file mode 100644 index 00000000..809d134e --- /dev/null +++ b/spec/factories/runner.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# This factory does not request the runner management as the id is already provided. +FactoryBot.define do + factory :runner do + runner_id { 'test-runner-id' } + association :execution_environment, factory: :ruby + association :user, factory: :external_user + waiting_time { 1.0 } + end +end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb new file mode 100644 index 00000000..2c5c4b84 --- /dev/null +++ b/spec/models/runner_spec.rb @@ -0,0 +1,357 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Runner do + let(:runner) { FactoryBot.create :runner } + let(:runner_id) { runner.runner_id } + let(:error_message) { 'test error message' } + let(:response_body) { nil } + let(:user) { FactoryBot.build :external_user } + let(:execution_environment) { FactoryBot.create :ruby } + + # All requests handle a BadRequest (400) response the same way. + shared_examples 'BadRequest (400) error handling' do + let(:response_body) { {message: error_message}.to_json } + let(:response_status) { 400 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/) + end + end + + # All requests handle a Unauthorized (401) response the same way. + shared_examples 'Unauthorized (401) error handling' do + let(:response_status) { 401 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unauthorized) + end + end + + # All requests except creation and destruction handle a NotFound (404) response the same way. + shared_examples 'NotFound (404) error handling' do + let(:response_status) { 404 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) + end + + it 'destroys the runner locally' do + expect { action.call }.to change(described_class, :count).by(-1) + .and raise_error(Runner::Error::NotFound) + end + end + + # All requests handle an InternalServerError (500) response the same way. + shared_examples 'InternalServerError (500) error handling' do + shared_examples 'InternalServerError (500) with error code' do |error_code, error_class| + let(:response_status) { 500 } + let(:response_body) { {message: error_message, errorCode: error_code}.to_json } + + it 'raises an error' do + expect { action.call }.to raise_error(error_class) do |error| + expect(error.message).to match(/#{error_message}/) + expect(error.message).to match(/#{error_code}/) + end + end + end + + context 'when error code is nomad overload' do + include_examples( + 'InternalServerError (500) with error code', + described_class.error_nomad_overload, Runner::Error::NotAvailable + ) + end + + context 'when error code is not nomad overload' do + include_examples( + 'InternalServerError (500) with error code', + described_class.error_unknown, Runner::Error::InternalServerError + ) + end + end + + # All requests handle an unknown response status the same way. + shared_examples 'unknown response status error handling' do + let(:response_status) { 1337 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + describe 'attribute validation' do + let(:runner) { FactoryBot.create :runner } + + it 'validates the presence of the runner id' do + described_class.skip_callback(:validation, :before, :request_remotely) + runner.update(runner_id: nil) + expect(runner.errors[:runner_id]).to be_present + described_class.set_callback(:validation, :before, :request_remotely) + end + + it 'validates the presence of an execution environment' do + runner.update(execution_environment: nil) + expect(runner.errors[:execution_environment]).to be_present + end + + it 'validates the presence of a user' do + runner.update(user: nil) + expect(runner.errors[:user]).to be_present + end + end + + describe 'creation' do + let(:action) { -> { described_class.create(user: user, execution_environment: execution_environment) } } + let!(:create_stub) do + WebMock + .stub_request(:post, "#{Runner::BASE_URL}/runners") + .with( + body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + context 'when a runner is created' do + let(:response_body) { {runnerId: runner_id}.to_json } + let(:response_status) { 200 } + + it 'requests a runner from the runner management' do + action.call + expect(create_stub).to have_been_requested.once + end + + it 'does not call the runner management again when updating' do + runner = action.call + runner.runner_id = 'another_id' + successfully_saved = runner.save + expect(successfully_saved).to be_truthy + expect(create_stub).to have_been_requested.once + end + end + + context 'when the runner management returns Ok (200) with an id' do + let(:response_body) { {runnerId: runner_id}.to_json } + let(:response_status) { 200 } + + it 'sets the runner id according to the response' do + runner = action.call + expect(runner.runner_id).to eq(runner_id) + expect(runner).to be_persisted + end + end + + context 'when the runner management returns Ok (200) without an id' do + let(:response_body) { {}.to_json } + let(:response_status) { 200 } + + it 'does not save the runner' do + runner = action.call + expect(runner).not_to be_persisted + end + end + + context 'when the runner management returns Ok (200) with invalid JSON' do + let(:response_body) { '{hello}' } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + context 'when the runner management returns BadRequest (400)' do + include_examples 'BadRequest (400) error handling' + end + + context 'when the runner management returns Unauthorized (401)' do + include_examples 'Unauthorized (401) error handling' + end + + context 'when the runner management returns NotFound (404)' do + let(:response_status) { 404 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/) + end + end + + context 'when the runner management returns InternalServerError (500)' do + include_examples 'InternalServerError (500) error handling' + end + + context 'when the runner management returns an unknown response status' do + include_examples 'unknown response status error handling' + end + end + + describe 'execute command' do + let(:command) { 'ls' } + let(:action) { -> { runner.execute_command(command) } } + let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } + let!(:execute_command_stub) do + WebMock + .stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute") + .with( + body: {command: command, timeLimit: execution_environment.permitted_execution_time}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + context 'when #execute_command is called' do + let(:response_status) { 200 } + let(:response_body) { {websocketUrl: websocket_url}.to_json } + + it 'schedules an execution in the runner management' do + action.call + expect(execute_command_stub).to have_been_requested.once + end + end + + context 'when the runner management returns Ok (200) with a websocket url' do + let(:response_status) { 200 } + let(:response_body) { {websocketUrl: websocket_url}.to_json } + + it 'returns the url' do + url = action.call + expect(url).to eq(websocket_url) + end + end + + context 'when the runner management returns Ok (200) without a websocket url' do + let(:response_body) { {}.to_json } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + context 'when the runner management returns Ok (200) with invalid JSON' do + let(:response_body) { '{hello}' } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + context 'when the runner management returns BadRequest (400)' do + include_examples 'BadRequest (400) error handling' + end + + context 'when the runner management returns Unauthorized (401)' do + include_examples 'Unauthorized (401) error handling' + end + + context 'when the runner management returns NotFound (404)' do + include_examples 'NotFound (404) error handling' + end + + context 'when the runner management returns InternalServerError (500)' do + include_examples 'InternalServerError (500) error handling' + end + + context 'when the runner management returns an unknown response status' do + include_examples 'unknown response status error handling' + end + end + + describe 'destruction' do + let(:action) { -> { runner.destroy_remotely } } + let(:response_status) { 204 } + let!(:destroy_stub) do + WebMock + .stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}") + .to_return(body: response_body, status: response_status) + end + + it 'deletes the runner from the runner management' do + action.call + expect(destroy_stub).to have_been_requested.once + end + + it 'does not destroy the runner locally' do + expect { action.call }.not_to change(described_class, :count) + end + + context 'when the runner management returns NoContent (204)' do + it 'does not raise an error' do + expect { action.call }.not_to raise_error + end + end + + context 'when the runner management returns Unauthorized (401)' do + include_examples 'Unauthorized (401) error handling' + end + + context 'when the runner management returns NotFound (404)' do + let(:response_status) { 404 } + + it 'raises an exception' do + expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) + end + end + + context 'when the runner management returns InternalServerError (500)' do + include_examples 'InternalServerError (500) error handling' + end + + context 'when the runner management returns an unknown response status' do + include_examples 'unknown response status error handling' + end + end + + describe 'copy files' do + let(:filename) { 'main.py' } + let(:file_content) { 'print("Hello World!")' } + let(:action) { -> { runner.copy_files({filename => file_content}) } } + let(:encoded_file_content) { Base64.strict_encode64(file_content) } + let(:response_status) { 204 } + let!(:copy_files_stub) do + WebMock + .stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files") + .with( + body: {copy: [{path: filename, content: encoded_file_content}]}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + it 'sends the files to the runner management' do + action.call + expect(copy_files_stub).to have_been_requested.once + end + + context 'when the runner management returns NoContent (204)' do + let(:response_status) { 204 } + + it 'does not raise an error' do + expect { action.call }.not_to raise_error + end + end + + context 'when the runner management returns BadRequest (400)' do + include_examples 'BadRequest (400) error handling' + end + + context 'when the runner management returns Unauthorized (401)' do + include_examples 'Unauthorized (401) error handling' + end + + context 'when the runner management returns NotFound (404)' do + include_examples 'NotFound (404) error handling' + end + + context 'when the runner management returns InternalServerError (500)' do + include_examples 'InternalServerError (500) error handling' + end + + context 'when the runner management returns an unknown response status' do + include_examples 'unknown response status error handling' + end + end +end From 598de3bcff1020d3e82472d8a1ec4d615c535bd9 Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Mon, 7 Jun 2021 16:31:15 +0200 Subject: [PATCH 030/156] Add button to synchronize all execution environments This adds a button to the execution environment index page that, when clicked, causes all execution environments to be synchronized to the runner management (Poseidon) by creating or replacing them. CodeOcean does not synchronize it's execution environments on startup or when a new runner management configuration is used for the first time. The administrator has to manually start this process by pressing this button. The equivalent for syncing just one execution environment is updating it. --- .../execution_environments_controller.rb | 13 +++++++++++++ app/policies/execution_environment_policy.rb | 4 ++++ app/views/execution_environments/index.html.slim | 7 ++++++- config/locales/de.yml | 4 ++++ config/locales/en.yml | 4 ++++ config/routes.rb | 2 ++ 6 files changed, 33 insertions(+), 1 deletion(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 95132b09..e9a0ba0c 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -164,6 +164,19 @@ class ExecutionEnvironmentsController < ApplicationController end end + def synchronize_all_to_poseidon + authorize ExecutionEnvironment + + return unless RUNNER_MANAGEMENT_PRESENT + + success = ExecutionEnvironment.all.map(&:copy_to_poseidon).all? + if success + redirect_to ExecutionEnvironment, notice: t('execution_environments.index.synchronize_all.success') + else + redirect_to ExecutionEnvironment, alert: t('execution_environments.index.synchronize_all.failure') + end + end + def copy_execution_environment_to_poseidon unless RUNNER_MANAGEMENT_PRESENT && @execution_environment.copy_to_poseidon t('execution_environments.form.errors.not_synced_to_poseidon') diff --git a/app/policies/execution_environment_policy.rb b/app/policies/execution_environment_policy.rb index 9ed8522c..cf134527 100644 --- a/app/policies/execution_environment_policy.rb +++ b/app/policies/execution_environment_policy.rb @@ -8,4 +8,8 @@ class ExecutionEnvironmentPolicy < AdminOnlyPolicy [:index?].each do |action| define_method(action) { admin? || teacher? } end + + def synchronize_all_to_poseidon? + admin? + end end diff --git a/app/views/execution_environments/index.html.slim b/app/views/execution_environments/index.html.slim index 10501911..9f6d47cc 100644 --- a/app/views/execution_environments/index.html.slim +++ b/app/views/execution_environments/index.html.slim @@ -1,4 +1,9 @@ -h1 = ExecutionEnvironment.model_name.human(count: 2) +h1.d-inline-block = ExecutionEnvironment.model_name.human(count: 2) + +- if ExecutionEnvironment::RUNNER_MANAGEMENT_PRESENT + = button_to( { action: :synchronize_all_to_poseidon, method: :post }, { form_class: 'float-right mb-2', class: 'btn btn-success' }) + i.fa.fa-upload + = t('execution_environments.index.synchronize_all.button') .table-responsive table.table diff --git a/config/locales/de.yml b/config/locales/de.yml index 344f1d58..abab5b43 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -287,6 +287,10 @@ de: not_synced_to_poseidon: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zu Poseidon synchronisiert. index: shell: Shell + synchronize_all: + button: Alle synchronisieren + success: Alle Ausführungsumgebungen wurden erfolgreich synchronisiert. + failure: Beim Synchronisieren mindestens einer Ausführungsumgebung ist ein Fehler aufgetreten. shell: command: Befehl headline: Shell diff --git a/config/locales/en.yml b/config/locales/en.yml index 183a1186..dd241006 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -287,6 +287,10 @@ en: not_synced_to_poseidon: The ExecutionEnvironment was created but not synced to Poseidon due to an error. index: shell: Shell + synchronize_all: + button: Synchronize all + success: All execution environemnts were synchronized successfully. + failure: At least one execution environment could not be synchronised due to an error. shell: command: Command headline: Shell diff --git a/config/routes.rb b/config/routes.rb index 1d578a76..1608ac05 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -66,6 +66,8 @@ Rails.application.routes.draw do post 'shell', as: :execute_command, action: :execute_command get :statistics end + + post :synchronize_all_to_poseidon, on: :collection end post '/import_exercise' => 'exercises#import_exercise' From 0280c0282e98e2236b19011576681a2e113476de Mon Sep 17 00:00:00 2001 From: Konrad Hanff Date: Tue, 8 Jun 2021 13:58:09 +0200 Subject: [PATCH 031/156] Add tests for synchronizing all execution environments This adds policy tests to ensure only an admin can synchronize all execution environments. It also adds controller tests that check that all execution environments get synchronized. --- .../execution_environments_controller_spec.rb | 16 ++++++++++++++++ .../execution_environment_policy_spec.rb | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index 3507fc8a..fa76b4dd 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -192,4 +192,20 @@ describe ExecutionEnvironmentsController do end end end + + describe '#synchronize_all_to_poseidon' do + let(:execution_environments) { FactoryBot.build_list(:ruby, 3) } + + it 'copies all execution environments to Poseidon' do + allow(ExecutionEnvironment).to receive(:all).and_return(execution_environments) + + execution_environments.each do |execution_environment| + allow(execution_environment).to receive(:copy_to_poseidon).and_return(true) + end + + post :synchronize_all_to_poseidon + + expect(execution_environments).to all(have_received(:copy_to_poseidon).once) + end + end end diff --git a/spec/policies/execution_environment_policy_spec.rb b/spec/policies/execution_environment_policy_spec.rb index 41715025..9d0b2539 100644 --- a/spec/policies/execution_environment_policy_spec.rb +++ b/spec/policies/execution_environment_policy_spec.rb @@ -58,4 +58,20 @@ describe ExecutionEnvironmentPolicy do end end end + + permissions(:synchronize_all_to_poseidon?) do + it 'grants access to the admin' do + expect(policy).to permit(FactoryBot.build(:admin)) + end + + shared_examples 'it does not grant access' do |user| + it "does not grant access to a user with role #{user.role}" do + expect(policy).not_to permit(user) + end + end + + %i[teacher external_user].each do |user| + include_examples 'it does not grant access', FactoryBot.build(user) + end + end end From cf58be97ee72abd4d2d85d90d856b0dfdced7f0f Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 8 Jun 2021 12:36:49 +0200 Subject: [PATCH 032/156] Fix 17 previously failing specs --- .../concerns/submission_scoring.rb | 25 ++++--------- app/controllers/exercises_controller.rb | 3 +- .../remote_evaluation_controller.rb | 3 +- .../request_for_comments_controller.rb | 4 +-- app/controllers/submissions_controller.rb | 31 +++++++--------- spec/concerns/submission_scoring_spec.rb | 36 +++++++++---------- spec/controllers/exercises_controller_spec.rb | 28 ++++++++++++--- spec/features/editor_spec.rb | 7 ++-- 8 files changed, 66 insertions(+), 71 deletions(-) diff --git a/app/controllers/concerns/submission_scoring.rb b/app/controllers/concerns/submission_scoring.rb index 85da5106..ebcfb3d2 100644 --- a/app/controllers/concerns/submission_scoring.rb +++ b/app/controllers/concerns/submission_scoring.rb @@ -31,38 +31,27 @@ module SubmissionScoring LinterCheckRun.create_from(testrun, assessment) assessment = assessor.translate_linter(assessment, I18n.locale) - # replace file name with hint if linter is not used for grading. Refactor! - filename = t('exercises.implement.not_graded') if file.weight.zero? - end + # replace file name with hint if linter is not used for grading. Refactor! + filename = t('exercises.implement.not_graded', locale: :de) if file.weight.zero? + end output.merge!(assessment) output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight) end - private :collect_test_results - - def execute_test_file(file, submission) - # TODO: replace DockerClient here - DockerClient.new(execution_environment: file.context.execution_environment).execute_test_command(submission, - file.name_with_extension) - end - - private :execute_test_file - - def feedback_message(_file, output) - # TODO: why did we comment out set_locale and render_markdown? - set_locale + # TODO: make this a controller concern again to bring the locales nearer to the view + def feedback_message(file, output) if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' I18n.t('exercises.implement.default_test_feedback') elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' I18n.t('exercises.implement.default_linter_feedback') else - render_markdown(file.feedback_message) + # render_markdown(file.feedback_message) + file.feedback_message end end def score_submission(outputs) - # outputs = collect_test_results(submission) submission = self score = 0.0 if outputs.present? diff --git a/app/controllers/exercises_controller.rb b/app/controllers/exercises_controller.rb index f3507103..af9b0086 100644 --- a/app/controllers/exercises_controller.rb +++ b/app/controllers/exercises_controller.rb @@ -4,7 +4,6 @@ class ExercisesController < ApplicationController include CommonBehavior include Lti include SubmissionParameters - include SubmissionScoring include TimeHelper before_action :handle_file_uploads, only: %i[create update] @@ -533,7 +532,7 @@ working_time_accumulated: working_time_accumulated}) def submit @submission = Submission.create(submission_params) - score_submission(@submission) + @submission.calculate_score if @submission.user.external_user? && lti_outcome_service?(@submission.exercise_id, @submission.user.id) transmit_lti_score else diff --git a/app/controllers/remote_evaluation_controller.rb b/app/controllers/remote_evaluation_controller.rb index 2dd3bee7..96db6c8b 100644 --- a/app/controllers/remote_evaluation_controller.rb +++ b/app/controllers/remote_evaluation_controller.rb @@ -2,7 +2,6 @@ class RemoteEvaluationController < ApplicationController include RemoteEvaluationParameters - include SubmissionScoring include Lti skip_after_action :verify_authorized @@ -63,7 +62,7 @@ status: 202} validation_token = remote_evaluation_params[:validation_token] if (remote_evaluation_mapping = RemoteEvaluationMapping.find_by(validation_token: validation_token)) @submission = Submission.create(build_submission_params(cause, remote_evaluation_mapping)) - score_submission(@submission) + @submission.calculate_score else # TODO: better output # TODO: check token expired? diff --git a/app/controllers/request_for_comments_controller.rb b/app/controllers/request_for_comments_controller.rb index f3959af4..2fd4046e 100644 --- a/app/controllers/request_for_comments_controller.rb +++ b/app/controllers/request_for_comments_controller.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class RequestForCommentsController < ApplicationController - include SubmissionScoring - before_action :require_user! before_action :set_request_for_comment, only: %i[show mark_as_solved set_thank_you_note] before_action :set_study_group_grouping, @@ -121,7 +119,7 @@ class RequestForCommentsController < ApplicationController if @request_for_comment.save # create thread here and execute tests. A run is triggered from the frontend and does not need to be handled here. Thread.new do - score_submission(@request_for_comment.submission) + @request_for_comment.submission.calculate_score ensure ActiveRecord::Base.connection_pool.release_connection end diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 93bdab1d..55972d54 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -5,7 +5,6 @@ class SubmissionsController < ApplicationController include CommonBehavior include Lti include SubmissionParameters - include SubmissionScoring include Tubesock::Hijack before_action :set_submission, @@ -237,25 +236,21 @@ class SubmissionsController < ApplicationController end def score - Thread.new do - hijack do |tubesock| - return kill_socket(tubesock) if @embed_options[:disable_run] + hijack do |tubesock| + return kill_socket(tubesock) if @embed_options[:disable_run] - tubesock.send_data(@submission.calculate_score) - # To enable hints when scoring a submission, uncomment the next line: - # send_hints(tubesock, StructuredError.where(submission: @submission)) - rescue Runner::Error::ExecutionTimeout => e - tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) - Rails.logger.debug { "Running a submission failed: #{e.message}" } - rescue Runner::Error => e - tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) - Rails.logger.debug { "Runner error while scoring a submission: #{e.message}" } - ensure - tubesock.send_data JSON.dump({cmd: :exit}) - tubesock.close - end + tubesock.send_data(@submission.calculate_score) + # To enable hints when scoring a submission, uncomment the next line: + # send_hints(tubesock, StructuredError.where(submission: @submission)) + rescue Runner::Error::ExecutionTimeout => e + tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) + Rails.logger.debug { "Running a submission failed: #{e.message}" } + rescue Runner::Error => e + tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) + Rails.logger.debug { "Runner error while scoring a submission: #{e.message}" } ensure - ActiveRecord::Base.connection_pool.release_connection + tubesock.send_data JSON.dump({cmd: :exit}) + tubesock.close end end diff --git a/spec/concerns/submission_scoring_spec.rb b/spec/concerns/submission_scoring_spec.rb index 086b2ae7..74d97ab8 100644 --- a/spec/concerns/submission_scoring_spec.rb +++ b/spec/concerns/submission_scoring_spec.rb @@ -2,36 +2,34 @@ require 'rails_helper' -class Controller < AnonymousController - include SubmissionScoring -end - -# This is broken since the Runner was added. -describe SubmissionScoring, skip: true do - let(:controller) { Controller.new } +describe SubmissionScoring do let(:submission) { FactoryBot.create(:submission, cause: 'submit') } - before do - controller.instance_variable_set(:@current_user, FactoryBot.create(:external_user)) - controller.instance_variable_set(:@_params, {}) - end - describe '#collect_test_results' do - after { controller.send(:collect_test_results, submission) } + let(:runner) { FactoryBot.create :runner } + + before do + allow(Runner).to receive(:for).and_return(runner) + allow(runner).to receive(:copy_files) + allow(runner).to receive(:execute_interactively).and_return(1.0) + end + + after { submission.calculate_score } it 'executes every teacher-defined test file' do + allow(submission).to receive(:score_submission) submission.collect_files.select(&:teacher_defined_assessment?).each do |file| - allow(controller).to receive(:execute_test_file).with(file, submission).and_return({}) + allow(submission).to receive(:test_result).with(any_args, file).and_return({}) end end + + it 'scores the submission' do + allow(submission).to receive(:score_submission).and_return([]) + end end describe '#score_submission', cleaning_strategy: :truncation do - after { controller.score_submission(submission) } - - it 'collects the test results' do - allow(controller).to receive(:collect_test_results).and_return([]) - end + after { submission.score_submission([]) } it 'assigns a score to the submissions' do expect(submission).to receive(:update).with(score: anything) diff --git a/spec/controllers/exercises_controller_spec.rb b/spec/controllers/exercises_controller_spec.rb index 6d75ef64..fa0c7628 100644 --- a/spec/controllers/exercises_controller_spec.rb +++ b/spec/controllers/exercises_controller_spec.rb @@ -236,17 +236,35 @@ describe ExercisesController do expect_template(:statistics) end - # This is broken since the Runner was added. - describe 'POST #submit', skip: true do + describe 'POST #submit' do let(:output) { {} } let(:perform_request) { post :submit, format: :json, params: {id: exercise.id, submission: {cause: 'submit', exercise_id: exercise.id}} } let(:user) { FactoryBot.create(:external_user) } + let(:scoring_response) do + [{ + status: 'ok', + stdout: '', + stderr: '', + waiting_for_container_time: 0, + container_execution_time: 0, + file_role: 'teacher_defined_test', + count: 1, + failed: 0, + error_messages: [], + passed: 1, + score: 1.0, + filename: 'index.html_spec.rb', + message: 'Well done.', + weight: 2.0, + }] + end before do FactoryBot.create(:lti_parameter, external_user: user, exercise: exercise) - allow_any_instance_of(Submission).to receive(:normalized_score).and_return(1) - allow(controller).to receive(:collect_test_results).and_return([{score: 1, weight: 1}]) - allow(controller).to receive(:score_submission).and_call_original + submission = FactoryBot.build(:submission, exercise: exercise, user: user) + allow(submission).to receive(:normalized_score).and_return(1) + allow(submission).to receive(:calculate_score).and_return(JSON.dump(scoring_response)) + allow(Submission).to receive(:create).and_return(submission) end context 'when LTI outcomes are supported' do diff --git a/spec/features/editor_spec.rb b/spec/features/editor_spec.rb index 1a8152ef..2f2b1126 100644 --- a/spec/features/editor_spec.rb +++ b/spec/features/editor_spec.rb @@ -94,10 +94,9 @@ describe 'Editor', js: true do end it 'contains a button for submitting the exercise' do - # This is broken since the Runner was added. - skip - - allow_any_instance_of(SubmissionsController).to receive(:score_submission).and_return(scoring_response) + submission = FactoryBot.build(:submission, user: user, exercise: exercise) + allow(submission).to receive(:calculate_score).and_return(JSON.dump(scoring_response)) + allow(Submission).to receive(:find).and_return(submission) click_button(I18n.t('exercises.editor.score')) expect(page).not_to have_css('#submit_outdated') expect(page).to have_css('#submit') From d0d1b1bffd7f7a6ab0949cab8d0094390ac0708d Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:39:56 +0200 Subject: [PATCH 033/156] Introduce strategy for runner behavior The runner model is only a class responsible for storing information now. Based on the configuration it picks a strategy for the runner management. The Poseidon strategy is already implemented and tested. The Docker strategy will follow. --- app/controllers/submissions_controller.rb | 17 +- app/models/runner.rb | 143 ++------- app/models/submission.rb | 23 +- config/code_ocean.yml.ci | 1 + config/code_ocean.yml.example | 1 + lib/runner/strategy.rb | 24 ++ lib/runner/strategy/docker.rb | 3 + lib/runner/strategy/poseidon.rb | 103 ++++++ spec/concerns/submission_scoring_spec.rb | 2 +- spec/factories/runner.rb | 2 +- spec/lib/runner/strategy/docker_spec.rb | 15 + spec/lib/runner/strategy/poseidon_spec.rb | 273 ++++++++++++++++ spec/models/runner_spec.rb | 367 ++++------------------ 13 files changed, 541 insertions(+), 433 deletions(-) create mode 100644 lib/runner/strategy.rb create mode 100644 lib/runner/strategy/docker.rb create mode 100644 lib/runner/strategy/poseidon.rb create mode 100644 spec/lib/runner/strategy/docker_spec.rb create mode 100644 spec/lib/runner/strategy/poseidon_spec.rb diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 55972d54..05e86e4d 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -131,7 +131,7 @@ class SubmissionsController < ApplicationController end end - def handle_websockets(tubesock, runner, socket) + def handle_websockets(tubesock, socket) tubesock.send_data JSON.dump({cmd: :status, status: :container_running}) @output = +'' @@ -150,11 +150,10 @@ class SubmissionsController < ApplicationController socket.on :exit do |exit_code| EventMachine.stop_event_loop - status = runner.status if @output.empty? tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) end - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) unless status == :timeouted + tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) kill_socket(tubesock) end @@ -170,6 +169,7 @@ class SubmissionsController < ApplicationController else Rails.logger.info("Unknown command from client: #{event[:cmd]}") end + rescue JSON::ParserError Rails.logger.debug { "Data received from client is not valid json: #{data}" } Sentry.set_extras(data: data) @@ -183,15 +183,16 @@ class SubmissionsController < ApplicationController hijack do |tubesock| return kill_socket(tubesock) if @embed_options[:disable_run] - @container_execution_time = @submission.run(sanitize_filename) do |runner, socket| - @waiting_for_container_time = runner.waiting_time - handle_websockets(tubesock, runner, socket) + durations = @submission.run(sanitize_filename) do |socket| + handle_websockets(tubesock, socket) end + @container_execution_time = durations[:execution_duration] + @waiting_for_container_time = durations[:waiting_duration] save_run_output rescue Runner::Error::ExecutionTimeout => e tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) kill_socket(tubesock) - Rails.logger.debug { "Running a submission failed: #{e.message}" } + Rails.logger.debug { "Running a submission timed out: #{e.message}" } rescue Runner::Error => e tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) kill_socket(tubesock) @@ -244,7 +245,7 @@ class SubmissionsController < ApplicationController # send_hints(tubesock, StructuredError.where(submission: @submission)) rescue Runner::Error::ExecutionTimeout => e tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) - Rails.logger.debug { "Running a submission failed: #{e.message}" } + Rails.logger.debug { "Scoring a submission timed out: #{e.message}" } rescue Runner::Error => e tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) Rails.logger.debug { "Runner error while scoring a submission: #{e.message}" } diff --git a/app/models/runner.rb b/app/models/runner.rb index f8861a2a..707563bc 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -1,138 +1,57 @@ # frozen_string_literal: true +require 'forwardable' + class Runner < ApplicationRecord - BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] - HEADERS = {'Content-Type' => 'application/json'}.freeze - UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds - ERRORS = %w[NOMAD_UNREACHABLE NOMAD_OVERLOAD NOMAD_INTERNAL_SERVER_ERROR UNKNOWN].freeze - - ERRORS.each do |error| - define_singleton_method :"error_#{error.downcase}" do - error - end - end - belongs_to :execution_environment belongs_to :user, polymorphic: true - before_validation :request_remotely + before_validation :request_id validates :execution_environment, :user, :runner_id, presence: true + STRATEGY_NAME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] + UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds + BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] + DELEGATED_STRATEGY_METHODS = %i[destroy_at_management attach_to_execution copy_files].freeze + + attr_accessor :strategy + + def self.strategy_class + "runner/strategy/#{STRATEGY_NAME}".camelize.constantize + end + def self.for(user, exercise) execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) - runner = find_or_create_by(user: user, execution_environment: execution_environment) - unless runner.persisted? - # runner was not saved in the database (was not valid) - raise Runner::Error::InternalServerError.new("Provided runner could not be saved: #{runner.errors.inspect}") + runner = find_by(user: user, execution_environment: execution_environment) + if runner.nil? + runner = Runner.create(user: user, execution_environment: execution_environment) + raise Runner::Error::Unknown.new("Runner could not be saved: #{runner.errors.inspect}") unless runner.persisted? + else + runner.strategy = strategy_class.new(runner.runner_id, runner.execution_environment) end runner end - def copy_files(files) - url = "#{runner_url}/files" - body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} - response = Faraday.patch(url, body.to_json, HEADERS) - handle_error response unless response.status == 204 - end - - def execute_command(command) - url = "#{runner_url}/execute" - body = {command: command, timeLimit: execution_environment.permitted_execution_time} - response = Faraday.post(url, body.to_json, HEADERS) - if response.status == 200 - response_body = parse response - websocket_url = response_body[:websocketUrl] - if websocket_url.present? - return websocket_url - else - raise Runner::Error::Unknown.new('Runner management sent unexpected response') - end - end - - handle_error response - end - - def execute_interactively(command) - starting_time = Time.zone.now - websocket_url = execute_command(command) - EventMachine.run do - socket = Runner::Connection.new(websocket_url) - yield(self, socket) if block_given? - end - Time.zone.now - starting_time # execution time - end - - # This method is currently not used. - # This does *not* destroy the ActiveRecord model. - def destroy_remotely - response = Faraday.delete runner_url - return if response.status == 204 - - if response.status == 404 - raise Runner::Error::NotFound.new('Runner not found') - else - handle_error response + DELEGATED_STRATEGY_METHODS.each do |method| + define_method(method) do |*args, &block| + @strategy.send(method, *args, &block) + rescue Runner::Error::NotFound + update(runner_id: self.class.strategy_class.request_from_management(execution_environment)) + @strategy = self.class.strategy_class.new(runner_id, execution_environment) + @strategy.send(method, *args, &block) end end private - def request_remotely + def request_id return if runner_id.present? - url = "#{BASE_URL}/runners" - body = {executionEnvironmentId: execution_environment.id, inactivityTimeout: UNUSED_EXPIRATION_TIME} - response = Faraday.post(url, body.to_json, HEADERS) - - case response.status - when 200 - response_body = parse response - runner_id = response_body[:runnerId] - throw(:abort) if runner_id.blank? - self.runner_id = response_body[:runnerId] - when 404 - raise Runner::Error::NotFound.new('Execution environment not found') - else - handle_error response - end - end - - def handle_error(response) - case response.status - when 400 - response_body = parse response - raise Runner::Error::BadRequest.new(response_body[:message]) - when 401 - raise Runner::Error::Unauthorized.new('Authentication with runner management failed') - when 404 - # The runner does not exist in the runner management (e.g. due to an inactivity timeout). - # Delete the runner model in this case as it can not be used anymore. - destroy - raise Runner::Error::NotFound.new('Runner not found') - when 500 - response_body = parse response - error_code = response_body[:errorCode] - if error_code == Runner.error_nomad_overload - raise Runner::Error::NotAvailable.new("No runner available (#{error_code}): #{response_body[:message]}") - else - raise Runner::Error::InternalServerError.new("#{response_body[:errorCode]}: #{response_body[:message]}") - end - else - raise Runner::Error::Unknown.new('Runner management sent unexpected response') - end - end - - def runner_url - "#{BASE_URL}/runners/#{runner_id}" - end - - def parse(response) - JSON.parse(response.body).deep_symbolize_keys - rescue JSON::ParserError => e - # the runner management should not send invalid json - raise Runner::Error::Unknown.new("Error parsing response from runner management: #{e.message}") + strategy_class = self.class.strategy_class + self.runner_id = strategy_class.request_from_management(execution_environment) + @strategy = strategy_class.new(runner_id, execution_environment) end end diff --git a/app/models/submission.rb b/app/models/submission.rb index 249fdc3f..c2fd3121 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -141,13 +141,13 @@ class Submission < ApplicationRecord def calculate_score score = nil - prepared_runner do |runner| + prepared_runner do |runner, waiting_duration| scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension - stdout = '' - stderr = '' + stdout = +'' + stderr = +'' exit_code = 1 # default to error - execution_time = runner.execute_interactively(score_command) do |_runner, socket| + execution_time = runner.attach_to_execution(score_command) do |socket| socket.on :stderr do |data| stderr << data end @@ -161,7 +161,7 @@ class Submission < ApplicationRecord end output = { file_role: file.role, - waiting_for_container_time: runner.waiting_time, + waiting_for_container_time: waiting_duration, container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed, stdout: stdout, @@ -176,11 +176,12 @@ class Submission < ApplicationRecord def run(file, &block) run_command = command_for execution_environment.run_command, file - execution_time = 0 - prepared_runner do |runner| - execution_time = runner.execute_interactively(run_command, &block) + durations = {} + prepared_runner do |runner, waiting_duration| + durations[:execution_duration] = runner.attach_to_execution(run_command, &block) + durations[:waiting_duration] = waiting_duration end - execution_time + durations end private @@ -197,8 +198,8 @@ class Submission < ApplicationRecord request_time = Time.zone.now runner = Runner.for(user, exercise) copy_files_to runner - runner.waiting_time = Time.zone.now - request_time - yield(runner) if block_given? + waiting_duration = Time.zone.now - request_time + yield(runner, waiting_duration) if block_given? end def command_for(template, file) diff --git a/config/code_ocean.yml.ci b/config/code_ocean.yml.ci index 6865a478..8b8d2a64 100644 --- a/config/code_ocean.yml.ci +++ b/config/code_ocean.yml.ci @@ -10,5 +10,6 @@ test: prometheus_exporter: enabled: false runner_management: + strategy: poseidon url: https://runners.example.org unused_runner_expiration_time: 180 diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index f948ac6a..6b694123 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -12,6 +12,7 @@ default: &default prometheus_exporter: enabled: false runner_management: + strategy: poseidon url: https://runners.example.org unused_runner_expiration_time: 180 diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb new file mode 100644 index 00000000..dce76adf --- /dev/null +++ b/lib/runner/strategy.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class Runner::Strategy + def initialize(runner_id, environment) + @runner_id = runner_id + @execution_environment = environment + end + + def self.request_from_management + raise NotImplementedError + end + + def destroy_at_management + raise NotImplementedError + end + + def copy_files(_files) + raise NotImplementedError + end + + def attach_to_execution(_command) + raise NotImplementedError + end +end diff --git a/lib/runner/strategy/docker.rb b/lib/runner/strategy/docker.rb new file mode 100644 index 00000000..fd7cfafc --- /dev/null +++ b/lib/runner/strategy/docker.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Runner::Strategy::Docker < Runner::Strategy; end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb new file mode 100644 index 00000000..6a642bb8 --- /dev/null +++ b/lib/runner/strategy/poseidon.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +class Runner::Strategy::Poseidon < Runner::Strategy + HEADERS = {'Content-Type' => 'application/json'}.freeze + ERRORS = %w[NOMAD_UNREACHABLE NOMAD_OVERLOAD NOMAD_INTERNAL_SERVER_ERROR UNKNOWN].freeze + + ERRORS.each do |error| + define_singleton_method :"error_#{error.downcase}" do + error + end + end + + def self.request_from_management(environment) + url = "#{Runner::BASE_URL}/runners" + body = {executionEnvironmentId: environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME} + response = Faraday.post(url, body.to_json, HEADERS) + + case response.status + when 200 + response_body = parse response + runner_id = response_body[:runnerId] + runner_id.presence || raise(Runner::Error::Unknown.new('Poseidon did not send a runner id')) + when 404 + raise Runner::Error::NotFound.new('Execution environment not found') + else + handle_error response + end + end + + def self.handle_error(response) + case response.status + when 400 + response_body = parse response + raise Runner::Error::BadRequest.new(response_body[:message]) + when 401 + raise Runner::Error::Unauthorized.new('Authentication with Poseidon failed') + when 404 + raise Runner::Error::NotFound.new('Runner not found') + when 500 + response_body = parse response + error_code = response_body[:errorCode] + if error_code == error_nomad_overload + raise Runner::Error::NotAvailable.new("Poseidon has no runner available (#{error_code}): #{response_body[:message]}") + else + raise Runner::Error::InternalServerError.new("Poseidon sent #{response_body[:errorCode]}: #{response_body[:message]}") + end + else + raise Runner::Error::Unknown.new("Poseidon sent unexpected response status code #{response.status}") + end + end + + def self.parse(response) + JSON.parse(response.body).deep_symbolize_keys + rescue JSON::ParserError => e + # Poseidon should not send invalid json + raise Runner::Error::Unknown.new("Error parsing response from Poseidon: #{e.message}") + end + + def copy_files(files) + url = "#{runner_url}/files" + body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} + response = Faraday.patch(url, body.to_json, HEADERS) + self.class.handle_error response unless response.status == 204 + end + + def attach_to_execution(command) + starting_time = Time.zone.now + websocket_url = execute_command(command) + EventMachine.run do + socket = Runner::Connection.new(websocket_url) + yield(socket) if block_given? + end + Time.zone.now - starting_time # execution duration + end + + def destroy_at_management + response = Faraday.delete runner_url + self.class.handle_error response unless response.status == 204 + end + + private + + def execute_command(command) + url = "#{runner_url}/execute" + body = {command: command, timeLimit: @execution_environment.permitted_execution_time} + response = Faraday.post(url, body.to_json, HEADERS) + if response.status == 200 + response_body = self.class.parse response + websocket_url = response_body[:websocketUrl] + if websocket_url.present? + return websocket_url + else + raise Runner::Error::Unknown.new('Poseidon did not send websocket url') + end + end + + self.class.handle_error response + end + + def runner_url + "#{Runner::BASE_URL}/runners/#{@runner_id}" + end +end diff --git a/spec/concerns/submission_scoring_spec.rb b/spec/concerns/submission_scoring_spec.rb index 74d97ab8..8eb1506f 100644 --- a/spec/concerns/submission_scoring_spec.rb +++ b/spec/concerns/submission_scoring_spec.rb @@ -11,7 +11,7 @@ describe SubmissionScoring do before do allow(Runner).to receive(:for).and_return(runner) allow(runner).to receive(:copy_files) - allow(runner).to receive(:execute_interactively).and_return(1.0) + allow(runner).to receive(:attach_to_execution).and_return(1.0) end after { submission.calculate_score } diff --git a/spec/factories/runner.rb b/spec/factories/runner.rb index 809d134e..636a8055 100644 --- a/spec/factories/runner.rb +++ b/spec/factories/runner.rb @@ -3,7 +3,7 @@ # This factory does not request the runner management as the id is already provided. FactoryBot.define do factory :runner do - runner_id { 'test-runner-id' } + sequence(:runner_id) {|n| "test-runner-id-#{n}" } association :execution_environment, factory: :ruby association :user, factory: :external_user waiting_time { 1.0 } diff --git a/spec/lib/runner/strategy/docker_spec.rb b/spec/lib/runner/strategy/docker_spec.rb new file mode 100644 index 00000000..4110ebf9 --- /dev/null +++ b/spec/lib/runner/strategy/docker_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Runner::Strategy::Docker do + let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } + let(:execution_environment) { FactoryBot.create :ruby } + let(:docker) { described_class.new(runner_id, execution_environment) } + + # TODO: add tests for these methods when implemented + it 'defines all methods all runner management strategies must define' do + expect(docker.public_methods).to include(*Runner::DELEGATED_STRATEGY_METHODS) + expect(described_class.public_methods).to include(:request_from_management) + end +end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb new file mode 100644 index 00000000..e8370351 --- /dev/null +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -0,0 +1,273 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Runner::Strategy::Poseidon do + let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } + let(:execution_environment) { FactoryBot.create :ruby } + let(:poseidon) { described_class.new(runner_id, execution_environment) } + let(:error_message) { 'test error message' } + let(:response_body) { nil } + + # All requests handle a BadRequest (400) response the same way. + shared_examples 'BadRequest (400) error handling' do + context 'when Poseidon returns BadRequest (400)' do + let(:response_body) { {message: error_message}.to_json } + let(:response_status) { 400 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/) + end + end + end + + # All requests handle a Unauthorized (401) response the same way. + shared_examples 'Unauthorized (401) error handling' do + context 'when Poseidon returns Unauthorized (401)' do + let(:response_status) { 401 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unauthorized) + end + end + end + + # All requests except creation handle a NotFound (404) response the same way. + shared_examples 'NotFound (404) error handling' do + context 'when Poseidon returns NotFound (404)' do + let(:response_status) { 404 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) + end + end + end + + # All requests handle an InternalServerError (500) response the same way. + shared_examples 'InternalServerError (500) error handling' do + context 'when Poseidon returns InternalServerError (500)' do + shared_examples 'InternalServerError (500) with error code' do |error_code, error_class| + let(:response_status) { 500 } + let(:response_body) { {message: error_message, errorCode: error_code}.to_json } + + it 'raises an error' do + expect { action.call }.to raise_error(error_class) do |error| + expect(error.message).to match(/#{error_message}/) + expect(error.message).to match(/#{error_code}/) + end + end + end + + context 'when error code is nomad overload' do + include_examples( + 'InternalServerError (500) with error code', + described_class.error_nomad_overload, Runner::Error::NotAvailable + ) + end + + context 'when error code is not nomad overload' do + include_examples( + 'InternalServerError (500) with error code', + described_class.error_unknown, Runner::Error::InternalServerError + ) + end + end + end + + # All requests handle an unknown response status the same way. + shared_examples 'unknown response status error handling' do + context 'when Poseidon returns an unknown response status' do + let(:response_status) { 1337 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown, /#{response_status}/) + end + end + end + + describe '::request_from_management' do + let(:action) { -> { described_class.request_from_management(execution_environment) } } + let!(:request_runner_stub) do + WebMock + .stub_request(:post, "#{Runner::BASE_URL}/runners") + .with( + body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + context 'when Poseidon returns Ok (200) with an id' do + let(:response_body) { {runnerId: runner_id}.to_json } + let(:response_status) { 200 } + + it 'successfully requests Poseidon' do + action.call + expect(request_runner_stub).to have_been_requested.once + end + + it 'returns the received runner id' do + id = action.call + expect(id).to eq(runner_id) + end + end + + context 'when Poseidon returns Ok (200) without an id' do + let(:response_body) { {}.to_json } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + context 'when Poseidon returns Ok (200) with invalid JSON' do + let(:response_body) { '{hello}' } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + include_examples 'BadRequest (400) error handling' + include_examples 'Unauthorized (401) error handling' + + context 'when Poseidon returns NotFound (404)' do + let(:response_status) { 404 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/) + end + end + + include_examples 'InternalServerError (500) error handling' + include_examples 'unknown response status error handling' + end + + describe '#execute_command' do + let(:command) { 'ls' } + let(:action) { -> { poseidon.send(:execute_command, command) } } + let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } + let!(:execute_command_stub) do + WebMock + .stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute") + .with( + body: {command: command, timeLimit: execution_environment.permitted_execution_time}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + context 'when Poseidon returns Ok (200) with a websocket url' do + let(:response_status) { 200 } + let(:response_body) { {websocketUrl: websocket_url}.to_json } + + it 'schedules an execution in Poseidon' do + action.call + expect(execute_command_stub).to have_been_requested.once + end + + it 'returns the url' do + url = action.call + expect(url).to eq(websocket_url) + end + end + + context 'when Poseidon returns Ok (200) without a websocket url' do + let(:response_body) { {}.to_json } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + context 'when Poseidon returns Ok (200) with invalid JSON' do + let(:response_body) { '{hello}' } + let(:response_status) { 200 } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + include_examples 'BadRequest (400) error handling' + include_examples 'Unauthorized (401) error handling' + include_examples 'NotFound (404) error handling' + include_examples 'InternalServerError (500) error handling' + include_examples 'unknown response status error handling' + end + + describe '#destroy_at_management' do + let(:action) { -> { poseidon.destroy_at_management } } + let!(:destroy_stub) do + WebMock + .stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}") + .to_return(body: response_body, status: response_status) + end + + context 'when Poseidon returns NoContent (204)' do + let(:response_status) { 204 } + + it 'deletes the runner from Poseidon' do + action.call + expect(destroy_stub).to have_been_requested.once + end + end + + include_examples 'Unauthorized (401) error handling' + include_examples 'NotFound (404) error handling' + include_examples 'InternalServerError (500) error handling' + include_examples 'unknown response status error handling' + end + + describe '#copy_files' do + let(:filename) { 'main.py' } + let(:file_content) { 'print("Hello World!")' } + let(:action) { -> { poseidon.copy_files({filename => file_content}) } } + let(:encoded_file_content) { Base64.strict_encode64(file_content) } + let!(:copy_files_stub) do + WebMock + .stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files") + .with( + body: {copy: [{path: filename, content: encoded_file_content}]}, + headers: {'Content-Type' => 'application/json'} + ) + .to_return(body: response_body, status: response_status) + end + + context 'when Poseidon returns NoContent (204)' do + let(:response_status) { 204 } + + it 'sends the files to Poseidon' do + action.call + expect(copy_files_stub).to have_been_requested.once + end + end + + include_examples 'BadRequest (400) error handling' + include_examples 'Unauthorized (401) error handling' + include_examples 'NotFound (404) error handling' + include_examples 'InternalServerError (500) error handling' + include_examples 'unknown response status error handling' + end + + describe '#attach_to_execution' do + # TODO: add more tests here + + let(:command) { 'ls' } + let(:action) { -> { poseidon.attach_to_execution command } } + let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } + + it 'returns the execution time' do + allow(poseidon).to receive(:execute_command).with(command).and_return(websocket_url) + allow(EventMachine).to receive(:run) + + starting_time = Time.zone.now + execution_time = action.call + test_time = Time.zone.now - starting_time + expect(execution_time).to be_between(0.0, test_time) + end + end +end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 2c5c4b84..4c3043c0 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -3,92 +3,17 @@ require 'rails_helper' describe Runner do - let(:runner) { FactoryBot.create :runner } - let(:runner_id) { runner.runner_id } - let(:error_message) { 'test error message' } - let(:response_body) { nil } - let(:user) { FactoryBot.build :external_user } - let(:execution_environment) { FactoryBot.create :ruby } - - # All requests handle a BadRequest (400) response the same way. - shared_examples 'BadRequest (400) error handling' do - let(:response_body) { {message: error_message}.to_json } - let(:response_status) { 400 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/) - end - end - - # All requests handle a Unauthorized (401) response the same way. - shared_examples 'Unauthorized (401) error handling' do - let(:response_status) { 401 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unauthorized) - end - end - - # All requests except creation and destruction handle a NotFound (404) response the same way. - shared_examples 'NotFound (404) error handling' do - let(:response_status) { 404 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) - end - - it 'destroys the runner locally' do - expect { action.call }.to change(described_class, :count).by(-1) - .and raise_error(Runner::Error::NotFound) - end - end - - # All requests handle an InternalServerError (500) response the same way. - shared_examples 'InternalServerError (500) error handling' do - shared_examples 'InternalServerError (500) with error code' do |error_code, error_class| - let(:response_status) { 500 } - let(:response_body) { {message: error_message, errorCode: error_code}.to_json } - - it 'raises an error' do - expect { action.call }.to raise_error(error_class) do |error| - expect(error.message).to match(/#{error_message}/) - expect(error.message).to match(/#{error_code}/) - end - end - end - - context 'when error code is nomad overload' do - include_examples( - 'InternalServerError (500) with error code', - described_class.error_nomad_overload, Runner::Error::NotAvailable - ) - end - - context 'when error code is not nomad overload' do - include_examples( - 'InternalServerError (500) with error code', - described_class.error_unknown, Runner::Error::InternalServerError - ) - end - end - - # All requests handle an unknown response status the same way. - shared_examples 'unknown response status error handling' do - let(:response_status) { 1337 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) - end - end + let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } + let(:strategy_class) { described_class.strategy_class } describe 'attribute validation' do let(:runner) { FactoryBot.create :runner } it 'validates the presence of the runner id' do - described_class.skip_callback(:validation, :before, :request_remotely) + described_class.skip_callback(:validation, :before, :request_id) runner.update(runner_id: nil) expect(runner.errors[:runner_id]).to be_present - described_class.set_callback(:validation, :before, :request_remotely) + described_class.set_callback(:validation, :before, :request_id) end it 'validates the presence of an execution environment' do @@ -102,256 +27,98 @@ describe Runner do end end - describe 'creation' do - let(:action) { -> { described_class.create(user: user, execution_environment: execution_environment) } } - let!(:create_stub) do - WebMock - .stub_request(:post, "#{Runner::BASE_URL}/runners") - .with( - body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME}, - headers: {'Content-Type' => 'application/json'} - ) - .to_return(body: response_body, status: response_status) - end - - context 'when a runner is created' do - let(:response_body) { {runnerId: runner_id}.to_json } - let(:response_status) { 200 } - - it 'requests a runner from the runner management' do - action.call - expect(create_stub).to have_been_requested.once - end - - it 'does not call the runner management again when updating' do - runner = action.call - runner.runner_id = 'another_id' - successfully_saved = runner.save - expect(successfully_saved).to be_truthy - expect(create_stub).to have_been_requested.once + describe '::strategy_class' do + shared_examples 'uses the strategy defined in the constant' do |strategy, strategy_class| + it "uses #{strategy_class} as strategy class for constant #{strategy}" do + stub_const('Runner::STRATEGY_NAME', strategy) + expect(described_class.strategy_class).to eq(strategy_class) end end - context 'when the runner management returns Ok (200) with an id' do - let(:response_body) { {runnerId: runner_id}.to_json } - let(:response_status) { 200 } + {poseidon: Runner::Strategy::Poseidon, docker: Runner::Strategy::Docker}.each do |strategy, strategy_class| + include_examples 'uses the strategy defined in the constant', strategy, strategy_class + end - it 'sets the runner id according to the response' do - runner = action.call - expect(runner.runner_id).to eq(runner_id) - expect(runner).to be_persisted + shared_examples 'delegates method sends to its strategy' do |method, *args| + context "when sending #{method}" do + let(:strategy) { instance_double(strategy_class) } + let(:runner) { described_class.create } + + before do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + allow(strategy_class).to receive(:new).and_return(strategy) + end + + it "delegates the method #{method}" do + expect(strategy).to receive(method) + runner.send(method, *args) + end end end - context 'when the runner management returns Ok (200) without an id' do - let(:response_body) { {}.to_json } - let(:response_status) { 200 } + include_examples 'delegates method sends to its strategy', :destroy_at_management + include_examples 'delegates method sends to its strategy', :copy_files, nil + include_examples 'delegates method sends to its strategy', :attach_to_execution, nil + end - it 'does not save the runner' do - runner = action.call - expect(runner).not_to be_persisted - end + describe '#request_id' do + it 'requests a runner from the runner management when created' do + expect(strategy_class).to receive(:request_from_management) + described_class.create end - context 'when the runner management returns Ok (200) with invalid JSON' do - let(:response_body) { '{hello}' } - let(:response_status) { 200 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) - end + it 'sets the runner id when created' do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + runner = described_class.create + expect(runner.runner_id).to eq(runner_id) end - context 'when the runner management returns BadRequest (400)' do - include_examples 'BadRequest (400) error handling' + it 'sets the strategy when created' do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + runner = described_class.create + expect(runner.strategy).to be_present end - context 'when the runner management returns Unauthorized (401)' do - include_examples 'Unauthorized (401) error handling' - end - - context 'when the runner management returns NotFound (404)' do - let(:response_status) { 404 } - - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/) - end - end - - context 'when the runner management returns InternalServerError (500)' do - include_examples 'InternalServerError (500) error handling' - end - - context 'when the runner management returns an unknown response status' do - include_examples 'unknown response status error handling' + it 'does not call the runner management again when validating the model' do + expect(strategy_class).to receive(:request_from_management).and_return(runner_id).once + runner = described_class.create + runner.valid? end end - describe 'execute command' do - let(:command) { 'ls' } - let(:action) { -> { runner.execute_command(command) } } - let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } - let!(:execute_command_stub) do - WebMock - .stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute") - .with( - body: {command: command, timeLimit: execution_environment.permitted_execution_time}, - headers: {'Content-Type' => 'application/json'} - ) - .to_return(body: response_body, status: response_status) - end + describe '::for' do + let(:user) { FactoryBot.create :external_user } + let(:exercise) { FactoryBot.create :fibonacci } - context 'when #execute_command is called' do - let(:response_status) { 200 } - let(:response_body) { {websocketUrl: websocket_url}.to_json } - - it 'schedules an execution in the runner management' do - action.call - expect(execute_command_stub).to have_been_requested.once - end - end - - context 'when the runner management returns Ok (200) with a websocket url' do - let(:response_status) { 200 } - let(:response_body) { {websocketUrl: websocket_url}.to_json } - - it 'returns the url' do - url = action.call - expect(url).to eq(websocket_url) - end - end - - context 'when the runner management returns Ok (200) without a websocket url' do - let(:response_body) { {}.to_json } - let(:response_status) { 200 } + context 'when the runner could not be saved' do + before { allow(strategy_class).to receive(:request_from_management).and_return(nil) } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { described_class.for(user, exercise) }.to raise_error(Runner::Error::Unknown, /could not be saved/) end end - context 'when the runner management returns Ok (200) with invalid JSON' do - let(:response_body) { '{hello}' } - let(:response_status) { 200 } + context 'when a runner already exists' do + let!(:existing_runner) { FactoryBot.create(:runner, user: user, execution_environment: exercise.execution_environment) } - it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + it 'returns the existing runner' do + new_runner = described_class.for(user, exercise) + expect(new_runner).to eq(existing_runner) + end + + it 'sets the strategy' do + runner = described_class.for(user, exercise) + expect(runner.strategy).to be_present end end - context 'when the runner management returns BadRequest (400)' do - include_examples 'BadRequest (400) error handling' - end + context 'when no runner exists' do + before { allow(strategy_class).to receive(:request_from_management).and_return(runner_id) } - context 'when the runner management returns Unauthorized (401)' do - include_examples 'Unauthorized (401) error handling' - end - - context 'when the runner management returns NotFound (404)' do - include_examples 'NotFound (404) error handling' - end - - context 'when the runner management returns InternalServerError (500)' do - include_examples 'InternalServerError (500) error handling' - end - - context 'when the runner management returns an unknown response status' do - include_examples 'unknown response status error handling' - end - end - - describe 'destruction' do - let(:action) { -> { runner.destroy_remotely } } - let(:response_status) { 204 } - let!(:destroy_stub) do - WebMock - .stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}") - .to_return(body: response_body, status: response_status) - end - - it 'deletes the runner from the runner management' do - action.call - expect(destroy_stub).to have_been_requested.once - end - - it 'does not destroy the runner locally' do - expect { action.call }.not_to change(described_class, :count) - end - - context 'when the runner management returns NoContent (204)' do - it 'does not raise an error' do - expect { action.call }.not_to raise_error + it 'returns a new runner' do + runner = described_class.for(user, exercise) + expect(runner).to be_valid end end - - context 'when the runner management returns Unauthorized (401)' do - include_examples 'Unauthorized (401) error handling' - end - - context 'when the runner management returns NotFound (404)' do - let(:response_status) { 404 } - - it 'raises an exception' do - expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) - end - end - - context 'when the runner management returns InternalServerError (500)' do - include_examples 'InternalServerError (500) error handling' - end - - context 'when the runner management returns an unknown response status' do - include_examples 'unknown response status error handling' - end - end - - describe 'copy files' do - let(:filename) { 'main.py' } - let(:file_content) { 'print("Hello World!")' } - let(:action) { -> { runner.copy_files({filename => file_content}) } } - let(:encoded_file_content) { Base64.strict_encode64(file_content) } - let(:response_status) { 204 } - let!(:copy_files_stub) do - WebMock - .stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files") - .with( - body: {copy: [{path: filename, content: encoded_file_content}]}, - headers: {'Content-Type' => 'application/json'} - ) - .to_return(body: response_body, status: response_status) - end - - it 'sends the files to the runner management' do - action.call - expect(copy_files_stub).to have_been_requested.once - end - - context 'when the runner management returns NoContent (204)' do - let(:response_status) { 204 } - - it 'does not raise an error' do - expect { action.call }.not_to raise_error - end - end - - context 'when the runner management returns BadRequest (400)' do - include_examples 'BadRequest (400) error handling' - end - - context 'when the runner management returns Unauthorized (401)' do - include_examples 'Unauthorized (401) error handling' - end - - context 'when the runner management returns NotFound (404)' do - include_examples 'NotFound (404) error handling' - end - - context 'when the runner management returns InternalServerError (500)' do - include_examples 'InternalServerError (500) error handling' - end - - context 'when the runner management returns an unknown response status' do - include_examples 'unknown response status error handling' - end end end From b847daf823c8025c045fb7faba1ba1a6d2f418f0 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Fri, 11 Jun 2021 12:15:13 +0200 Subject: [PATCH 034/156] Remove waiting_time from runner model After removing the logic that stores the duration that has been waited for a runner in the runner, this now also removes the column from the database as it is not used anymore. --- .../20210611101330_remove_waiting_time_from_runners.rb | 7 +++++++ db/schema.rb | 5 ++--- spec/factories/runner.rb | 3 +-- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20210611101330_remove_waiting_time_from_runners.rb diff --git a/db/migrate/20210611101330_remove_waiting_time_from_runners.rb b/db/migrate/20210611101330_remove_waiting_time_from_runners.rb new file mode 100644 index 00000000..5a81c62e --- /dev/null +++ b/db/migrate/20210611101330_remove_waiting_time_from_runners.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class RemoveWaitingTimeFromRunners < ActiveRecord::Migration[6.1] + def change + remove_column :runners, :waiting_time + end +end diff --git a/db/schema.rb b/db/schema.rb index beefc5b1..2772b0ca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_06_02_071834) do +ActiveRecord::Schema.define(version: 2021_06_11_101330) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -112,7 +112,7 @@ ActiveRecord::Schema.define(version: 2021_06_02_071834) do t.integer "file_type_id" t.integer "memory_limit" t.boolean "network_enabled" - t.integer "cpu_limit" + t.integer "cpu_limit", default: 20 end create_table "exercise_collection_items", id: :serial, force: :cascade do |t| @@ -345,7 +345,6 @@ ActiveRecord::Schema.define(version: 2021_06_02_071834) do t.bigint "execution_environment_id" t.string "user_type" t.bigint "user_id" - t.float "waiting_time" t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.index ["execution_environment_id"], name: "index_runners_on_execution_environment_id" diff --git a/spec/factories/runner.rb b/spec/factories/runner.rb index 636a8055..28c6ffd9 100644 --- a/spec/factories/runner.rb +++ b/spec/factories/runner.rb @@ -3,9 +3,8 @@ # This factory does not request the runner management as the id is already provided. FactoryBot.define do factory :runner do - sequence(:runner_id) {|n| "test-runner-id-#{n}" } + runner_id { SecureRandom.uuid } association :execution_environment, factory: :ruby association :user, factory: :external_user - waiting_time { 1.0 } end end From b6bc578aea9d31ba078230e05e6230a2263e7744 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Fri, 11 Jun 2021 12:01:16 +0200 Subject: [PATCH 035/156] Move submission scoring from controller concern to submission model Localization and markdown formatting is now done in a controller spec in order to bring this logic away from the data and towards the view. --- .../concerns/scoring_result_formatting.rb | 11 +++ .../concerns/submission_scoring.rb | 93 ------------------- .../remote_evaluation_controller.rb | 3 +- app/controllers/submissions_controller.rb | 3 +- app/models/submission.rb | 90 ++++++++++++++++-- .../scoring_result_formatting_spec.rb | 54 +++++++++++ spec/concerns/submission_scoring_spec.rb | 38 -------- spec/controllers/exercises_controller_spec.rb | 4 +- spec/features/editor_spec.rb | 4 +- spec/models/submission_spec.rb | 31 +++++++ 10 files changed, 188 insertions(+), 143 deletions(-) create mode 100644 app/controllers/concerns/scoring_result_formatting.rb delete mode 100644 app/controllers/concerns/submission_scoring.rb create mode 100644 spec/concerns/scoring_result_formatting_spec.rb delete mode 100644 spec/concerns/submission_scoring_spec.rb diff --git a/app/controllers/concerns/scoring_result_formatting.rb b/app/controllers/concerns/scoring_result_formatting.rb new file mode 100644 index 00000000..27096413 --- /dev/null +++ b/app/controllers/concerns/scoring_result_formatting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module ScoringResultFormatting + def format_scoring_results(outputs) + outputs.map do |output| + output[:message] = t(output[:message], default: render_markdown(output[:message])) + output[:filename] = t(output[:filename], default: output[:filename]) + output + end + end +end diff --git a/app/controllers/concerns/submission_scoring.rb b/app/controllers/concerns/submission_scoring.rb deleted file mode 100644 index ebcfb3d2..00000000 --- a/app/controllers/concerns/submission_scoring.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -module SubmissionScoring - def test_result(output, file) - submission = self - # Mnemosyne.trace 'custom.codeocean.collect_test_results', meta: { submission: submission.id } do - # Mnemosyne.trace 'custom.codeocean.collect_test_results_block', meta: { file: file.id, submission: submission.id } do - assessor = Assessor.new(execution_environment: submission.execution_environment) - assessment = assessor.assess(output) - passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score]).positive?) - testrun_output = passed ? nil : "status: #{output[:status]}\n stdout: #{output[:stdout]}\n stderr: #{output[:stderr]}" - if testrun_output.present? - submission.exercise.execution_environment.error_templates.each do |template| - pattern = Regexp.new(template.signature).freeze - StructuredError.create_from_template(template, testrun_output, submission) if pattern.match(testrun_output) - end - end - testrun = Testrun.create( - submission: submission, - cause: 'assess', # Required to differ run and assess for RfC show - file: file, # Test file that was executed - passed: passed, - output: testrun_output, - container_execution_time: output[:container_execution_time], - waiting_for_container_time: output[:waiting_for_container_time] - ) - - filename = file.name_with_extension - - if file.teacher_defined_linter? - LinterCheckRun.create_from(testrun, assessment) - assessment = assessor.translate_linter(assessment, I18n.locale) - - # replace file name with hint if linter is not used for grading. Refactor! - filename = t('exercises.implement.not_graded', locale: :de) if file.weight.zero? - end - - output.merge!(assessment) - output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight) - end - - # TODO: make this a controller concern again to bring the locales nearer to the view - def feedback_message(file, output) - if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' - I18n.t('exercises.implement.default_test_feedback') - elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' - I18n.t('exercises.implement.default_linter_feedback') - else - # render_markdown(file.feedback_message) - file.feedback_message - end - end - - def score_submission(outputs) - submission = self - score = 0.0 - if outputs.present? - outputs.each do |output| - score += output[:score] * output[:weight] unless output.nil? - - if output.present? && output[:status] == :timeout - output[:stderr] += "\n\n#{t('exercises.editor.timeout', - permitted_execution_time: submission.exercise.execution_environment.permitted_execution_time.to_s)}" - end - end - end - submission.update(score: score) - if submission.normalized_score.to_d == 1.0.to_d - Thread.new do - RequestForComment.where(exercise_id: submission.exercise_id, user_id: submission.user_id, - user_type: submission.user_type).each do |rfc| - rfc.full_score_reached = true - rfc.save - end - ensure - ActiveRecord::Base.connection_pool.release_connection - end - end - if @embed_options.present? && @embed_options[:hide_test_results] && outputs.present? - outputs.each do |output| - output.except!(:error_messages, :count, :failed, :filename, :message, :passed, :stderr, :stdout) - end - end - - # Return all test results except for those of a linter if not allowed - show_linter = Python20CourseWeek.show_linter? submission.exercise - outputs&.reject do |output| - next if show_linter || output.blank? - - output[:file_role] == 'teacher_defined_linter' - end - end -end diff --git a/app/controllers/remote_evaluation_controller.rb b/app/controllers/remote_evaluation_controller.rb index 96db6c8b..2f23eacb 100644 --- a/app/controllers/remote_evaluation_controller.rb +++ b/app/controllers/remote_evaluation_controller.rb @@ -3,6 +3,7 @@ class RemoteEvaluationController < ApplicationController include RemoteEvaluationParameters include Lti + include ScoringResultFormatting skip_after_action :verify_authorized skip_before_action :verify_authenticity_token @@ -62,7 +63,7 @@ status: 202} validation_token = remote_evaluation_params[:validation_token] if (remote_evaluation_mapping = RemoteEvaluationMapping.find_by(validation_token: validation_token)) @submission = Submission.create(build_submission_params(cause, remote_evaluation_mapping)) - @submission.calculate_score + format_scoring_results(@submission.calculate_score) else # TODO: better output # TODO: check token expired? diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 05e86e4d..06bb8876 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -5,6 +5,7 @@ class SubmissionsController < ApplicationController include CommonBehavior include Lti include SubmissionParameters + include ScoringResultFormatting include Tubesock::Hijack before_action :set_submission, @@ -240,7 +241,7 @@ class SubmissionsController < ApplicationController hijack do |tubesock| return kill_socket(tubesock) if @embed_options[:disable_run] - tubesock.send_data(@submission.calculate_score) + tubesock.send_data(JSON.dump(format_scoring_results(@submission.calculate_score))) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) rescue Runner::Error::ExecutionTimeout => e diff --git a/app/models/submission.rb b/app/models/submission.rb index c2fd3121..7810cbb8 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -4,7 +4,6 @@ class Submission < ApplicationRecord include Context include Creation include ActionCableHelper - include SubmissionScoring require 'concurrent/future' @@ -140,9 +139,9 @@ class Submission < ApplicationRecord end def calculate_score - score = nil + file_scores = nil prepared_runner do |runner, waiting_duration| - scores = collect_files.select(&:teacher_defined_assessment?).map do |file| + file_scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension stdout = +'' stderr = +'' @@ -167,11 +166,10 @@ class Submission < ApplicationRecord stdout: stdout, stderr: stderr, } - test_result(output, file) + score_file(output, file) end - score = score_submission(scores) end - JSON.dump(score) + combine_file_scores(file_scores) end def run(file, &block) @@ -214,4 +212,84 @@ class Submission < ApplicationRecord module_name: File.basename(filename, File.extname(filename)).underscore, } end + + def score_file(output, file) + # Mnemosyne.trace 'custom.codeocean.collect_test_results', meta: { submission: id } do + # Mnemosyne.trace 'custom.codeocean.collect_test_results_block', meta: { file: file.id, submission: id } do + assessor = Assessor.new(execution_environment: execution_environment) + assessment = assessor.assess(output) + passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score]).positive?) + testrun_output = passed ? nil : "status: #{output[:status]}\n stdout: #{output[:stdout]}\n stderr: #{output[:stderr]}" + if testrun_output.present? + execution_environment.error_templates.each do |template| + pattern = Regexp.new(template.signature).freeze + StructuredError.create_from_template(template, testrun_output, self) if pattern.match(testrun_output) + end + end + testrun = Testrun.create( + submission: self, + cause: 'assess', # Required to differ run and assess for RfC show + file: file, # Test file that was executed + passed: passed, + output: testrun_output, + container_execution_time: output[:container_execution_time], + waiting_for_container_time: output[:waiting_for_container_time] + ) + + filename = file.name_with_extension + + if file.teacher_defined_linter? + LinterCheckRun.create_from(testrun, assessment) + assessment = assessor.translate_linter(assessment, session[:locale]) + + # replace file name with hint if linter is not used for grading. Refactor! + filename = 'exercises.implement.not_graded' if file.weight.zero? + end + + output.merge!(assessment) + output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight) + end + + def feedback_message(file, output) + if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' + 'exercises.implement.default_test_feedback' + elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' + 'exercises.implement.default_linter_feedback' + else + file.feedback_message + end + end + + def combine_file_scores(outputs) + score = 0.0 + if outputs.present? + outputs.each do |output| + score += output[:score] * output[:weight] unless output.nil? + end + end + update(score: score) + if normalized_score.to_d == 1.0.to_d + Thread.new do + RequestForComment.find_each(exercise_id: exercise_id, user_id: user_id, user_type: user_type) do |rfc| + rfc.full_score_reached = true + rfc.save + end + ensure + ActiveRecord::Base.connection_pool.release_connection + end + end + if @embed_options.present? && @embed_options[:hide_test_results] && outputs.present? + outputs.each do |output| + output.except!(:error_messages, :count, :failed, :filename, :message, :passed, :stderr, :stdout) + end + end + + # Return all test results except for those of a linter if not allowed + show_linter = Python20CourseWeek.show_linter? exercise + outputs&.reject do |output| + next if show_linter || output.blank? + + output[:file_role] == 'teacher_defined_linter' + end + end end diff --git a/spec/concerns/scoring_result_formatting_spec.rb b/spec/concerns/scoring_result_formatting_spec.rb new file mode 100644 index 00000000..d66f9e3d --- /dev/null +++ b/spec/concerns/scoring_result_formatting_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'rails_helper' + +class Controller < AnonymousController + include ScoringResultFormatting +end + +describe ScoringResultFormatting do + let(:controller) { Controller.new } + let(:filename) { 'exercise.py' } + let(:feedback_message) { '**good work**' } + let(:outputs) { [{filename: filename, message: feedback_message}] } + + describe 'feedback message' do + let(:new_feedback_message) { controller.format_scoring_results(outputs).first[:message] } + + context 'when the feedback message is not a path to a locale' do + let(:feedback_message) { '**good work**' } + + it 'renders the feedback message as markdown' do + expect(new_feedback_message).to match('

good work

') + end + end + + context 'when the feedback message is a valid path to a locale' do + let(:feedback_message) { 'exercises.implement.default_test_feedback' } + + it 'replaces the feedback message with the locale' do + expect(new_feedback_message).to eq(I18n.t(feedback_message)) + end + end + end + + describe 'filename' do + let(:new_filename) { controller.format_scoring_results(outputs).first[:filename] } + + context 'when the filename is not a path to a locale' do + let(:filename) { 'exercise.py' } + + it 'does not alter the filename' do + expect(new_filename).to eq(filename) + end + end + + context 'when the filename is a valid path to a locale' do + let(:filename) { 'exercises.implement.not_graded' } + + it 'replaces the filename with the locale' do + expect(new_filename).to eq(I18n.t(filename)) + end + end + end +end diff --git a/spec/concerns/submission_scoring_spec.rb b/spec/concerns/submission_scoring_spec.rb deleted file mode 100644 index 8eb1506f..00000000 --- a/spec/concerns/submission_scoring_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe SubmissionScoring do - let(:submission) { FactoryBot.create(:submission, cause: 'submit') } - - describe '#collect_test_results' do - let(:runner) { FactoryBot.create :runner } - - before do - allow(Runner).to receive(:for).and_return(runner) - allow(runner).to receive(:copy_files) - allow(runner).to receive(:attach_to_execution).and_return(1.0) - end - - after { submission.calculate_score } - - it 'executes every teacher-defined test file' do - allow(submission).to receive(:score_submission) - submission.collect_files.select(&:teacher_defined_assessment?).each do |file| - allow(submission).to receive(:test_result).with(any_args, file).and_return({}) - end - end - - it 'scores the submission' do - allow(submission).to receive(:score_submission).and_return([]) - end - end - - describe '#score_submission', cleaning_strategy: :truncation do - after { submission.score_submission([]) } - - it 'assigns a score to the submissions' do - expect(submission).to receive(:update).with(score: anything) - end - end -end diff --git a/spec/controllers/exercises_controller_spec.rb b/spec/controllers/exercises_controller_spec.rb index fa0c7628..683f0d40 100644 --- a/spec/controllers/exercises_controller_spec.rb +++ b/spec/controllers/exercises_controller_spec.rb @@ -242,7 +242,7 @@ describe ExercisesController do let(:user) { FactoryBot.create(:external_user) } let(:scoring_response) do [{ - status: 'ok', + status: :ok, stdout: '', stderr: '', waiting_for_container_time: 0, @@ -263,7 +263,7 @@ describe ExercisesController do FactoryBot.create(:lti_parameter, external_user: user, exercise: exercise) submission = FactoryBot.build(:submission, exercise: exercise, user: user) allow(submission).to receive(:normalized_score).and_return(1) - allow(submission).to receive(:calculate_score).and_return(JSON.dump(scoring_response)) + allow(submission).to receive(:calculate_score).and_return(scoring_response) allow(Submission).to receive(:create).and_return(submission) end diff --git a/spec/features/editor_spec.rb b/spec/features/editor_spec.rb index 2f2b1126..06919ea7 100644 --- a/spec/features/editor_spec.rb +++ b/spec/features/editor_spec.rb @@ -6,7 +6,7 @@ describe 'Editor', js: true do let(:exercise) { FactoryBot.create(:audio_video, description: Forgery(:lorem_ipsum).sentence) } let(:scoring_response) do [{ - status: 'ok', + status: :ok, stdout: '', stderr: '', waiting_for_container_time: 0, @@ -95,7 +95,7 @@ describe 'Editor', js: true do it 'contains a button for submitting the exercise' do submission = FactoryBot.build(:submission, user: user, exercise: exercise) - allow(submission).to receive(:calculate_score).and_return(JSON.dump(scoring_response)) + allow(submission).to receive(:calculate_score).and_return(scoring_response) allow(Submission).to receive(:find).and_return(submission) click_button(I18n.t('exercises.editor.score')) expect(page).not_to have_css('#submit_outdated') diff --git a/spec/models/submission_spec.rb b/spec/models/submission_spec.rb index b4d4eb7b..60a86923 100644 --- a/spec/models/submission_spec.rb +++ b/spec/models/submission_spec.rb @@ -141,4 +141,35 @@ describe Submission do end end end + + describe '#calculate_score' do + let(:runner) { FactoryBot.create :runner } + + before do + allow(Runner).to receive(:for).and_return(runner) + allow(runner).to receive(:copy_files) + allow(runner).to receive(:attach_to_execution).and_return(1.0) + end + + after { submission.calculate_score } + + it 'executes every teacher-defined test file' do + allow(submission).to receive(:combine_file_scores) + submission.collect_files.select(&:teacher_defined_assessment?).each do |file| + expect(submission).to receive(:score_file).with(any_args, file) + end + end + + it 'scores the submission' do + expect(submission).to receive(:combine_file_scores) + end + end + + describe '#combine_file_scores' do + after { submission.send(:combine_file_scores, []) } + + it 'assigns a score to the submissions' do + expect(submission).to receive(:update).with(score: anything) + end + end end From 413f9b27058714bd40f6eee2df1b35afccc73af2 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 14 Jun 2021 09:22:29 +0200 Subject: [PATCH 036/156] Improve error resilience and handling Timeouts are now handled correctly and the Runner automatically creates the execution environment if it could not be found in Poseidon. The runner is deleted locally if Poseidon returns a bad request error. --- app/controllers/submissions_controller.rb | 4 + app/models/runner.rb | 18 ++++- ..._exposed_ports_in_execution_environment.rb | 2 +- lib/runner/connection.rb | 5 +- lib/runner/strategy/poseidon.rb | 42 +++++++--- spec/lib/runner/strategy/poseidon_spec.rb | 17 ++++ spec/models/runner_spec.rb | 77 ++++++++++++++++--- 7 files changed, 136 insertions(+), 29 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 06bb8876..4abe2632 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -150,6 +150,10 @@ class SubmissionsController < ApplicationController end socket.on :exit do |exit_code| + # As this is sometimes called before the timeout is handled, we must not close the + # socket to the user here. The socket will be closed after handling the timeout. + next if exit_code == Runner::Connection::TIMEOUT_EXIT_STATUS + EventMachine.stop_event_loop if @output.empty? tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) diff --git a/app/models/runner.rb b/app/models/runner.rb index 707563bc..c1aee053 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -39,8 +39,8 @@ class Runner < ApplicationRecord define_method(method) do |*args, &block| @strategy.send(method, *args, &block) rescue Runner::Error::NotFound - update(runner_id: self.class.strategy_class.request_from_management(execution_environment)) - @strategy = self.class.strategy_class.new(runner_id, execution_environment) + request_new_id + save @strategy.send(method, *args, &block) end end @@ -48,10 +48,22 @@ class Runner < ApplicationRecord private def request_id - return if runner_id.present? + request_new_id if runner_id.blank? + end + def request_new_id strategy_class = self.class.strategy_class self.runner_id = strategy_class.request_from_management(execution_environment) @strategy = strategy_class.new(runner_id, execution_environment) + rescue Runner::Error::NotFound + if strategy_class.sync_environment(execution_environment) + raise Runner::Error::NotFound.new( + "The execution environment with id #{execution_environment.id} was not found and was successfully synced with the runner management" + ) + else + raise Runner::Error::NotFound.new( + "The execution environment with id #{execution_environment.id} was not found and could not be synced with the runner management" + ) + end end end diff --git a/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb b/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb index 32db9950..c50cc53f 100644 --- a/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb +++ b/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb @@ -3,7 +3,7 @@ class CleanExposedPortsInExecutionEnvironment < ActiveRecord::Migration[6.1] def change ExecutionEnvironment.all.each do |execution_environment| - continue if execution_environment.exposed_ports.nil? + next if execution_environment.exposed_ports.nil? cleaned = execution_environment.exposed_ports.gsub(/[[:space:]]/, '') list = cleaned.split(',').map(&:to_i).uniq diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 55378e04..8dd8f348 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -7,6 +7,7 @@ class Runner::Connection # These are events for which callbacks can be registered. EVENTS = %i[start output exit stdout stderr].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) + TIMEOUT_EXIT_STATUS = -100 def initialize(url) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) @@ -20,7 +21,8 @@ class Runner::Connection # This registers empty default callbacks. EVENTS.each {|event_type| instance_variable_set(:"@#{event_type}_callback", ->(e) {}) } @start_callback = -> {} - @exit_code = 0 + # Fail if no exit status was returned. + @exit_code = 1 end def on(event, &block) @@ -80,6 +82,7 @@ class Runner::Connection def handle_start(_event); end def handle_timeout(_event) + @exit_code = TIMEOUT_EXIT_STATUS raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') end end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 6a642bb8..15c19da0 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -10,6 +10,10 @@ class Runner::Strategy::Poseidon < Runner::Strategy end end + def self.sync_environment(environment) + environment.copy_to_poseidon + end + def self.request_from_management(environment) url = "#{Runner::BASE_URL}/runners" body = {executionEnvironmentId: environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME} @@ -21,10 +25,12 @@ class Runner::Strategy::Poseidon < Runner::Strategy runner_id = response_body[:runnerId] runner_id.presence || raise(Runner::Error::Unknown.new('Poseidon did not send a runner id')) when 404 - raise Runner::Error::NotFound.new('Execution environment not found') + raise Runner::Error::EnvironmentNotFound.new else handle_error response end + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") end def self.handle_error(response) @@ -35,7 +41,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy when 401 raise Runner::Error::Unauthorized.new('Authentication with Poseidon failed') when 404 - raise Runner::Error::NotFound.new('Runner not found') + raise Runner::Error::RunnerNotFound.new when 500 response_body = parse response error_code = response_body[:errorCode] @@ -60,7 +66,12 @@ class Runner::Strategy::Poseidon < Runner::Strategy url = "#{runner_url}/files" body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} response = Faraday.patch(url, body.to_json, HEADERS) - self.class.handle_error response unless response.status == 204 + return if response.status == 204 + + Runner.destroy(@runner_id) if response.status == 400 + self.class.handle_error response + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") end def attach_to_execution(command) @@ -68,7 +79,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy websocket_url = execute_command(command) EventMachine.run do socket = Runner::Connection.new(websocket_url) - yield(socket) if block_given? + yield(socket) end Time.zone.now - starting_time # execution duration end @@ -76,6 +87,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy def destroy_at_management response = Faraday.delete runner_url self.class.handle_error response unless response.status == 204 + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") end private @@ -84,17 +97,22 @@ class Runner::Strategy::Poseidon < Runner::Strategy url = "#{runner_url}/execute" body = {command: command, timeLimit: @execution_environment.permitted_execution_time} response = Faraday.post(url, body.to_json, HEADERS) - if response.status == 200 - response_body = self.class.parse response - websocket_url = response_body[:websocketUrl] - if websocket_url.present? - return websocket_url - else - raise Runner::Error::Unknown.new('Poseidon did not send websocket url') - end + case response.status + when 200 + response_body = self.class.parse response + websocket_url = response_body[:websocketUrl] + if websocket_url.present? + return websocket_url + else + raise Runner::Error::Unknown.new('Poseidon did not send websocket url') + end + when 400 + Runner.destroy(@runner_id) end self.class.handle_error response + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") end def runner_url diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index e8370351..7f5ac368 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -16,11 +16,26 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 400 } it 'raises an error' do + allow(Runner).to receive(:destroy).with(runner_id) expect { action.call }.to raise_error(Runner::Error::BadRequest, /#{error_message}/) end end end + # Only #copy_files and #execute_command destroy the runner locally in case + # of a BadRequest (400) response. + shared_examples 'BadRequest (400) destroys local runner' do + context 'when Poseidon returns BadRequest (400)' do + let(:response_body) { {message: error_message}.to_json } + let(:response_status) { 400 } + + it 'destroys the runner locally' do + expect(Runner).to receive(:destroy).with(runner_id) + expect { action.call }.to raise_error(Runner::Error::BadRequest) + end + end + end + # All requests handle a Unauthorized (401) response the same way. shared_examples 'Unauthorized (401) error handling' do context 'when Poseidon returns Unauthorized (401)' do @@ -193,6 +208,7 @@ describe Runner::Strategy::Poseidon do end include_examples 'BadRequest (400) error handling' + include_examples 'BadRequest (400) destroys local runner' include_examples 'Unauthorized (401) error handling' include_examples 'NotFound (404) error handling' include_examples 'InternalServerError (500) error handling' @@ -247,6 +263,7 @@ describe Runner::Strategy::Poseidon do end include_examples 'BadRequest (400) error handling' + include_examples 'BadRequest (400) destroys local runner' include_examples 'Unauthorized (401) error handling' include_examples 'NotFound (404) error handling' include_examples 'InternalServerError (500) error handling' diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 4c3043c0..b2f0e6cb 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -61,28 +61,81 @@ describe Runner do include_examples 'delegates method sends to its strategy', :attach_to_execution, nil end - describe '#request_id' do - it 'requests a runner from the runner management when created' do + describe 'creation' do + let(:user) { FactoryBot.create :external_user } + let(:execution_environment) { FactoryBot.create :ruby } + let(:create_action) { -> { described_class.create(user: user, execution_environment: execution_environment) } } + + it 'requests a runner id from the runner management' do expect(strategy_class).to receive(:request_from_management) - described_class.create + create_action.call end - it 'sets the runner id when created' do + it 'returns a valid runner' do allow(strategy_class).to receive(:request_from_management).and_return(runner_id) - runner = described_class.create - expect(runner.runner_id).to eq(runner_id) + expect(create_action.call).to be_valid end - it 'sets the strategy when created' do + it 'sets the strategy' do allow(strategy_class).to receive(:request_from_management).and_return(runner_id) - runner = described_class.create - expect(runner.strategy).to be_present + strategy = strategy_class.new(runner_id, execution_environment) + allow(strategy_class).to receive(:new).with(runner_id, execution_environment).and_return(strategy) + runner = create_action.call + expect(runner.strategy).to eq(strategy) end - it 'does not call the runner management again when validating the model' do + it 'does not call the runner management again while a runner id is set' do expect(strategy_class).to receive(:request_from_management).and_return(runner_id).once - runner = described_class.create - runner.valid? + runner = create_action.call + runner.update(user: FactoryBot.create(:external_user)) + end + end + + describe '#request_new_id' do + let(:runner) { FactoryBot.create :runner } + + context 'when the environment is available in the runner management' do + it 'requests the runner management' do + expect(strategy_class).to receive(:request_from_management) + runner.send(:request_new_id) + end + + it 'updates the runner id' do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + runner.send(:request_new_id) + expect(runner.runner_id).to eq(runner_id) + end + + it 'updates the strategy' do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + strategy = strategy_class.new(runner_id, runner.execution_environment) + allow(strategy_class).to receive(:new).with(runner_id, runner.execution_environment).and_return(strategy) + runner.send(:request_new_id) + expect(runner.strategy).to eq(strategy) + end + end + + context 'when the environment could not be found in the runner management' do + let(:environment_id) { runner.execution_environment.id } + + before { allow(strategy_class).to receive(:request_from_management).and_raise(Runner::Error::NotFound) } + + it 'syncs the execution environment' do + expect(strategy_class).to receive(:sync_environment).with(runner.execution_environment) + runner.send(:request_new_id) + rescue Runner::Error::NotFound + # Ignored because this error is expected (see tests below). + end + + it 'raises an error when the environment could be synced' do + allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(true) + expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*successfully synced/) + end + + it 'raises an error when the environment could not be synced' do + allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(false) + expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*could not be synced/) + end end end From b48b45de9f3503fc34a930463a9b6298e83aab28 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 14 Jun 2021 09:56:27 +0200 Subject: [PATCH 037/156] Refactor error classes All runner errors are now in a single file. The not found error has been splitted into an error for runner not found and for environment not found. --- app/errors/runner/error.rb | 20 ++++++++++++++++++- app/errors/runner/error/bad_request.rb | 3 --- app/errors/runner/error/execution_timeout.rb | 3 --- .../runner/error/internal_server_error.rb | 3 --- app/errors/runner/error/not_available.rb | 3 --- app/errors/runner/error/not_found.rb | 3 --- app/errors/runner/error/unauthorized.rb | 3 --- app/errors/runner/error/unknown.rb | 3 --- app/models/runner.rb | 8 ++++---- spec/lib/runner/strategy/poseidon_spec.rb | 4 ++-- spec/models/runner_spec.rb | 8 ++++---- 11 files changed, 29 insertions(+), 32 deletions(-) delete mode 100644 app/errors/runner/error/bad_request.rb delete mode 100644 app/errors/runner/error/execution_timeout.rb delete mode 100644 app/errors/runner/error/internal_server_error.rb delete mode 100644 app/errors/runner/error/not_available.rb delete mode 100644 app/errors/runner/error/not_found.rb delete mode 100644 app/errors/runner/error/unauthorized.rb delete mode 100644 app/errors/runner/error/unknown.rb diff --git a/app/errors/runner/error.rb b/app/errors/runner/error.rb index 3470f553..ba6aeee1 100644 --- a/app/errors/runner/error.rb +++ b/app/errors/runner/error.rb @@ -1,3 +1,21 @@ # frozen_string_literal: true -class Runner::Error < ApplicationError; end +class Runner + class Error < ApplicationError + class BadRequest < Error; end + + class EnvironmentNotFound < Error; end + + class ExecutionTimeout < Error; end + + class InternalServerError < Error; end + + class NotAvailable < Error; end + + class Unauthorized < Error; end + + class RunnerNotFound < Error; end + + class Unknown < Error; end + end +end diff --git a/app/errors/runner/error/bad_request.rb b/app/errors/runner/error/bad_request.rb deleted file mode 100644 index 98b8f5ca..00000000 --- a/app/errors/runner/error/bad_request.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::BadRequest < Runner::Error; end diff --git a/app/errors/runner/error/execution_timeout.rb b/app/errors/runner/error/execution_timeout.rb deleted file mode 100644 index c10806a5..00000000 --- a/app/errors/runner/error/execution_timeout.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::ExecutionTimeout < Runner::Error; end diff --git a/app/errors/runner/error/internal_server_error.rb b/app/errors/runner/error/internal_server_error.rb deleted file mode 100644 index 68f3f0b8..00000000 --- a/app/errors/runner/error/internal_server_error.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::InternalServerError < Runner::Error; end diff --git a/app/errors/runner/error/not_available.rb b/app/errors/runner/error/not_available.rb deleted file mode 100644 index 4785ea59..00000000 --- a/app/errors/runner/error/not_available.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::NotAvailable < Runner::Error; end diff --git a/app/errors/runner/error/not_found.rb b/app/errors/runner/error/not_found.rb deleted file mode 100644 index 329bcc71..00000000 --- a/app/errors/runner/error/not_found.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::NotFound < Runner::Error; end diff --git a/app/errors/runner/error/unauthorized.rb b/app/errors/runner/error/unauthorized.rb deleted file mode 100644 index 8bccf7fa..00000000 --- a/app/errors/runner/error/unauthorized.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::Unauthorized < Runner::Error; end diff --git a/app/errors/runner/error/unknown.rb b/app/errors/runner/error/unknown.rb deleted file mode 100644 index 6e1a1fce..00000000 --- a/app/errors/runner/error/unknown.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Error::Unknown < Runner::Error; end diff --git a/app/models/runner.rb b/app/models/runner.rb index c1aee053..0293f181 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -38,7 +38,7 @@ class Runner < ApplicationRecord DELEGATED_STRATEGY_METHODS.each do |method| define_method(method) do |*args, &block| @strategy.send(method, *args, &block) - rescue Runner::Error::NotFound + rescue Runner::Error::RunnerNotFound request_new_id save @strategy.send(method, *args, &block) @@ -55,13 +55,13 @@ class Runner < ApplicationRecord strategy_class = self.class.strategy_class self.runner_id = strategy_class.request_from_management(execution_environment) @strategy = strategy_class.new(runner_id, execution_environment) - rescue Runner::Error::NotFound + rescue Runner::Error::EnvironmentNotFound if strategy_class.sync_environment(execution_environment) - raise Runner::Error::NotFound.new( + raise Runner::Error::EnvironmentNotFound.new( "The execution environment with id #{execution_environment.id} was not found and was successfully synced with the runner management" ) else - raise Runner::Error::NotFound.new( + raise Runner::Error::EnvironmentNotFound.new( "The execution environment with id #{execution_environment.id} was not found and could not be synced with the runner management" ) end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 7f5ac368..15795701 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -53,7 +53,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 404 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::NotFound, /Runner/) + expect { action.call }.to raise_error(Runner::Error::RunnerNotFound) end end end @@ -152,7 +152,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 404 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::NotFound, /Execution environment/) + expect { action.call }.to raise_error(Runner::Error::EnvironmentNotFound) end end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index b2f0e6cb..e8f57966 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -118,23 +118,23 @@ describe Runner do context 'when the environment could not be found in the runner management' do let(:environment_id) { runner.execution_environment.id } - before { allow(strategy_class).to receive(:request_from_management).and_raise(Runner::Error::NotFound) } + before { allow(strategy_class).to receive(:request_from_management).and_raise(Runner::Error::EnvironmentNotFound) } it 'syncs the execution environment' do expect(strategy_class).to receive(:sync_environment).with(runner.execution_environment) runner.send(:request_new_id) - rescue Runner::Error::NotFound + rescue Runner::Error::EnvironmentNotFound # Ignored because this error is expected (see tests below). end it 'raises an error when the environment could be synced' do allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(true) - expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*successfully synced/) + expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::EnvironmentNotFound, /#{environment_id}.*successfully synced/) end it 'raises an error when the environment could not be synced' do allow(strategy_class).to receive(:sync_environment).with(runner.execution_environment).and_return(false) - expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::NotFound, /#{environment_id}.*could not be synced/) + expect { runner.send(:request_new_id) }.to raise_error(Runner::Error::EnvironmentNotFound, /#{environment_id}.*could not be synced/) end end end From 1d3f0d7ad86c2f06df1a8cc255899fc09c9bda3c Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 14 Jun 2021 10:53:46 +0200 Subject: [PATCH 038/156] Handle Faraday errors --- app/models/execution_environment.rb | 3 +++ app/models/submission.rb | 2 +- spec/lib/runner/strategy/poseidon_spec.rb | 18 ++++++++++++++++++ spec/models/execution_environment_spec.rb | 5 +++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 3db35c35..9d420715 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -51,6 +51,9 @@ class ExecutionEnvironment < ApplicationRecord Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") false + rescue Faraday::Error => e + Rails.logger.warn("Could not create execution environment because of Faraday error: #{e.inspect}") + false end def to_json(*_args) diff --git a/app/models/submission.rb b/app/models/submission.rb index 7810cbb8..4bd2653b 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -197,7 +197,7 @@ class Submission < ApplicationRecord runner = Runner.for(user, exercise) copy_files_to runner waiting_duration = Time.zone.now - request_time - yield(runner, waiting_duration) if block_given? + yield(runner, waiting_duration) end def command_for(template, file) diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 15795701..d3acf5f9 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -100,6 +100,20 @@ describe Runner::Strategy::Poseidon do end end + # All requests handle a Faraday error the same way. + shared_examples 'Faraday error handling' do + context 'when Faraday throws an error' do + # The response status is not needed in this context but the describes block this context is embedded + # into expect this variable to be set in order to properly stub requests to the runner management. + let(:response_status) { -1 } + + it 'raises an error' do + %i[post patch delete].each {|message| allow(Faraday).to receive(message).and_raise(Faraday::TimeoutError) } + expect { action.call }.to raise_error(Runner::Error::Unknown, /Faraday/) + end + end + end + describe '::request_from_management' do let(:action) { -> { described_class.request_from_management(execution_environment) } } let!(:request_runner_stub) do @@ -158,6 +172,7 @@ describe Runner::Strategy::Poseidon do include_examples 'InternalServerError (500) error handling' include_examples 'unknown response status error handling' + include_examples 'Faraday error handling' end describe '#execute_command' do @@ -213,6 +228,7 @@ describe Runner::Strategy::Poseidon do include_examples 'NotFound (404) error handling' include_examples 'InternalServerError (500) error handling' include_examples 'unknown response status error handling' + include_examples 'Faraday error handling' end describe '#destroy_at_management' do @@ -236,6 +252,7 @@ describe Runner::Strategy::Poseidon do include_examples 'NotFound (404) error handling' include_examples 'InternalServerError (500) error handling' include_examples 'unknown response status error handling' + include_examples 'Faraday error handling' end describe '#copy_files' do @@ -268,6 +285,7 @@ describe Runner::Strategy::Poseidon do include_examples 'NotFound (404) error handling' include_examples 'InternalServerError (500) error handling' include_examples 'unknown response status error handling' + include_examples 'Faraday error handling' end describe '#attach_to_execution' do diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index 54e01111..6441b896 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -227,5 +227,10 @@ describe ExecutionEnvironment do [400, 500].each do |status| include_examples 'returns false when the api request failed', status end + + it 'returns false if Faraday raises an error' do + allow(Faraday).to receive(:put).and_raise(Faraday::TimeoutError) + expect(execution_environment.copy_to_poseidon).to be_falsey + end end end From 704407b9fc3b4d8442477dcca6080c76ae8d2144 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 10 Jun 2021 16:17:02 +0200 Subject: [PATCH 039/156] Add strategy for DockerContainerPool In order to provide an alternative to Poseidon, a strategy for the DockerContainerPool is added that is used by the runner model. Co-authored-by: Sebastian Serth --- app/controllers/submissions_controller.rb | 6 +- app/models/runner.rb | 49 ++++--- app/models/submission.rb | 12 +- lib/docker_container_pool.rb | 6 + lib/runner/connection.rb | 37 +++-- lib/runner/strategy.rb | 3 +- lib/runner/strategy/docker.rb | 3 - lib/runner/strategy/docker_container_pool.rb | 138 ++++++++++++++++++ lib/runner/strategy/poseidon.rb | 33 ++++- ..._spec.rb => docker_container_pool_spec.rb} | 6 +- spec/lib/runner/strategy/poseidon_spec.rb | 8 +- spec/models/runner_spec.rb | 50 ++++++- 12 files changed, 282 insertions(+), 69 deletions(-) delete mode 100644 lib/runner/strategy/docker.rb create mode 100644 lib/runner/strategy/docker_container_pool.rb rename spec/lib/runner/strategy/{docker_spec.rb => docker_container_pool_spec.rb} (62%) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 4abe2632..b01cf970 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -137,7 +137,7 @@ class SubmissionsController < ApplicationController @output = +'' socket.on :output do |data| - Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data}") + Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data.inspect}") @output << data if @output.size + data.size <= max_output_buffer_size end @@ -150,10 +150,6 @@ class SubmissionsController < ApplicationController end socket.on :exit do |exit_code| - # As this is sometimes called before the timeout is handled, we must not close the - # socket to the user here. The socket will be closed after handling the timeout. - next if exit_code == Runner::Connection::TIMEOUT_EXIT_STATUS - EventMachine.stop_event_loop if @output.empty? tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) diff --git a/app/models/runner.rb b/app/models/runner.rb index 0293f181..5fd0628b 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'forwardable' - class Runner < ApplicationRecord belongs_to :execution_environment belongs_to :user, polymorphic: true @@ -13,7 +11,6 @@ class Runner < ApplicationRecord STRATEGY_NAME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] - DELEGATED_STRATEGY_METHODS = %i[destroy_at_management attach_to_execution copy_files].freeze attr_accessor :strategy @@ -35,14 +32,20 @@ class Runner < ApplicationRecord runner end - DELEGATED_STRATEGY_METHODS.each do |method| - define_method(method) do |*args, &block| - @strategy.send(method, *args, &block) - rescue Runner::Error::RunnerNotFound - request_new_id - save - @strategy.send(method, *args, &block) - end + def copy_files(files) + @strategy.copy_files(files) + rescue Runner::Error::RunnerNotFound + request_new_id + save + @strategy.copy_files(files) + end + + def attach_to_execution(command, &block) + @strategy.attach_to_execution(command, &block) + end + + def destroy_at_management + @strategy.destroy_at_management end private @@ -53,17 +56,19 @@ class Runner < ApplicationRecord def request_new_id strategy_class = self.class.strategy_class - self.runner_id = strategy_class.request_from_management(execution_environment) - @strategy = strategy_class.new(runner_id, execution_environment) - rescue Runner::Error::EnvironmentNotFound - if strategy_class.sync_environment(execution_environment) - raise Runner::Error::EnvironmentNotFound.new( - "The execution environment with id #{execution_environment.id} was not found and was successfully synced with the runner management" - ) - else - raise Runner::Error::EnvironmentNotFound.new( - "The execution environment with id #{execution_environment.id} was not found and could not be synced with the runner management" - ) + begin + self.runner_id = strategy_class.request_from_management(execution_environment) + @strategy = strategy_class.new(runner_id, execution_environment) + rescue Runner::Error::EnvironmentNotFound + if strategy_class.sync_environment(execution_environment) + raise Runner::Error::EnvironmentNotFound.new( + "The execution environment with id #{execution_environment.id} was not found and was successfully synced with the runner management" + ) + else + raise Runner::Error::EnvironmentNotFound.new( + "The execution environment with id #{execution_environment.id} was not found and could not be synced with the runner management" + ) + end end end end diff --git a/app/models/submission.rb b/app/models/submission.rb index 4bd2653b..11857df6 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -184,18 +184,10 @@ class Submission < ApplicationRecord private - def copy_files_to(runner) - files = {} - collect_files.each do |file| - files[file.name_with_extension] = file.content - end - runner.copy_files(files) - end - def prepared_runner request_time = Time.zone.now runner = Runner.for(user, exercise) - copy_files_to runner + runner.copy_files(collect_files) waiting_duration = Time.zone.now - request_time yield(runner, waiting_duration) end @@ -270,7 +262,7 @@ class Submission < ApplicationRecord update(score: score) if normalized_score.to_d == 1.0.to_d Thread.new do - RequestForComment.find_each(exercise_id: exercise_id, user_id: user_id, user_type: user_type) do |rfc| + RequestForComment.where(exercise_id: exercise_id, user_id: user_id, user_type: user_type).find_each do |rfc| rfc.full_score_reached = true rfc.save end diff --git a/lib/docker_container_pool.rb b/lib/docker_container_pool.rb index 85ea7381..a290c82c 100644 --- a/lib/docker_container_pool.rb +++ b/lib/docker_container_pool.rb @@ -3,6 +3,11 @@ require 'concurrent/future' require 'concurrent/timer_task' +# get_container, destroy_container was moved to lib/runner/strategy/docker_container_pool.rb. +# return_container is not used anymore because runners are not shared between users anymore. +# create_container is done by the DockerContainerPool. +# dump_info and quantities are still in use. + class DockerContainerPool def self.config # TODO: Why erb? @@ -22,6 +27,7 @@ class DockerContainerPool nil end + # not in use because DockerClient::RECYCLE_CONTAINERS == false def self.return_container(container, execution_environment) Faraday.get("#{config[:location]}/docker_container_pool/return_container/#{container.id}") rescue StandardError => e diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 8dd8f348..4bf725af 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -7,10 +7,13 @@ class Runner::Connection # These are events for which callbacks can be registered. EVENTS = %i[start output exit stdout stderr].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) - TIMEOUT_EXIT_STATUS = -100 - def initialize(url) + attr_writer :status + + def initialize(url, strategy) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) + @strategy = strategy + @status = :established # For every event type of faye websockets, the corresponding # RunnerConnection method starting with `on_` is called. @@ -37,18 +40,19 @@ class Runner::Connection private - def decode(event) - JSON.parse(event).deep_symbolize_keys + def decode(_raw_event) + raise NotImplementedError end - def encode(data) - data + def encode(_data) + raise NotImplementedError end - def on_message(event) - return unless BACKEND_OUTPUT_SCHEMA.valid?(JSON.parse(event.data)) + def on_message(raw_event) + event = decode(raw_event) + return unless BACKEND_OUTPUT_SCHEMA.valid?(event) - event = decode(event.data) + event = event.deep_symbolize_keys # There is one `handle_` method for every message type defined in the WebSocket schema. __send__("handle_#{event[:type]}", event) end @@ -60,10 +64,20 @@ class Runner::Connection def on_error(_event); end def on_close(_event) - @exit_callback.call @exit_code + case @status + when :timeout + raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') + when :terminated + @exit_callback.call @exit_code + else # :established + # If the runner is killed by the DockerContainerPool after the maximum allowed time per user and + # while the owning user is running an execution, the command execution stops and log output is incomplete. + raise Runner::Error::Unknown.new('Execution terminated with an unknown reason') + end end def handle_exit(event) + @status = :terminated @exit_code = event[:data] end @@ -82,7 +96,6 @@ class Runner::Connection def handle_start(_event); end def handle_timeout(_event) - @exit_code = TIMEOUT_EXIT_STATUS - raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') + @status = :timeout end end diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index dce76adf..0007caab 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true class Runner::Strategy - def initialize(runner_id, environment) - @runner_id = runner_id + def initialize(_runner_id, environment) @execution_environment = environment end diff --git a/lib/runner/strategy/docker.rb b/lib/runner/strategy/docker.rb deleted file mode 100644 index fd7cfafc..00000000 --- a/lib/runner/strategy/docker.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -class Runner::Strategy::Docker < Runner::Strategy; end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb new file mode 100644 index 00000000..7003f901 --- /dev/null +++ b/lib/runner/strategy/docker_container_pool.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +class Runner::Strategy::DockerContainerPool < Runner::Strategy + attr_reader :container_id, :command, :execution_environment + + def self.config + # Since the docker configuration file contains code that must be executed, we use ERB templating. + @config ||= CodeOcean::Config.new(:docker).read(erb: true) + end + + def self.request_from_management(environment) + container_id = JSON.parse(Faraday.get("#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}").body)['id'] + container_id.presence || raise(Runner::Error::NotAvailable.new("DockerContainerPool didn't return a container id")) + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to DockerContainerPool failed: #{e.inspect}") + rescue JSON::ParserError => e + raise Runner::Error::Unknown.new("DockerContainerPool returned invalid JSON: #{e.inspect}") + end + + def initialize(runner_id, _environment) + super + @container_id = runner_id + end + + def copy_files(files) + FileUtils.mkdir_p(local_workspace_path) + clean_workspace + files.each do |file| + if file.path.present? + local_directory_path = local_path(file.path) + FileUtils.mkdir_p(local_directory_path) + end + + local_file_path = local_path(file.filepath) + if file.file_type.binary? + FileUtils.cp(file.native_file.path, local_file_path) + else + begin + File.open(local_file_path, 'w') {|f| f.write(file.content) } + rescue IOError => e + # TODO: try catch i/o exception and log failed attempts + # Does this fix the issue @Sebastian? What exceptions did you have in mind? + raise Runner::Error::Unknown.new("Could not create workspace file #{file.filepath}: #{e.inspect}") + end + end + end + FileUtils.chmod_R('+rwX', local_workspace_path) + end + + def destroy_at_management + Faraday.get("#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}") + rescue Faraday::Error => e + raise Runner::Error::Unknown.new("Faraday request to DockerContainerPool failed: #{e.inspect}") + end + + def attach_to_execution(command) + @command = command + starting_time = Time.zone.now + query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' + websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{@container_id}/attach/ws?#{query_params}" + + EventMachine.run do + socket = Connection.new(websocket_url, self) + EventMachine.add_timer(@execution_environment.permitted_execution_time) do + socket.status = :timeout + destroy_at_management + end + socket.send(command) + yield(socket) + end + Time.zone.now - starting_time # execution duration in seconds + end + + private + + def container + return @container if @container.present? + + @container = Docker::Container.get(@container_id) + raise Runner::Error::RunnerNotFound unless @container.info['State']['Running'] + + @container + rescue Docker::Error::NotFoundError + raise Runner::Error::RunnerNotFound + end + + def local_path(path) + unclean_path = local_workspace_path.join(path) + clean_path = File.expand_path(unclean_path) + unless clean_path.to_s.start_with? local_workspace_path.to_s + raise Runner::Error::Unknown.new("Local filepath #{clean_path.inspect} not allowed") + end + + Pathname.new(clean_path) + end + + def clean_workspace + FileUtils.rm_r(local_workspace_path.children, secure: true) + rescue Errno::ENOENT => e + raise Runner::Error::Unknown.new("The workspace directory does not exist and cannot be deleted: #{e.inspect}") + rescue Errno::EACCES => e + # TODO: Why was this rescued before @Sebastian? + raise Runner::Error::Unknown.new("Not allowed to clean workspace #{local_workspace_path}: #{e.inspect}") + end + + def local_workspace_path + @local_workspace_path ||= Pathname.new(container.binds.first.split(':').first) + end + + class Connection < Runner::Connection + def initialize(*args) + @stream = 'stdout' + super + end + + def encode(data) + "#{data}\n" + end + + def decode(raw_event) + case raw_event.data + when /@#{@strategy.container_id[0..11]}/ + # Assume correct termination for now and return exit code 0 + # TODO: Can we use the actual exit code here? + @exit_code = 0 + @status = :terminated + @socket.close + when /#{format(@strategy.execution_environment.test_command, class_name: '.*', filename: '.*', module_name: '.*')}/ + # TODO: Super dirty hack to redirect test output to stderr (remove attr_reader afterwards) + @stream = 'stderr' + when /#{@strategy.command}/ + when /bash: cmd:canvasevent: command not found/ + else + {'type' => @stream, 'data' => raw_event.data} + end + end + end +end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 15c19da0..1938c229 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -62,13 +62,24 @@ class Runner::Strategy::Poseidon < Runner::Strategy raise Runner::Error::Unknown.new("Error parsing response from Poseidon: #{e.message}") end + def initialize(runner_id, _environment) + super + @allocation_id = runner_id + end + def copy_files(files) + copy = files.map do |file| + { + path: file.filepath, + content: Base64.strict_encode64(file.content), + } + end url = "#{runner_url}/files" - body = {copy: files.map {|filename, content| {path: filename, content: Base64.strict_encode64(content)} }} + body = {copy: copy} response = Faraday.patch(url, body.to_json, HEADERS) return if response.status == 204 - Runner.destroy(@runner_id) if response.status == 400 + Runner.destroy(@allocation_id) if response.status == 400 self.class.handle_error response rescue Faraday::Error => e raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") @@ -78,7 +89,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy starting_time = Time.zone.now websocket_url = execute_command(command) EventMachine.run do - socket = Runner::Connection.new(websocket_url) + socket = Connection.new(websocket_url, self) yield(socket) end Time.zone.now - starting_time # execution duration @@ -107,7 +118,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy raise Runner::Error::Unknown.new('Poseidon did not send websocket url') end when 400 - Runner.destroy(@runner_id) + Runner.destroy(@allocation_id) end self.class.handle_error response @@ -116,6 +127,18 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def runner_url - "#{Runner::BASE_URL}/runners/#{@runner_id}" + "#{Runner::BASE_URL}/runners/#{@allocation_id}" + end + + class Connection < Runner::Connection + def decode(raw_event) + JSON.parse(raw_event.data) + rescue JSON::ParserError => e + raise Runner::Error::Unknown.new("The websocket message from Poseidon could not be decoded to JSON: #{e.inspect}") + end + + def encode(data) + data + end end end diff --git a/spec/lib/runner/strategy/docker_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb similarity index 62% rename from spec/lib/runner/strategy/docker_spec.rb rename to spec/lib/runner/strategy/docker_container_pool_spec.rb index 4110ebf9..67c9b20f 100644 --- a/spec/lib/runner/strategy/docker_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -2,14 +2,14 @@ require 'rails_helper' -describe Runner::Strategy::Docker do +describe Runner::Strategy::DockerContainerPool do let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } let(:execution_environment) { FactoryBot.create :ruby } - let(:docker) { described_class.new(runner_id, execution_environment) } + let(:container_pool) { described_class.new(runner_id, execution_environment) } # TODO: add tests for these methods when implemented it 'defines all methods all runner management strategies must define' do - expect(docker.public_methods).to include(*Runner::DELEGATED_STRATEGY_METHODS) + expect(container_pool.public_methods).to include(:destroy_at_management, :copy_files, :attach_to_execution) expect(described_class.public_methods).to include(:request_from_management) end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index d3acf5f9..62bbd638 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -256,15 +256,15 @@ describe Runner::Strategy::Poseidon do end describe '#copy_files' do - let(:filename) { 'main.py' } let(:file_content) { 'print("Hello World!")' } - let(:action) { -> { poseidon.copy_files({filename => file_content}) } } - let(:encoded_file_content) { Base64.strict_encode64(file_content) } + let(:file) { FactoryBot.build(:file, content: file_content) } + let(:action) { -> { poseidon.copy_files([file]) } } + let(:encoded_file_content) { Base64.strict_encode64(file.content) } let!(:copy_files_stub) do WebMock .stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files") .with( - body: {copy: [{path: filename, content: encoded_file_content}]}, + body: {copy: [{path: file.filepath, content: encoded_file_content}]}, headers: {'Content-Type' => 'application/json'} ) .to_return(body: response_body, status: response_status) diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index e8f57966..b9395645 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -35,10 +35,16 @@ describe Runner do end end - {poseidon: Runner::Strategy::Poseidon, docker: Runner::Strategy::Docker}.each do |strategy, strategy_class| + available_strategies = { + poseidon: Runner::Strategy::Poseidon, + docker_container_pool: Runner::Strategy::DockerContainerPool, + } + available_strategies.each do |strategy, strategy_class| include_examples 'uses the strategy defined in the constant', strategy, strategy_class end + end + describe 'method delegation' do shared_examples 'delegates method sends to its strategy' do |method, *args| context "when sending #{method}" do let(:strategy) { instance_double(strategy_class) } @@ -49,7 +55,7 @@ describe Runner do allow(strategy_class).to receive(:new).and_return(strategy) end - it "delegates the method #{method}" do + it 'delegates to its strategy' do expect(strategy).to receive(method) runner.send(method, *args) end @@ -57,10 +63,48 @@ describe Runner do end include_examples 'delegates method sends to its strategy', :destroy_at_management - include_examples 'delegates method sends to its strategy', :copy_files, nil include_examples 'delegates method sends to its strategy', :attach_to_execution, nil end + describe '#copy_files' do + let(:strategy) { instance_double(strategy_class) } + let(:runner) { described_class.create } + + before do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + allow(strategy_class).to receive(:new).and_return(strategy) + end + + context 'when no error is raised' do + it 'delegates to its strategy' do + expect(strategy).to receive(:copy_files).once + runner.copy_files(nil) + end + end + + context 'when a RunnerNotFound exception is raised' do + before do + was_called = false + allow(strategy).to receive(:copy_files) do + unless was_called + was_called = true + raise Runner::Error::RunnerNotFound.new + end + end + end + + it 'requests a new id' do + expect(runner).to receive(:request_new_id) + runner.copy_files(nil) + end + + it 'retries to copy the files' do + expect(strategy).to receive(:copy_files).twice + runner.copy_files(nil) + end + end + end + describe 'creation' do let(:user) { FactoryBot.create :external_user } let(:execution_environment) { FactoryBot.create :ruby } From db2d1e316485ee14bd360115ff6a83325628b84a Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Wed, 23 Jun 2021 17:01:35 +0200 Subject: [PATCH 040/156] Add tests for DockerContainerPool strategy --- .../strategy/docker_container_pool_spec.rb | 271 +++++++++++++++++- spec/lib/runner/strategy/poseidon_spec.rb | 4 +- spec/models/runner_spec.rb | 3 +- 3 files changed, 271 insertions(+), 7 deletions(-) diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index 67c9b20f..d5c5a455 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -1,15 +1,278 @@ # frozen_string_literal: true require 'rails_helper' +require 'pathname' describe Runner::Strategy::DockerContainerPool do let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } let(:execution_environment) { FactoryBot.create :ruby } let(:container_pool) { described_class.new(runner_id, execution_environment) } + let(:docker_container_pool_url) { 'http://localhost:1234' } + let(:config) { {pool: {location: docker_container_pool_url}} } + let(:container) { instance_double(Docker::Container) } - # TODO: add tests for these methods when implemented - it 'defines all methods all runner management strategies must define' do - expect(container_pool.public_methods).to include(:destroy_at_management, :copy_files, :attach_to_execution) - expect(described_class.public_methods).to include(:request_from_management) + before do + allow(described_class).to receive(:config).and_return(config) + allow(container).to receive(:id).and_return(runner_id) + end + + # All requests handle a Faraday error the same way. + shared_examples 'Faraday error handling' do + it 'raises a runner error' do + allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError) + expect { action.call }.to raise_error(Runner::Error::Unknown, /Faraday/) + end + end + + describe '::request_from_management' do + let(:action) { -> { described_class.request_from_management(execution_environment) } } + let(:response_body) { nil } + let!(:request_runner_stub) do + WebMock + .stub_request(:get, "#{docker_container_pool_url}/docker_container_pool/get_container/#{execution_environment.id}") + .to_return(body: response_body, status: 200) + end + + context 'when the DockerContainerPool returns an id' do + let(:response_body) { {id: runner_id}.to_json } + + it 'successfully requests the DockerContainerPool' do + action.call + expect(request_runner_stub).to have_been_requested.once + end + + it 'returns the received runner id' do + id = action.call + expect(id).to eq(runner_id) + end + end + + context 'when the DockerContainerPool does not return an id' do + let(:response_body) { {}.to_json } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::NotAvailable) + end + end + + context 'when the DockerContainerPool returns invalid JSON' do + let(:response_body) { '{hello}' } + + it 'raises an error' do + expect { action.call }.to raise_error(Runner::Error::Unknown) + end + end + + include_examples 'Faraday error handling' + end + + describe '#destroy_at_management' do + let(:action) { -> { container_pool.destroy_at_management } } + let!(:destroy_runner_stub) do + WebMock + .stub_request(:get, "#{docker_container_pool_url}/docker_container_pool/destroy_container/#{runner_id}") + .to_return(body: nil, status: 200) + end + + before { allow(container_pool).to receive(:container).and_return(container) } + + it 'successfully requests the DockerContainerPool' do + action.call + expect(destroy_runner_stub).to have_been_requested.once + end + + include_examples 'Faraday error handling' + end + + describe '#copy_files' do + let(:files) { [] } + let(:action) { -> { container_pool.copy_files(files) } } + let(:local_path) { Pathname.new('/tmp/container20') } + + before do + allow(container_pool).to receive(:local_workspace_path).and_return(local_path) + allow(container_pool).to receive(:clean_workspace) + allow(FileUtils).to receive(:chmod_R) + end + + it 'creates the workspace directory' do + expect(FileUtils).to receive(:mkdir_p).with(local_path) + container_pool.copy_files(files) + end + + it 'cleans the workspace' do + expect(container_pool).to receive(:clean_workspace) + container_pool.copy_files(files) + end + + it 'sets permission bits on the workspace' do + expect(FileUtils).to receive(:chmod_R).with('+rwX', local_path) + container_pool.copy_files(files) + end + + context 'when receiving a normal file' do + let(:file_content) { 'print("Hello World!")' } + let(:files) { [FactoryBot.build(:file, content: file_content)] } + + it 'writes the file to disk' do + file = instance_double(File) + allow(File).to receive(:open).and_yield(file) + expect(file).to receive(:write).with(file_content) + container_pool.copy_files(files) + end + + it 'creates the file inside the workspace' do + expect(File).to receive(:open).with(local_path.join(files.first.filepath), 'w') + container_pool.copy_files(files) + end + + it 'raises an error in case of an IOError' do + allow(File).to receive(:open).and_raise(IOError) + expect { container_pool.copy_files(files) }.to raise_error(Runner::Error::Unknown, /#{files.first.filepath}/) + end + + it 'does not create a directory for it' do + expect(FileUtils).not_to receive(:mkdir_p) + end + + context 'when the file is inside a directory' do + let(:directory) { 'temp/dir' } + let(:files) { [FactoryBot.build(:file, path: directory)] } + + before do + allow(File).to receive(:open) + allow(FileUtils).to receive(:mkdir_p).with(local_path) + allow(FileUtils).to receive(:mkdir_p).with(local_path.join(directory)) + end + + it 'cleans the directory path' do + allow(container_pool).to receive(:local_path).and_call_original + expect(container_pool).to receive(:local_path).with(directory).and_call_original + container_pool.copy_files(files) + end + + it 'creates the directory of the file' do + expect(FileUtils).to receive(:mkdir_p).with(local_path.join(directory)) + container_pool.copy_files(files) + end + end + end + + context 'when receiving a binary file' do + let(:files) { [FactoryBot.build(:file, :image)] } + + it 'copies the file inside the workspace' do + expect(FileUtils).to receive(:cp).with(files.first.native_file.path, local_path.join(files.first.filepath)) + container_pool.copy_files(files) + end + end + + context 'when receiving multiple files' do + let(:files) { FactoryBot.build_list(:file, 3) } + + it 'creates all files' do + files.each do |file| + expect(File).to receive(:open).with(local_path.join(file.filepath), 'w') + end + container_pool.copy_files(files) + end + end + end + + describe '#local_workspace_path' do + before { allow(container_pool).to receive(:container).and_return(container) } + + it 'returns the local part of the mount binding' do + local_path = 'tmp/container20' + allow(container).to receive(:binds).and_return(["#{local_path}:/workspace"]) + expect(container_pool.send(:local_workspace_path)).to eq(Pathname.new(local_path)) + end + end + + describe '#local_path' do + let(:local_workspace) { Pathname.new('/tmp/workspace') } + + before { allow(container_pool).to receive(:local_workspace_path).and_return(local_workspace) } + + it 'raises an error for relative paths outside of the workspace' do + expect { container_pool.send(:local_path, '../exercise.py') }.to raise_error(Runner::Error::Unknown, %r{tmp/exercise.py}) + end + + it 'raises an error for absolute paths outside of the workspace' do + expect { container_pool.send(:local_path, '/test') }.to raise_error(Runner::Error::Unknown, %r{/test}) + end + + it 'removes .. from the path' do + expect(container_pool.send(:local_path, 'test/../exercise.py')).to eq(Pathname.new('/tmp/workspace/exercise.py')) + end + + it 'joins the given path with the local workspace path' do + expect(container_pool.send(:local_path, 'exercise.py')).to eq(Pathname.new('/tmp/workspace/exercise.py')) + end + end + + describe '#clean_workspace' do + let(:local_workspace) { instance_double(Pathname) } + + before { allow(container_pool).to receive(:local_workspace_path).and_return(local_workspace) } + + it 'removes all children of the workspace recursively' do + children = %w[test.py exercise.rb subfolder].map {|child| Pathname.new(child) } + allow(local_workspace).to receive(:children).and_return(children) + expect(FileUtils).to receive(:rm_r).with(children, secure: true) + container_pool.send(:clean_workspace) + end + + it 'raises an error if the workspace does not exist' do + allow(local_workspace).to receive(:children).and_raise(Errno::ENOENT) + expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::Unknown, /not exist/) + end + + it 'raises an error if it lacks permission for deleting an entry' do + allow(local_workspace).to receive(:children).and_return(['test.py']) + allow(FileUtils).to receive(:remove_entry_secure).and_raise(Errno::EACCES) + expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::Unknown, /Not allowed/) + end + end + + describe '#container' do + it 'raises an error if there is no container for the saved id' do + allow(Docker::Container).to receive(:get).and_raise(Docker::Error::NotFoundError) + expect { container_pool.send(:container) }.to raise_error(Runner::Error::RunnerNotFound) + end + + it 'raises an error if the received container is not running' do + allow(Docker::Container).to receive(:get).and_return(container) + allow(container).to receive(:info).and_return({'State' => {'Running' => false}}) + expect { container_pool.send(:container) }.to raise_error(Runner::Error::RunnerNotFound) + end + + it 'returns the received container' do + allow(Docker::Container).to receive(:get).and_return(container) + allow(container).to receive(:info).and_return({'State' => {'Running' => true}}) + expect(container_pool.send(:container)).to eq(container) + end + + it 'does not request a container if one is saved' do + container_pool.instance_variable_set(:@container, container) + expect(Docker::Container).not_to receive(:get) + container_pool.send(:container) + end + end + + describe '#attach_to_execution' do + # TODO: add more tests here + + let(:command) { 'ls' } + let(:action) { -> { container_pool.attach_to_execution(command) } } + let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } + + it 'returns the execution time' do + allow(EventMachine).to receive(:run) + starting_time = Time.zone.now + execution_time = action.call + test_time = Time.zone.now - starting_time + expect(execution_time).to be_between(0.0, test_time).exclusive + end end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 62bbd638..add7037e 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -292,7 +292,7 @@ describe Runner::Strategy::Poseidon do # TODO: add more tests here let(:command) { 'ls' } - let(:action) { -> { poseidon.attach_to_execution command } } + let(:action) { -> { poseidon.attach_to_execution(command) } } let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } it 'returns the execution time' do @@ -302,7 +302,7 @@ describe Runner::Strategy::Poseidon do starting_time = Time.zone.now execution_time = action.call test_time = Time.zone.now - starting_time - expect(execution_time).to be_between(0.0, test_time) + expect(execution_time).to be_between(0.0, test_time).exclusive end end end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index b9395645..771f2b4d 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -98,7 +98,8 @@ describe Runner do runner.copy_files(nil) end - it 'retries to copy the files' do + it 'calls copy_file twice' do + # copy_files is called again after a new runner was requested. expect(strategy).to receive(:copy_files).twice runner.copy_files(nil) end From cc412b73bccfa140a1a3eb34c8fc36b5979427ab Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 24 Jun 2021 12:42:43 +0200 Subject: [PATCH 041/156] Introduce more error types --- app/errors/runner/error.rb | 6 ++++++ lib/runner/strategy/docker_container_pool.rb | 14 +++++++------- lib/runner/strategy/poseidon.rb | 18 +++++++++--------- .../strategy/docker_container_pool_spec.rb | 14 +++++++------- spec/lib/runner/strategy/poseidon_spec.rb | 12 ++++++------ 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/app/errors/runner/error.rb b/app/errors/runner/error.rb index ba6aeee1..41b54c31 100644 --- a/app/errors/runner/error.rb +++ b/app/errors/runner/error.rb @@ -16,6 +16,12 @@ class Runner class RunnerNotFound < Error; end + class FaradayError < Error; end + + class UnexpectedResponse < Error; end + + class WorkspaceError < Error; end + class Unknown < Error; end end end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 7003f901..84d5219e 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -12,9 +12,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy container_id = JSON.parse(Faraday.get("#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}").body)['id'] container_id.presence || raise(Runner::Error::NotAvailable.new("DockerContainerPool didn't return a container id")) rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to DockerContainerPool failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") rescue JSON::ParserError => e - raise Runner::Error::Unknown.new("DockerContainerPool returned invalid JSON: #{e.inspect}") + raise Runner::Error::UnexpectedResponse.new("DockerContainerPool returned invalid JSON: #{e.inspect}") end def initialize(runner_id, _environment) @@ -40,7 +40,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy rescue IOError => e # TODO: try catch i/o exception and log failed attempts # Does this fix the issue @Sebastian? What exceptions did you have in mind? - raise Runner::Error::Unknown.new("Could not create workspace file #{file.filepath}: #{e.inspect}") + raise Runner::Error::WorkspaceError.new("Could not create file #{file.filepath}: #{e.inspect}") end end end @@ -50,7 +50,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def destroy_at_management Faraday.get("#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}") rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to DockerContainerPool failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") end def attach_to_execution(command) @@ -88,7 +88,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy unclean_path = local_workspace_path.join(path) clean_path = File.expand_path(unclean_path) unless clean_path.to_s.start_with? local_workspace_path.to_s - raise Runner::Error::Unknown.new("Local filepath #{clean_path.inspect} not allowed") + raise Runner::Error::WorkspaceError.new("Local filepath #{clean_path.inspect} not allowed") end Pathname.new(clean_path) @@ -97,10 +97,10 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def clean_workspace FileUtils.rm_r(local_workspace_path.children, secure: true) rescue Errno::ENOENT => e - raise Runner::Error::Unknown.new("The workspace directory does not exist and cannot be deleted: #{e.inspect}") + raise Runner::Error::WorkspaceError.new("The workspace directory does not exist and cannot be deleted: #{e.inspect}") rescue Errno::EACCES => e # TODO: Why was this rescued before @Sebastian? - raise Runner::Error::Unknown.new("Not allowed to clean workspace #{local_workspace_path}: #{e.inspect}") + raise Runner::Error::WorkspaceError.new("Not allowed to clean workspace #{local_workspace_path}: #{e.inspect}") end def local_workspace_path diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 1938c229..fba0b475 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -23,14 +23,14 @@ class Runner::Strategy::Poseidon < Runner::Strategy when 200 response_body = parse response runner_id = response_body[:runnerId] - runner_id.presence || raise(Runner::Error::Unknown.new('Poseidon did not send a runner id')) + runner_id.presence || raise(Runner::Error::UnexpectedResponse.new('Poseidon did not send a runner id')) when 404 raise Runner::Error::EnvironmentNotFound.new else handle_error response end rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end def self.handle_error(response) @@ -51,7 +51,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy raise Runner::Error::InternalServerError.new("Poseidon sent #{response_body[:errorCode]}: #{response_body[:message]}") end else - raise Runner::Error::Unknown.new("Poseidon sent unexpected response status code #{response.status}") + raise Runner::Error::UnexpectedResponse.new("Poseidon sent unexpected response status code #{response.status}") end end @@ -59,7 +59,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy JSON.parse(response.body).deep_symbolize_keys rescue JSON::ParserError => e # Poseidon should not send invalid json - raise Runner::Error::Unknown.new("Error parsing response from Poseidon: #{e.message}") + raise Runner::Error::UnexpectedResponse.new("Error parsing response from Poseidon: #{e.message}") end def initialize(runner_id, _environment) @@ -82,7 +82,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy Runner.destroy(@allocation_id) if response.status == 400 self.class.handle_error response rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end def attach_to_execution(command) @@ -99,7 +99,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy response = Faraday.delete runner_url self.class.handle_error response unless response.status == 204 rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end private @@ -115,7 +115,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy if websocket_url.present? return websocket_url else - raise Runner::Error::Unknown.new('Poseidon did not send websocket url') + raise Runner::Error::UnexpectedResponse.new('Poseidon did not send websocket url') end when 400 Runner.destroy(@allocation_id) @@ -123,7 +123,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy self.class.handle_error response rescue Faraday::Error => e - raise Runner::Error::Unknown.new("Faraday request to runner management failed: #{e.inspect}") + raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end def runner_url @@ -134,7 +134,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy def decode(raw_event) JSON.parse(raw_event.data) rescue JSON::ParserError => e - raise Runner::Error::Unknown.new("The websocket message from Poseidon could not be decoded to JSON: #{e.inspect}") + raise Runner::Error::UnexpectedResponse.new("The websocket message from Poseidon could not be decoded to JSON: #{e.inspect}") end def encode(data) diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index d5c5a455..c8dabb91 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -20,7 +20,7 @@ describe Runner::Strategy::DockerContainerPool do shared_examples 'Faraday error handling' do it 'raises a runner error' do allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError) - expect { action.call }.to raise_error(Runner::Error::Unknown, /Faraday/) + expect { action.call }.to raise_error(Runner::Error::FaradayError) end end @@ -59,7 +59,7 @@ describe Runner::Strategy::DockerContainerPool do let(:response_body) { '{hello}' } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse) end end @@ -128,7 +128,7 @@ describe Runner::Strategy::DockerContainerPool do it 'raises an error in case of an IOError' do allow(File).to receive(:open).and_raise(IOError) - expect { container_pool.copy_files(files) }.to raise_error(Runner::Error::Unknown, /#{files.first.filepath}/) + expect { container_pool.copy_files(files) }.to raise_error(Runner::Error::WorkspaceError, /#{files.first.filepath}/) end it 'does not create a directory for it' do @@ -195,11 +195,11 @@ describe Runner::Strategy::DockerContainerPool do before { allow(container_pool).to receive(:local_workspace_path).and_return(local_workspace) } it 'raises an error for relative paths outside of the workspace' do - expect { container_pool.send(:local_path, '../exercise.py') }.to raise_error(Runner::Error::Unknown, %r{tmp/exercise.py}) + expect { container_pool.send(:local_path, '../exercise.py') }.to raise_error(Runner::Error::WorkspaceError, %r{tmp/exercise.py}) end it 'raises an error for absolute paths outside of the workspace' do - expect { container_pool.send(:local_path, '/test') }.to raise_error(Runner::Error::Unknown, %r{/test}) + expect { container_pool.send(:local_path, '/test') }.to raise_error(Runner::Error::WorkspaceError, %r{/test}) end it 'removes .. from the path' do @@ -225,13 +225,13 @@ describe Runner::Strategy::DockerContainerPool do it 'raises an error if the workspace does not exist' do allow(local_workspace).to receive(:children).and_raise(Errno::ENOENT) - expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::Unknown, /not exist/) + expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::WorkspaceError, /not exist/) end it 'raises an error if it lacks permission for deleting an entry' do allow(local_workspace).to receive(:children).and_return(['test.py']) allow(FileUtils).to receive(:remove_entry_secure).and_raise(Errno::EACCES) - expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::Unknown, /Not allowed/) + expect { container_pool.send(:clean_workspace) }.to raise_error(Runner::Error::WorkspaceError, /Not allowed/) end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index add7037e..3de5d957 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -95,7 +95,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 1337 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown, /#{response_status}/) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse, /#{response_status}/) end end end @@ -109,7 +109,7 @@ describe Runner::Strategy::Poseidon do it 'raises an error' do %i[post patch delete].each {|message| allow(Faraday).to receive(message).and_raise(Faraday::TimeoutError) } - expect { action.call }.to raise_error(Runner::Error::Unknown, /Faraday/) + expect { action.call }.to raise_error(Runner::Error::FaradayError) end end end @@ -146,7 +146,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 200 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse) end end @@ -155,7 +155,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 200 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse) end end @@ -209,7 +209,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 200 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse) end end @@ -218,7 +218,7 @@ describe Runner::Strategy::Poseidon do let(:response_status) { 200 } it 'raises an error' do - expect { action.call }.to raise_error(Runner::Error::Unknown) + expect { action.call }.to raise_error(Runner::Error::UnexpectedResponse) end end From d1a5773e6032b96dc80d9840d253a584040b70b5 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 28 Jun 2021 10:11:15 +0200 Subject: [PATCH 042/156] Add debug log statements to runner connection --- app/controllers/submissions_controller.rb | 1 - lib/runner/connection.rb | 12 ++++++++---- lib/runner/strategy/docker_container_pool.rb | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index b01cf970..b2ad3d85 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -137,7 +137,6 @@ class SubmissionsController < ApplicationController @output = +'' socket.on :output do |data| - Rails.logger.info("#{Time.zone.now.getutc}: Container sending: #{data.inspect}") @output << data if @output.size + data.size <= max_output_buffer_size end diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 4bf725af..05f1242f 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -34,8 +34,10 @@ class Runner::Connection instance_variable_set(:"@#{event}_callback", block) end - def send(data) - @socket.send(encode(data)) + def send(raw_data) + encoded_message = encode(raw_data) + Rails.logger.debug("#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}") + @socket.send(encoded_message) end private @@ -49,6 +51,7 @@ class Runner::Connection end def on_message(raw_event) + Rails.logger.debug("#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}") event = decode(raw_event) return unless BACKEND_OUTPUT_SCHEMA.valid?(event) @@ -64,10 +67,11 @@ class Runner::Connection def on_error(_event); end def on_close(_event) + Rails.logger.debug("#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}") case @status when :timeout raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') - when :terminated + when :terminated_by_codeocean, :terminated_by_management @exit_callback.call @exit_code else # :established # If the runner is killed by the DockerContainerPool after the maximum allowed time per user and @@ -77,7 +81,7 @@ class Runner::Connection end def handle_exit(event) - @status = :terminated + @status = :terminated_by_management @exit_code = event[:data] end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 84d5219e..2a8026a4 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -123,7 +123,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy # Assume correct termination for now and return exit code 0 # TODO: Can we use the actual exit code here? @exit_code = 0 - @status = :terminated + @status = :terminated_by_codeocean @socket.close when /#{format(@strategy.execution_environment.test_command, class_name: '.*', filename: '.*', module_name: '.*')}/ # TODO: Super dirty hack to redirect test output to stderr (remove attr_reader afterwards) From 5608d61b3a8af7af688fded5745c4c46bc2ff578 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Fri, 25 Jun 2021 09:24:13 +0200 Subject: [PATCH 043/156] Replace metaprogramming in Runner::Connection This prevents someone who is controlling the websocket connection to send messages starting with 'handle_' to the connection object. --- lib/runner/connection.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 05f1242f..614efdd0 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -6,6 +6,7 @@ require 'json_schemer' class Runner::Connection # These are events for which callbacks can be registered. EVENTS = %i[start output exit stdout stderr].freeze + WEBSOCKET_MESSAGE_TYPES = %i[start stdout stderr error timeout exit].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) attr_writer :status @@ -56,8 +57,12 @@ class Runner::Connection return unless BACKEND_OUTPUT_SCHEMA.valid?(event) event = event.deep_symbolize_keys - # There is one `handle_` method for every message type defined in the WebSocket schema. - __send__("handle_#{event[:type]}", event) + message_type = event[:type] + if WEBSOCKET_MESSAGE_TYPES.include?(message_type) + __send__("handle_#{message_type}", event) + else + raise Runner::Error::UnexpectedResponse.new("Unknown websocket message type: #{message_type}") + end end def on_open(_event) From f98a8b9e7a70c1dff207d49d0671688dd1931168 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 1 Jul 2021 13:52:05 +0200 Subject: [PATCH 044/156] Resolve error handling todos in dcp strategy --- lib/runner/strategy/docker_container_pool.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 2a8026a4..9dce17df 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -38,8 +38,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy begin File.open(local_file_path, 'w') {|f| f.write(file.content) } rescue IOError => e - # TODO: try catch i/o exception and log failed attempts - # Does this fix the issue @Sebastian? What exceptions did you have in mind? raise Runner::Error::WorkspaceError.new("Could not create file #{file.filepath}: #{e.inspect}") end end @@ -99,7 +97,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy rescue Errno::ENOENT => e raise Runner::Error::WorkspaceError.new("The workspace directory does not exist and cannot be deleted: #{e.inspect}") rescue Errno::EACCES => e - # TODO: Why was this rescued before @Sebastian? raise Runner::Error::WorkspaceError.new("Not allowed to clean workspace #{local_workspace_path}: #{e.inspect}") end From 36578a28172952613b25cde189df55881ea11ef8 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Fri, 25 Jun 2021 12:10:30 +0200 Subject: [PATCH 045/156] Ensure to save Testrun even when an error occurs --- app/controllers/submissions_controller.rb | 30 +++++++--------- app/models/submission.rb | 42 +++++++++++++---------- lib/runner/connection.rb | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index b2ad3d85..376a886b 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -134,18 +134,17 @@ class SubmissionsController < ApplicationController def handle_websockets(tubesock, socket) tubesock.send_data JSON.dump({cmd: :status, status: :container_running}) - @output = +'' - - socket.on :output do |data| - @output << data if @output.size + data.size <= max_output_buffer_size - end socket.on :stdout do |data| - tubesock.send_data(JSON.dump({cmd: :write, stream: :stdout, data: data})) + json_data = JSON.dump({cmd: :write, stream: :stdout, data: data}) + @output << json_data[0, max_output_buffer_size - @output.size] + tubesock.send_data(json_data) end socket.on :stderr do |data| - tubesock.send_data(JSON.dump({cmd: :write, stream: :stderr, data: data})) + json_data = JSON.dump({cmd: :write, stream: :stderr, data: data}) + @output << json_data[0, max_output_buffer_size - @output.size] + tubesock.send_data(json_data) end socket.on :exit do |exit_code| @@ -180,6 +179,7 @@ class SubmissionsController < ApplicationController end def run + @output = +'' hijack do |tubesock| return kill_socket(tubesock) if @embed_options[:disable_run] @@ -188,20 +188,22 @@ class SubmissionsController < ApplicationController end @container_execution_time = durations[:execution_duration] @waiting_for_container_time = durations[:waiting_duration] - save_run_output rescue Runner::Error::ExecutionTimeout => e tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) kill_socket(tubesock) Rails.logger.debug { "Running a submission timed out: #{e.message}" } + @output = "timeout: #{@output}" rescue Runner::Error => e tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) kill_socket(tubesock) Rails.logger.debug { "Runner error while running a submission: #{e.message}" } + ensure + save_run_output end end def kill_socket(tubesock) - # search for errors and save them as StructuredError (for scoring runs see submission_scoring.rb) + # search for errors and save them as StructuredError (for scoring runs see submission.rb) errors = extract_errors send_hints(tubesock, errors) @@ -210,11 +212,8 @@ class SubmissionsController < ApplicationController tubesock.close end - # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb) + # save the output of this "run" as a "testrun" (scoring runs are saved in submission.rb) def save_run_output - return if @output.blank? - - @output = @output[0, max_output_buffer_size] # trim the string to max_output_buffer_size chars Testrun.create( file: @file, cause: 'run', @@ -243,12 +242,9 @@ class SubmissionsController < ApplicationController tubesock.send_data(JSON.dump(format_scoring_results(@submission.calculate_score))) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) - rescue Runner::Error::ExecutionTimeout => e - tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) - Rails.logger.debug { "Scoring a submission timed out: #{e.message}" } rescue Runner::Error => e tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) - Rails.logger.debug { "Runner error while scoring a submission: #{e.message}" } + Rails.logger.debug { "Runner error while scoring submission #{@submission.id}: #{e.message}" } ensure tubesock.send_data JSON.dump({cmd: :exit}) tubesock.close diff --git a/app/models/submission.rb b/app/models/submission.rb index 11857df6..648be071 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -143,29 +143,33 @@ class Submission < ApplicationRecord prepared_runner do |runner, waiting_duration| file_scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension + output = {file_role: file.role, waiting_for_container_time: waiting_duration} stdout = +'' stderr = +'' - exit_code = 1 # default to error - execution_time = runner.attach_to_execution(score_command) do |socket| - socket.on :stderr do |data| - stderr << data - end - socket.on :stdout do |data| - stdout << data - end - socket.on :exit do |received_exit_code| - exit_code = received_exit_code - EventMachine.stop_event_loop + begin + exit_code = 1 # default to error + execution_time = runner.attach_to_execution(score_command) do |socket| + socket.on :stderr do |data| + stderr << data + end + socket.on :stdout do |data| + stdout << data + end + socket.on :exit do |received_exit_code| + exit_code = received_exit_code + EventMachine.stop_event_loop + end end + output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) + rescue Runner::Error::ExecutionTimeout => e + Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}") + output.merge!(status: :timeout) + rescue Runner::Error => e + Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}") + output.merge!(status: :failed) + ensure + output.merge!(stdout: stdout, stderr: stderr) end - output = { - file_role: file.role, - waiting_for_container_time: waiting_duration, - container_execution_time: execution_time, - status: exit_code.zero? ? :ok : :failed, - stdout: stdout, - stderr: stderr, - } score_file(output, file) end end diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 614efdd0..5c205be9 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -57,7 +57,7 @@ class Runner::Connection return unless BACKEND_OUTPUT_SCHEMA.valid?(event) event = event.deep_symbolize_keys - message_type = event[:type] + message_type = event[:type].to_sym if WEBSOCKET_MESSAGE_TYPES.include?(message_type) __send__("handle_#{message_type}", event) else From 2dff81a510d14c49e6910d492e01a330d79442bf Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:03:20 +0200 Subject: [PATCH 046/156] Attach duration information to the exception object --- app/controllers/submissions_controller.rb | 9 +++ app/errors/runner/error.rb | 2 + app/models/runner.rb | 9 ++- app/models/submission.rb | 17 +++- lib/runner/strategy/docker_container_pool.rb | 2 - lib/runner/strategy/poseidon.rb | 2 - .../strategy/docker_container_pool_spec.rb | 10 +-- spec/lib/runner/strategy/poseidon_spec.rb | 12 +-- spec/models/runner_spec.rb | 81 ++++++++++++------- 9 files changed, 87 insertions(+), 57 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 376a886b..cf82f546 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -193,15 +193,23 @@ class SubmissionsController < ApplicationController kill_socket(tubesock) Rails.logger.debug { "Running a submission timed out: #{e.message}" } @output = "timeout: #{@output}" + extract_durations(e) rescue Runner::Error => e tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) kill_socket(tubesock) Rails.logger.debug { "Runner error while running a submission: #{e.message}" } + extract_durations(e) ensure save_run_output end end + def extract_durations(error) + @container_execution_time = error.execution_duration + @waiting_for_container_time = error.waiting_duration + end + private :extract_durations + def kill_socket(tubesock) # search for errors and save them as StructuredError (for scoring runs see submission.rb) errors = extract_errors @@ -293,6 +301,7 @@ class SubmissionsController < ApplicationController def statistics; end # TODO: make this run, but with the test command + # TODO: add this method to the before action for set_submission again # def test # hijack do |tubesock| # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? diff --git a/app/errors/runner/error.rb b/app/errors/runner/error.rb index 41b54c31..e943db73 100644 --- a/app/errors/runner/error.rb +++ b/app/errors/runner/error.rb @@ -2,6 +2,8 @@ class Runner class Error < ApplicationError + attr_accessor :waiting_duration, :execution_duration + class BadRequest < Error; end class EnvironmentNotFound < Error; end diff --git a/app/models/runner.rb b/app/models/runner.rb index 5fd0628b..4e93d065 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -41,7 +41,14 @@ class Runner < ApplicationRecord end def attach_to_execution(command, &block) - @strategy.attach_to_execution(command, &block) + starting_time = Time.zone.now + begin + @strategy.attach_to_execution(command, &block) + rescue Runner::Error => e + e.execution_duration = Time.zone.now - starting_time + raise + end + Time.zone.now - starting_time # execution duration end def destroy_at_management diff --git a/app/models/submission.rb b/app/models/submission.rb index 648be071..4916bc49 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -140,6 +140,7 @@ class Submission < ApplicationRecord def calculate_score file_scores = nil + # If prepared_runner raises an error, no Testrun will be created. prepared_runner do |runner, waiting_duration| file_scores = collect_files.select(&:teacher_defined_assessment?).map do |file| score_command = command_for execution_environment.test_command, file.name_with_extension @@ -163,10 +164,10 @@ class Submission < ApplicationRecord output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) rescue Runner::Error::ExecutionTimeout => e Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}") - output.merge!(status: :timeout) + output.merge!(status: :timeout, container_execution_time: e.execution_duration) rescue Runner::Error => e Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}") - output.merge!(status: :failed) + output.merge!(status: :failed, container_execution_time: e.execution_duration) ensure output.merge!(stdout: stdout, stderr: stderr) end @@ -182,6 +183,9 @@ class Submission < ApplicationRecord prepared_runner do |runner, waiting_duration| durations[:execution_duration] = runner.attach_to_execution(run_command, &block) durations[:waiting_duration] = waiting_duration + rescue Runner::Error => e + e.waiting_duration = waiting_duration + raise end durations end @@ -190,8 +194,13 @@ class Submission < ApplicationRecord def prepared_runner request_time = Time.zone.now - runner = Runner.for(user, exercise) - runner.copy_files(collect_files) + begin + runner = Runner.for(user, exercise) + runner.copy_files(collect_files) + rescue Runner::Error => e + e.waiting_duration = Time.zone.now - request_time + raise + end waiting_duration = Time.zone.now - request_time yield(runner, waiting_duration) end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 9dce17df..adc0089f 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -53,7 +53,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def attach_to_execution(command) @command = command - starting_time = Time.zone.now query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{@container_id}/attach/ws?#{query_params}" @@ -66,7 +65,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy socket.send(command) yield(socket) end - Time.zone.now - starting_time # execution duration in seconds end private diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index fba0b475..54d51aab 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -86,13 +86,11 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def attach_to_execution(command) - starting_time = Time.zone.now websocket_url = execute_command(command) EventMachine.run do socket = Connection.new(websocket_url, self) yield(socket) end - Time.zone.now - starting_time # execution duration end def destroy_at_management diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index c8dabb91..26568b6f 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -261,18 +261,10 @@ describe Runner::Strategy::DockerContainerPool do end describe '#attach_to_execution' do - # TODO: add more tests here + # TODO: add tests here let(:command) { 'ls' } let(:action) { -> { container_pool.attach_to_execution(command) } } let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } - - it 'returns the execution time' do - allow(EventMachine).to receive(:run) - starting_time = Time.zone.now - execution_time = action.call - test_time = Time.zone.now - starting_time - expect(execution_time).to be_between(0.0, test_time).exclusive - end end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 3de5d957..ef04a6f7 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -289,20 +289,10 @@ describe Runner::Strategy::Poseidon do end describe '#attach_to_execution' do - # TODO: add more tests here + # TODO: add tests here let(:command) { 'ls' } let(:action) { -> { poseidon.attach_to_execution(command) } } let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } - - it 'returns the execution time' do - allow(poseidon).to receive(:execute_command).with(command).and_return(websocket_url) - allow(EventMachine).to receive(:run) - - starting_time = Time.zone.now - execution_time = action.call - test_time = Time.zone.now - starting_time - expect(execution_time).to be_between(0.0, test_time).exclusive - end end end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 771f2b4d..5202d403 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -5,6 +5,7 @@ require 'rails_helper' describe Runner do let(:runner_id) { FactoryBot.attributes_for(:runner)[:runner_id] } let(:strategy_class) { described_class.strategy_class } + let(:strategy) { instance_double(strategy_class) } describe 'attribute validation' do let(:runner) { FactoryBot.create :runner } @@ -44,30 +45,7 @@ describe Runner do end end - describe 'method delegation' do - shared_examples 'delegates method sends to its strategy' do |method, *args| - context "when sending #{method}" do - let(:strategy) { instance_double(strategy_class) } - let(:runner) { described_class.create } - - before do - allow(strategy_class).to receive(:request_from_management).and_return(runner_id) - allow(strategy_class).to receive(:new).and_return(strategy) - end - - it 'delegates to its strategy' do - expect(strategy).to receive(method) - runner.send(method, *args) - end - end - end - - include_examples 'delegates method sends to its strategy', :destroy_at_management - include_examples 'delegates method sends to its strategy', :attach_to_execution, nil - end - - describe '#copy_files' do - let(:strategy) { instance_double(strategy_class) } + describe '#destroy_at_management' do let(:runner) { described_class.create } before do @@ -75,12 +53,59 @@ describe Runner do allow(strategy_class).to receive(:new).and_return(strategy) end - context 'when no error is raised' do - it 'delegates to its strategy' do - expect(strategy).to receive(:copy_files).once - runner.copy_files(nil) + it 'delegates to its strategy' do + expect(strategy).to receive(:destroy_at_management) + runner.destroy_at_management + end + end + + describe '#attach to execution' do + let(:runner) { described_class.create } + let(:command) { 'ls' } + + before do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + allow(strategy_class).to receive(:new).and_return(strategy) + end + + it 'delegates to its strategy' do + expect(strategy).to receive(:attach_to_execution) + runner.attach_to_execution(command) + end + + it 'returns the execution time' do + allow(strategy).to receive(:attach_to_execution) + starting_time = Time.zone.now + execution_time = runner.attach_to_execution(command) + test_time = Time.zone.now - starting_time + expect(execution_time).to be_between(0.0, test_time).exclusive + end + + context 'when a runner error is raised' do + before { allow(strategy).to receive(:attach_to_execution).and_raise(Runner::Error) } + + it 'attaches the execution time to the error' do + starting_time = Time.zone.now + expect { runner.attach_to_execution(command) }.to raise_error do |error| + test_time = Time.zone.now - starting_time + expect(error.execution_duration).to be_between(0.0, test_time).exclusive + end end end + end + + describe '#copy_files' do + let(:runner) { described_class.create } + + before do + allow(strategy_class).to receive(:request_from_management).and_return(runner_id) + allow(strategy_class).to receive(:new).and_return(strategy) + end + + it 'delegates to its strategy' do + expect(strategy).to receive(:copy_files).once + runner.copy_files(nil) + end context 'when a RunnerNotFound exception is raised' do before do From 5cc180d0e9148b632c8d7b7f6a9e8a8b9a08449d Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Mon, 26 Jul 2021 14:16:41 +0200 Subject: [PATCH 047/156] Fix rubocop, I18n, cleanup rebase --- Gemfile | 2 +- .../concerns/scoring_result_formatting.rb | 11 ---- .../remote_evaluation_controller.rb | 3 +- .../request_for_comments_controller.rb | 2 +- app/controllers/submissions_controller.rb | 12 +---- app/models/submission.rb | 21 +++++--- lib/runner/connection.rb | 6 +-- .../scoring_result_formatting_spec.rb | 54 ------------------- 8 files changed, 20 insertions(+), 91 deletions(-) delete mode 100644 app/controllers/concerns/scoring_result_formatting.rb delete mode 100644 spec/concerns/scoring_result_formatting_spec.rb diff --git a/Gemfile b/Gemfile index 073f6d8a..c75764ec 100644 --- a/Gemfile +++ b/Gemfile @@ -16,8 +16,8 @@ gem 'highline' gem 'i18n-js' gem 'ims-lti', '< 2.0.0' gem 'jbuilder' -gem 'js-routes' gem 'json_schemer' +gem 'js-routes' gem 'kramdown' gem 'mimemagic' gem 'nokogiri' diff --git a/app/controllers/concerns/scoring_result_formatting.rb b/app/controllers/concerns/scoring_result_formatting.rb deleted file mode 100644 index 27096413..00000000 --- a/app/controllers/concerns/scoring_result_formatting.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module ScoringResultFormatting - def format_scoring_results(outputs) - outputs.map do |output| - output[:message] = t(output[:message], default: render_markdown(output[:message])) - output[:filename] = t(output[:filename], default: output[:filename]) - output - end - end -end diff --git a/app/controllers/remote_evaluation_controller.rb b/app/controllers/remote_evaluation_controller.rb index 2f23eacb..96db6c8b 100644 --- a/app/controllers/remote_evaluation_controller.rb +++ b/app/controllers/remote_evaluation_controller.rb @@ -3,7 +3,6 @@ class RemoteEvaluationController < ApplicationController include RemoteEvaluationParameters include Lti - include ScoringResultFormatting skip_after_action :verify_authorized skip_before_action :verify_authenticity_token @@ -63,7 +62,7 @@ status: 202} validation_token = remote_evaluation_params[:validation_token] if (remote_evaluation_mapping = RemoteEvaluationMapping.find_by(validation_token: validation_token)) @submission = Submission.create(build_submission_params(cause, remote_evaluation_mapping)) - format_scoring_results(@submission.calculate_score) + @submission.calculate_score else # TODO: better output # TODO: check token expired? diff --git a/app/controllers/request_for_comments_controller.rb b/app/controllers/request_for_comments_controller.rb index 2fd4046e..4592d57d 100644 --- a/app/controllers/request_for_comments_controller.rb +++ b/app/controllers/request_for_comments_controller.rb @@ -119,7 +119,7 @@ class RequestForCommentsController < ApplicationController if @request_for_comment.save # create thread here and execute tests. A run is triggered from the frontend and does not need to be handled here. Thread.new do - @request_for_comment.submission.calculate_score + switch_locale { @request_for_comment.submission.calculate_score } ensure ActiveRecord::Base.connection_pool.release_connection end diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index cf82f546..97f9cd3c 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -5,7 +5,6 @@ class SubmissionsController < ApplicationController include CommonBehavior include Lti include SubmissionParameters - include ScoringResultFormatting include Tubesock::Hijack before_action :set_submission, @@ -35,15 +34,6 @@ class SubmissionsController < ApplicationController create_and_respond(object: @submission) end - def command_substitutions(filename) - { - class_name: File.basename(filename, File.extname(filename)).upcase_first, - filename: filename, - module_name: File.basename(filename, File.extname(filename)).underscore, - } - end - private :command_substitutions - def copy_comments # copy each annotation and set the target_file.id params[:annotations_arr]&.each do |annotation| @@ -247,7 +237,7 @@ class SubmissionsController < ApplicationController hijack do |tubesock| return kill_socket(tubesock) if @embed_options[:disable_run] - tubesock.send_data(JSON.dump(format_scoring_results(@submission.calculate_score))) + tubesock.send_data(JSON.dump(@submission.calculate_score)) # To enable hints when scoring a submission, uncomment the next line: # send_hints(tubesock, StructuredError.where(submission: @submission)) rescue Runner::Error => e diff --git a/app/models/submission.rb b/app/models/submission.rb index 4916bc49..532a0c0c 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -163,10 +163,10 @@ class Submission < ApplicationRecord end output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) rescue Runner::Error::ExecutionTimeout => e - Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}") + Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}" } output.merge!(status: :timeout, container_execution_time: e.execution_duration) rescue Runner::Error => e - Rails.logger.debug("Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}") + Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}" } output.merge!(status: :failed, container_execution_time: e.execution_duration) ensure output.merge!(stdout: stdout, stderr: stderr) @@ -212,7 +212,7 @@ class Submission < ApplicationRecord def command_substitutions(filename) { - class_name: File.basename(filename, File.extname(filename)).camelize, + class_name: File.basename(filename, File.extname(filename)).upcase_first, filename: filename, module_name: File.basename(filename, File.extname(filename)).underscore, } @@ -245,10 +245,10 @@ class Submission < ApplicationRecord if file.teacher_defined_linter? LinterCheckRun.create_from(testrun, assessment) - assessment = assessor.translate_linter(assessment, session[:locale]) + assessment = assessor.translate_linter(assessment, I18n.locale) # replace file name with hint if linter is not used for grading. Refactor! - filename = 'exercises.implement.not_graded' if file.weight.zero? + filename = I18n.t('exercises.implement.not_graded') if file.weight.zero? end output.merge!(assessment) @@ -257,11 +257,16 @@ class Submission < ApplicationRecord def feedback_message(file, output) if output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_test' - 'exercises.implement.default_test_feedback' + I18n.t('exercises.implement.default_test_feedback') elsif output[:score] == Assessor::MAXIMUM_SCORE && output[:file_role] == 'teacher_defined_linter' - 'exercises.implement.default_linter_feedback' + I18n.t('exercises.implement.default_linter_feedback') else - file.feedback_message + # The render_markdown method from application_helper.rb is not available in model classes. + ActionController::Base.helpers.sanitize( + Kramdown::Document.new(file.feedback_message).to_html, + tags: %w[strong], + attributes: [] + ) end end diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 5c205be9..bc5d55e8 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -37,7 +37,7 @@ class Runner::Connection def send(raw_data) encoded_message = encode(raw_data) - Rails.logger.debug("#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}") + Rails.logger.debug { "#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}" } @socket.send(encoded_message) end @@ -52,7 +52,7 @@ class Runner::Connection end def on_message(raw_event) - Rails.logger.debug("#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}") + Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } event = decode(raw_event) return unless BACKEND_OUTPUT_SCHEMA.valid?(event) @@ -72,7 +72,7 @@ class Runner::Connection def on_error(_event); end def on_close(_event) - Rails.logger.debug("#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}") + Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } case @status when :timeout raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') diff --git a/spec/concerns/scoring_result_formatting_spec.rb b/spec/concerns/scoring_result_formatting_spec.rb deleted file mode 100644 index d66f9e3d..00000000 --- a/spec/concerns/scoring_result_formatting_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -class Controller < AnonymousController - include ScoringResultFormatting -end - -describe ScoringResultFormatting do - let(:controller) { Controller.new } - let(:filename) { 'exercise.py' } - let(:feedback_message) { '**good work**' } - let(:outputs) { [{filename: filename, message: feedback_message}] } - - describe 'feedback message' do - let(:new_feedback_message) { controller.format_scoring_results(outputs).first[:message] } - - context 'when the feedback message is not a path to a locale' do - let(:feedback_message) { '**good work**' } - - it 'renders the feedback message as markdown' do - expect(new_feedback_message).to match('

good work

') - end - end - - context 'when the feedback message is a valid path to a locale' do - let(:feedback_message) { 'exercises.implement.default_test_feedback' } - - it 'replaces the feedback message with the locale' do - expect(new_feedback_message).to eq(I18n.t(feedback_message)) - end - end - end - - describe 'filename' do - let(:new_filename) { controller.format_scoring_results(outputs).first[:filename] } - - context 'when the filename is not a path to a locale' do - let(:filename) { 'exercise.py' } - - it 'does not alter the filename' do - expect(new_filename).to eq(filename) - end - end - - context 'when the filename is a valid path to a locale' do - let(:filename) { 'exercises.implement.not_graded' } - - it 'replaces the filename with the locale' do - expect(new_filename).to eq(I18n.t(filename)) - end - end - end -end From c7369366d5c049db05889b80ec6d416ffb760ae8 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 29 Jun 2021 16:53:26 +0200 Subject: [PATCH 048/156] Ensure that only one EventMachine is running --- app/controllers/submissions_controller.rb | 158 ++++++++++--------- app/models/runner.rb | 26 ++- app/models/submission.rb | 1 - lib/runner/connection.rb | 21 ++- lib/runner/event_loop.rb | 22 +++ lib/runner/strategy/docker_container_pool.rb | 22 +-- lib/runner/strategy/poseidon.rb | 12 +- 7 files changed, 166 insertions(+), 96 deletions(-) create mode 100644 lib/runner/event_loop.rb diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 97f9cd3c..b2e0bbe9 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -122,76 +122,85 @@ class SubmissionsController < ApplicationController end end - def handle_websockets(tubesock, socket) - tubesock.send_data JSON.dump({cmd: :status, status: :container_running}) - - socket.on :stdout do |data| - json_data = JSON.dump({cmd: :write, stream: :stdout, data: data}) - @output << json_data[0, max_output_buffer_size - @output.size] - tubesock.send_data(json_data) - end - - socket.on :stderr do |data| - json_data = JSON.dump({cmd: :write, stream: :stderr, data: data}) - @output << json_data[0, max_output_buffer_size - @output.size] - tubesock.send_data(json_data) - end - - socket.on :exit do |exit_code| - EventMachine.stop_event_loop - if @output.empty? - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) - end - tubesock.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) - kill_socket(tubesock) - end - - tubesock.onmessage do |event| - event = JSON.parse(event).deep_symbolize_keys - case event[:cmd].to_sym - when :client_kill - EventMachine.stop_event_loop - kill_socket(tubesock) - Rails.logger.debug('Client exited container.') - when :result - socket.send event[:data] - else - Rails.logger.info("Unknown command from client: #{event[:cmd]}") - end - - rescue JSON::ParserError - Rails.logger.debug { "Data received from client is not valid json: #{data}" } - Sentry.set_extras(data: data) - rescue TypeError - Rails.logger.debug { "JSON data received from client cannot be parsed to hash: #{data}" } - Sentry.set_extras(data: data) - end - end - def run - @output = +'' - hijack do |tubesock| - return kill_socket(tubesock) if @embed_options[:disable_run] + # These method-local socket variables are required in order to use one socket + # in the callbacks of the other socket. As the callbacks for the client socket + # are registered first, the runner socket may still be nil. + client_socket, runner_socket = nil - durations = @submission.run(sanitize_filename) do |socket| - handle_websockets(tubesock, socket) + hijack do |tubesock| + client_socket = tubesock + return kill_client_socket(client_socket) if @embed_options[:disable_run] + + client_socket.onclose do |_event| + runner_socket&.close(:terminated_by_client) + end + + client_socket.onmessage do |event| + event = JSON.parse(event).deep_symbolize_keys + case event[:cmd].to_sym + when :client_kill + close_client_connection(client_socket) + Rails.logger.debug('Client exited container.') + when :result + # The client cannot send something before the runner connection is established. + if runner_socket.present? + runner_socket.send event[:data] + else + Rails.logger.info("Could not forward data from client because runner connection was not established yet: #{event[:data].inspect}") + end + else + Rails.logger.info("Unknown command from client: #{event[:cmd]}") + end + rescue JSON::ParserError + Rails.logger.info("Data received from client is not valid json: #{data.inspect}") + Sentry.set_extras(data: data) + rescue TypeError + Rails.logger.info("JSON data received from client cannot be parsed as hash: #{data.inspect}") + Sentry.set_extras(data: data) end - @container_execution_time = durations[:execution_duration] - @waiting_for_container_time = durations[:waiting_duration] - rescue Runner::Error::ExecutionTimeout => e - tubesock.send_data JSON.dump({cmd: :status, status: :timeout}) - kill_socket(tubesock) - Rails.logger.debug { "Running a submission timed out: #{e.message}" } - @output = "timeout: #{@output}" - extract_durations(e) - rescue Runner::Error => e - tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) - kill_socket(tubesock) - Rails.logger.debug { "Runner error while running a submission: #{e.message}" } - extract_durations(e) - ensure - save_run_output end + + @output = +'' + durations = @submission.run(sanitize_filename) do |socket| + runner_socket = socket + client_socket.send_data JSON.dump({cmd: :status, status: :container_running}) + + runner_socket.on :stdout do |data| + json_data = JSON.dump({cmd: :write, stream: :stdout, data: data}) + @output << json_data[0, max_output_buffer_size - @output.size] + client_socket.send_data(json_data) + end + + runner_socket.on :stderr do |data| + json_data = JSON.dump({cmd: :write, stream: :stderr, data: data}) + @output << json_data[0, max_output_buffer_size - @output.size] + client_socket.send_data(json_data) + end + + runner_socket.on :exit do |exit_code| + if @output.empty? + client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) + end + client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) + close_client_connection(client_socket) + end + end + @container_execution_time = durations[:execution_duration] + @waiting_for_container_time = durations[:waiting_duration] + rescue Runner::Error::ExecutionTimeout => e + client_socket.send_data JSON.dump({cmd: :status, status: :timeout}) + close_client_connection(client_socket) + Rails.logger.debug { "Running a submission timed out: #{e.message}" } + @output = "timeout: #{@output}" + extract_durations(e) + rescue Runner::Error => e + client_socket.send_data JSON.dump({cmd: :status, status: :container_depleted}) + close_client_connection(client_socket) + Rails.logger.debug { "Runner error while running a submission: #{e.message}" } + extract_durations(e) + ensure + save_run_output end def extract_durations(error) @@ -200,14 +209,16 @@ class SubmissionsController < ApplicationController end private :extract_durations - def kill_socket(tubesock) + def close_client_connection(client_socket) # search for errors and save them as StructuredError (for scoring runs see submission.rb) errors = extract_errors - send_hints(tubesock, errors) + send_hints(client_socket, errors) + kill_client_socket(client_socket) + end - # Hijacked connection needs to be notified correctly - tubesock.send_data JSON.dump({cmd: :exit}) - tubesock.close + def kill_client_socket(client_socket) + client_socket.send_data JSON.dump({cmd: :exit}) + client_socket.close end # save the output of this "run" as a "testrun" (scoring runs are saved in submission.rb) @@ -235,7 +246,7 @@ class SubmissionsController < ApplicationController def score hijack do |tubesock| - return kill_socket(tubesock) if @embed_options[:disable_run] + return if @embed_options[:disable_run] tubesock.send_data(JSON.dump(@submission.calculate_score)) # To enable hints when scoring a submission, uncomment the next line: @@ -244,8 +255,7 @@ class SubmissionsController < ApplicationController tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) Rails.logger.debug { "Runner error while scoring submission #{@submission.id}: #{e.message}" } ensure - tubesock.send_data JSON.dump({cmd: :exit}) - tubesock.close + kill_client_socket(tubesock) end end diff --git a/app/models/runner.rb b/app/models/runner.rb index 4e93d065..37cad548 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -41,9 +41,17 @@ class Runner < ApplicationRecord end def attach_to_execution(command, &block) + ensure_event_machine starting_time = Time.zone.now begin - @strategy.attach_to_execution(command, &block) + # As the EventMachine reactor is probably shared with other threads, we cannot use EventMachine.run with + # stop_event_loop to wait for the WebSocket connection to terminate. Instead we use a self built event + # loop for that: Runner::EventLoop. The attach_to_execution method of the strategy is responsible for + # initializing its Runner::Connection with the given event loop. The Runner::Connection class ensures that + # this event loop is stopped after the socket was closed. + event_loop = Runner::EventLoop.new + @strategy.attach_to_execution(command, event_loop, &block) + event_loop.wait rescue Runner::Error => e e.execution_duration = Time.zone.now - starting_time raise @@ -57,6 +65,22 @@ class Runner < ApplicationRecord private + # If there are multiple threads trying to connect to the WebSocket of their execution at the same time, + # the Faye WebSocket connections will use the same reactor. We therefore only need to start an EventMachine + # if there isn't a running reactor yet. + # See this StackOverflow answer: https://stackoverflow.com/a/8247947 + def ensure_event_machine + unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? + event_loop = Runner::EventLoop.new + Thread.new do + EventMachine.run { event_loop.stop } + ensure + ActiveRecord::Base.connection_pool.release_connection + end + event_loop.wait + end + end + def request_id request_new_id if runner_id.blank? end diff --git a/app/models/submission.rb b/app/models/submission.rb index 532a0c0c..b9442084 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -158,7 +158,6 @@ class Submission < ApplicationRecord end socket.on :exit do |received_exit_code| exit_code = received_exit_code - EventMachine.stop_event_loop end end output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index bc5d55e8..d49c6eb8 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -11,12 +11,13 @@ class Runner::Connection attr_writer :status - def initialize(url, strategy) + def initialize(url, strategy, event_loop) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) @strategy = strategy @status = :established + @event_loop = event_loop - # For every event type of faye websockets, the corresponding + # For every event type of Faye WebSockets, the corresponding # RunnerConnection method starting with `on_` is called. %i[open message error close].each do |event_type| @socket.on(event_type) {|event| __send__(:"on_#{event_type}", event) } @@ -41,6 +42,17 @@ class Runner::Connection @socket.send(encoded_message) end + def close(status) + return unless active? + + @status = status + @socket.close + end + + def active? + @status == :established + end + private def decode(_raw_event) @@ -61,7 +73,7 @@ class Runner::Connection if WEBSOCKET_MESSAGE_TYPES.include?(message_type) __send__("handle_#{message_type}", event) else - raise Runner::Error::UnexpectedResponse.new("Unknown websocket message type: #{message_type}") + raise Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") end end @@ -78,6 +90,9 @@ class Runner::Connection raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') when :terminated_by_codeocean, :terminated_by_management @exit_callback.call @exit_code + @event_loop.stop + when :terminated_by_client + @event_loop.stop else # :established # If the runner is killed by the DockerContainerPool after the maximum allowed time per user and # while the owning user is running an execution, the command execution stops and log output is incomplete. diff --git a/lib/runner/event_loop.rb b/lib/runner/event_loop.rb new file mode 100644 index 00000000..ff30cc77 --- /dev/null +++ b/lib/runner/event_loop.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# EventLoop is an abstraction around Ruby's queue so that its usage is better +# understandable in our context. +class Runner::EventLoop + def initialize + @queue = Queue.new + end + + # wait waits until another thread calls stop on this EventLoop. + # There may only be one active wait call per loop at a time, otherwise it is not + # deterministic which one will be unblocked if stop is called. + def wait + @queue.pop + end + + # stop unblocks the currently active wait call. If there is none, the + # next call to wait will not be blocking. + def stop + @queue.push nil if @queue.empty? + end +end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index adc0089f..3da0dc75 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -51,19 +51,22 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") end - def attach_to_execution(command) + def attach_to_execution(command, event_loop) @command = command query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{@container_id}/attach/ws?#{query_params}" - EventMachine.run do - socket = Connection.new(websocket_url, self) - EventMachine.add_timer(@execution_environment.permitted_execution_time) do - socket.status = :timeout - destroy_at_management + socket = Connection.new(websocket_url, self, event_loop) + begin + Timeout.timeout(@execution_environment.permitted_execution_time) do + socket.send(command) + yield(socket) + event_loop.wait + event_loop.stop end - socket.send(command) - yield(socket) + rescue Timeout::Error + socket.close(:timeout) + destroy_at_management end end @@ -118,8 +121,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy # Assume correct termination for now and return exit code 0 # TODO: Can we use the actual exit code here? @exit_code = 0 - @status = :terminated_by_codeocean - @socket.close + close(:terminated_by_codeocean) when /#{format(@strategy.execution_environment.test_command, class_name: '.*', filename: '.*', module_name: '.*')}/ # TODO: Super dirty hack to redirect test output to stderr (remove attr_reader afterwards) @stream = 'stderr' diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 54d51aab..488d4914 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -85,12 +85,10 @@ class Runner::Strategy::Poseidon < Runner::Strategy raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end - def attach_to_execution(command) + def attach_to_execution(command, event_loop) websocket_url = execute_command(command) - EventMachine.run do - socket = Connection.new(websocket_url, self) - yield(socket) - end + socket = Connection.new(websocket_url, self, event_loop) + yield(socket) end def destroy_at_management @@ -113,7 +111,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy if websocket_url.present? return websocket_url else - raise Runner::Error::UnexpectedResponse.new('Poseidon did not send websocket url') + raise Runner::Error::UnexpectedResponse.new('Poseidon did not send a WebSocket URL') end when 400 Runner.destroy(@allocation_id) @@ -132,7 +130,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy def decode(raw_event) JSON.parse(raw_event.data) rescue JSON::ParserError => e - raise Runner::Error::UnexpectedResponse.new("The websocket message from Poseidon could not be decoded to JSON: #{e.inspect}") + raise Runner::Error::UnexpectedResponse.new("The WebSocket message from Poseidon could not be decoded to JSON: #{e.inspect}") end def encode(data) From c8e1a0bbcbe08334ac030bfb11c529189af7ab98 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 22 Jul 2021 09:09:38 +0200 Subject: [PATCH 049/156] Fix tests for Runner#attach_to_execution These tests were blocking because of the newly introduced EventLoop. The messages sent to the EventLoop are now mocked and the EventLoop isn't blocking anymore in the tests. --- spec/lib/runner/strategy/docker_container_pool_spec.rb | 3 ++- spec/lib/runner/strategy/poseidon_spec.rb | 3 ++- spec/models/runner_spec.rb | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index 26568b6f..9f75e241 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -264,7 +264,8 @@ describe Runner::Strategy::DockerContainerPool do # TODO: add tests here let(:command) { 'ls' } - let(:action) { -> { container_pool.attach_to_execution(command) } } + let(:event_loop) { Runner::EventLoop.new } + let(:action) { -> { container_pool.attach_to_execution(command, event_loop) } } let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index ef04a6f7..79c4935b 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -292,7 +292,8 @@ describe Runner::Strategy::Poseidon do # TODO: add tests here let(:command) { 'ls' } - let(:action) { -> { poseidon.attach_to_execution(command) } } + let(:event_loop) { Runner::EventLoop.new } + let(:action) { -> { poseidon.attach_to_execution(command, event_loop) } } let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } end end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 5202d403..4bea42e0 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -62,10 +62,13 @@ describe Runner do describe '#attach to execution' do let(:runner) { described_class.create } let(:command) { 'ls' } + let(:event_loop) { instance_double(Runner::EventLoop) } before do allow(strategy_class).to receive(:request_from_management).and_return(runner_id) allow(strategy_class).to receive(:new).and_return(strategy) + allow(event_loop).to receive(:wait) + allow(Runner::EventLoop).to receive(:new).and_return(event_loop) end it 'delegates to its strategy' do From 9e2cff7558d3d69517801b8bfe1e9e2d5cfe093e Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Thu, 22 Jul 2021 10:09:24 +0200 Subject: [PATCH 050/156] Attach connection errors to socket Raising the errors would crash the current thread. As this thread contains the Eventmachine, that would influence other connections as well. Attaching the errors to the connection and reading them after the connection was closed ensures that the thread stays alive while handling the errors in the main thread of the current request. --- app/models/runner.rb | 3 ++- lib/runner/connection.rb | 13 ++++----- lib/runner/strategy/docker_container_pool.rb | 1 + lib/runner/strategy/poseidon.rb | 4 ++- spec/models/runner_spec.rb | 28 ++++++++++++++++---- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 37cad548..30fa48a1 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -50,8 +50,9 @@ class Runner < ApplicationRecord # initializing its Runner::Connection with the given event loop. The Runner::Connection class ensures that # this event loop is stopped after the socket was closed. event_loop = Runner::EventLoop.new - @strategy.attach_to_execution(command, event_loop, &block) + socket = @strategy.attach_to_execution(command, event_loop, &block) event_loop.wait + raise socket.error if socket.error.present? rescue Runner::Error => e e.execution_duration = Time.zone.now - starting_time raise diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index d49c6eb8..d2cb4a21 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -10,6 +10,7 @@ class Runner::Connection BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) attr_writer :status + attr_reader :error def initialize(url, strategy, event_loop) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) @@ -73,7 +74,8 @@ class Runner::Connection if WEBSOCKET_MESSAGE_TYPES.include?(message_type) __send__("handle_#{message_type}", event) else - raise Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") + @error = Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") + close(:error) end end @@ -87,17 +89,16 @@ class Runner::Connection Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } case @status when :timeout - raise Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') + @error = Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') when :terminated_by_codeocean, :terminated_by_management @exit_callback.call @exit_code - @event_loop.stop - when :terminated_by_client - @event_loop.stop + when :terminated_by_client, :error else # :established # If the runner is killed by the DockerContainerPool after the maximum allowed time per user and # while the owning user is running an execution, the command execution stops and log output is incomplete. - raise Runner::Error::Unknown.new('Execution terminated with an unknown reason') + @error = Runner::Error::Unknown.new('Execution terminated with an unknown reason') end + @event_loop.stop end def handle_exit(event) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 3da0dc75..afa4261d 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -68,6 +68,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy socket.close(:timeout) destroy_at_management end + socket end private diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 488d4914..6a87d92b 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -89,6 +89,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy websocket_url = execute_command(command) socket = Connection.new(websocket_url, self, event_loop) yield(socket) + socket end def destroy_at_management @@ -130,7 +131,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy def decode(raw_event) JSON.parse(raw_event.data) rescue JSON::ParserError => e - raise Runner::Error::UnexpectedResponse.new("The WebSocket message from Poseidon could not be decoded to JSON: #{e.inspect}") + @error = Runner::Error::UnexpectedResponse.new("The WebSocket message from Poseidon could not be decoded to JSON: #{e.inspect}") + close(:error) end def encode(data) diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 4bea42e0..6105f819 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -63,12 +63,15 @@ describe Runner do let(:runner) { described_class.create } let(:command) { 'ls' } let(:event_loop) { instance_double(Runner::EventLoop) } + let(:connection) { instance_double(Runner::Connection) } before do allow(strategy_class).to receive(:request_from_management).and_return(runner_id) allow(strategy_class).to receive(:new).and_return(strategy) allow(event_loop).to receive(:wait) + allow(connection).to receive(:error).and_return(nil) allow(Runner::EventLoop).to receive(:new).and_return(event_loop) + allow(strategy).to receive(:attach_to_execution).and_return(connection) end it 'delegates to its strategy' do @@ -77,21 +80,36 @@ describe Runner do end it 'returns the execution time' do - allow(strategy).to receive(:attach_to_execution) starting_time = Time.zone.now execution_time = runner.attach_to_execution(command) test_time = Time.zone.now - starting_time expect(execution_time).to be_between(0.0, test_time).exclusive end - context 'when a runner error is raised' do - before { allow(strategy).to receive(:attach_to_execution).and_raise(Runner::Error) } + it 'blocks until the event loop is stopped' do + allow(event_loop).to receive(:wait) { sleep(1) } + execution_time = runner.attach_to_execution(command) + expect(execution_time).to be > 1 + end + + context 'when an error is returned' do + let(:error_message) { 'timeout' } + let(:error) { Runner::Error::ExecutionTimeout.new(error_message) } + + before { allow(connection).to receive(:error).and_return(error) } + + it 'raises the error' do + expect { runner.attach_to_execution(command) }.to raise_error do |raised_error| + expect(raised_error).to be_a(Runner::Error::ExecutionTimeout) + expect(raised_error.message).to eq(error_message) + end + end it 'attaches the execution time to the error' do starting_time = Time.zone.now - expect { runner.attach_to_execution(command) }.to raise_error do |error| + expect { runner.attach_to_execution(command) }.to raise_error do |raised_error| test_time = Time.zone.now - starting_time - expect(error.execution_duration).to be_between(0.0, test_time).exclusive + expect(raised_error.execution_duration).to be_between(0.0, test_time).exclusive end end end From e752df1b3c3d5168ebdc4eb41270f313738e27a6 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:40:50 +0200 Subject: [PATCH 051/156] Move EventMachine initialization to Runner::EventLoop --- app/models/runner.rb | 17 ----------------- lib/runner/event_loop.rb | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 30fa48a1..ee66a28b 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -41,7 +41,6 @@ class Runner < ApplicationRecord end def attach_to_execution(command, &block) - ensure_event_machine starting_time = Time.zone.now begin # As the EventMachine reactor is probably shared with other threads, we cannot use EventMachine.run with @@ -66,22 +65,6 @@ class Runner < ApplicationRecord private - # If there are multiple threads trying to connect to the WebSocket of their execution at the same time, - # the Faye WebSocket connections will use the same reactor. We therefore only need to start an EventMachine - # if there isn't a running reactor yet. - # See this StackOverflow answer: https://stackoverflow.com/a/8247947 - def ensure_event_machine - unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - event_loop = Runner::EventLoop.new - Thread.new do - EventMachine.run { event_loop.stop } - ensure - ActiveRecord::Base.connection_pool.release_connection - end - event_loop.wait - end - end - def request_id request_new_id if runner_id.blank? end diff --git a/lib/runner/event_loop.rb b/lib/runner/event_loop.rb index ff30cc77..037f08d0 100644 --- a/lib/runner/event_loop.rb +++ b/lib/runner/event_loop.rb @@ -5,6 +5,7 @@ class Runner::EventLoop def initialize @queue = Queue.new + ensure_event_machine end # wait waits until another thread calls stop on this EventLoop. @@ -19,4 +20,22 @@ class Runner::EventLoop def stop @queue.push nil if @queue.empty? end + + private + + # If there are multiple threads trying to connect to the WebSocket of their execution at the same time, + # the Faye WebSocket connections will use the same reactor. We therefore only need to start an EventMachine + # if there isn't a running reactor yet. + # See this StackOverflow answer: https://stackoverflow.com/a/8247947 + def ensure_event_machine + unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? + queue = Queue.new + Thread.new do + EventMachine.run { queue.push nil } + ensure + ActiveRecord::Base.connection_pool.release_connection + end + queue.pop + end + end end From 30603cb7ab494906cf0d74982c502a0c19c0eb7e Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 13 Sep 2021 12:49:56 +0200 Subject: [PATCH 052/156] Generalize method and constant names for runner management --- .../execution_environments_controller.rb | 24 ++++---- app/models/execution_environment.rb | 17 ------ app/models/runner.rb | 11 ++-- app/policies/execution_environment_policy.rb | 2 +- .../execution_environments/index.html.slim | 4 +- config/code_ocean.yml.ci | 1 + config/code_ocean.yml.example | 1 + config/locales/de.yml | 2 +- config/locales/en.yml | 2 +- config/routes.rb | 2 +- lib/runner/strategy.rb | 10 +++- lib/runner/strategy/docker_container_pool.rb | 5 ++ lib/runner/strategy/poseidon.rb | 23 ++++++-- .../execution_environments_controller_spec.rb | 31 +++++------ spec/lib/runner/strategy/poseidon_spec.rb | 55 +++++++++++++++++-- spec/models/execution_environment_spec.rb | 41 -------------- spec/models/runner_spec.rb | 16 +++++- .../execution_environment_policy_spec.rb | 2 +- 18 files changed, 139 insertions(+), 110 deletions(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index e9a0ba0c..648f7140 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -3,8 +3,6 @@ class ExecutionEnvironmentsController < ApplicationController include CommonBehavior - RUNNER_MANAGEMENT_PRESENT = CodeOcean::Config.new(:code_ocean).read[:runner_management].present? - before_action :set_docker_images, only: %i[create edit new update] before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell statistics] before_action :set_testing_framework_adapters, only: %i[create edit new update] @@ -18,7 +16,7 @@ class ExecutionEnvironmentsController < ApplicationController @execution_environment = ExecutionEnvironment.new(execution_environment_params) authorize! create_and_respond(object: @execution_environment) do - copy_execution_environment_to_poseidon + sync_to_runner_management end end @@ -160,27 +158,29 @@ class ExecutionEnvironmentsController < ApplicationController def update update_and_respond(object: @execution_environment, params: execution_environment_params) do - copy_execution_environment_to_poseidon + sync_to_runner_management end end - def synchronize_all_to_poseidon + def sync_all_to_runner_management authorize ExecutionEnvironment - return unless RUNNER_MANAGEMENT_PRESENT + return unless Runner.management_active? - success = ExecutionEnvironment.all.map(&:copy_to_poseidon).all? - if success + success = ExecutionEnvironment.all.map do |execution_environment| + Runner.strategy_class.sync_environment(execution_environment) + end + if success.all? redirect_to ExecutionEnvironment, notice: t('execution_environments.index.synchronize_all.success') else redirect_to ExecutionEnvironment, alert: t('execution_environments.index.synchronize_all.failure') end end - def copy_execution_environment_to_poseidon - unless RUNNER_MANAGEMENT_PRESENT && @execution_environment.copy_to_poseidon - t('execution_environments.form.errors.not_synced_to_poseidon') + def sync_to_runner_management + unless Runner.management_active? && Runner.strategy_class.sync_environment(@execution_environment) + t('execution_environments.form.errors.not_synced_to_runner_management') end end - private :copy_execution_environment_to_poseidon + private :sync_to_runner_management end diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 9d420715..cee38252 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -7,9 +7,6 @@ class ExecutionEnvironment < ApplicationRecord include DefaultValues VALIDATION_COMMAND = 'whoami' - RUNNER_MANAGEMENT_PRESENT = CodeOcean::Config.new(:code_ocean).read[:runner_management].present? - BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] if RUNNER_MANAGEMENT_PRESENT - HEADERS = {'Content-Type' => 'application/json'}.freeze DEFAULT_CPU_LIMIT = 20 after_initialize :set_default_values @@ -42,20 +39,6 @@ class ExecutionEnvironment < ApplicationRecord name end - def copy_to_poseidon - return false unless RUNNER_MANAGEMENT_PRESENT - - url = "#{BASE_URL}/execution-environments/#{id}" - response = Faraday.put(url, to_json, HEADERS) - return true if [201, 204].include? response.status - - Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") - false - rescue Faraday::Error => e - Rails.logger.warn("Could not create execution environment because of Faraday error: #{e.inspect}") - false - end - def to_json(*_args) { id: id, diff --git a/app/models/runner.rb b/app/models/runner.rb index ee66a28b..543452d7 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -8,14 +8,15 @@ class Runner < ApplicationRecord validates :execution_environment, :user, :runner_id, presence: true - STRATEGY_NAME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] - UNUSED_EXPIRATION_TIME = CodeOcean::Config.new(:code_ocean).read[:runner_management][:unused_runner_expiration_time].seconds - BASE_URL = CodeOcean::Config.new(:code_ocean).read[:runner_management][:url] - attr_accessor :strategy def self.strategy_class - "runner/strategy/#{STRATEGY_NAME}".camelize.constantize + strategy_name = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] + @strategy_class ||= "runner/strategy/#{strategy_name}".camelize.constantize + end + + def self.management_active? + @management_active ||= CodeOcean::Config.new(:code_ocean).read[:runner_management][:enabled] end def self.for(user, exercise) diff --git a/app/policies/execution_environment_policy.rb b/app/policies/execution_environment_policy.rb index cf134527..a83b1f29 100644 --- a/app/policies/execution_environment_policy.rb +++ b/app/policies/execution_environment_policy.rb @@ -9,7 +9,7 @@ class ExecutionEnvironmentPolicy < AdminOnlyPolicy define_method(action) { admin? || teacher? } end - def synchronize_all_to_poseidon? + def sync_all_to_runner_management? admin? end end diff --git a/app/views/execution_environments/index.html.slim b/app/views/execution_environments/index.html.slim index 9f6d47cc..dd6deb0b 100644 --- a/app/views/execution_environments/index.html.slim +++ b/app/views/execution_environments/index.html.slim @@ -1,7 +1,7 @@ h1.d-inline-block = ExecutionEnvironment.model_name.human(count: 2) -- if ExecutionEnvironment::RUNNER_MANAGEMENT_PRESENT - = button_to( { action: :synchronize_all_to_poseidon, method: :post }, { form_class: 'float-right mb-2', class: 'btn btn-success' }) +- if Runner.management_active? + = button_to( { action: :sync_all_to_runner_management, method: :post }, { form_class: 'float-right mb-2', class: 'btn btn-success' }) i.fa.fa-upload = t('execution_environments.index.synchronize_all.button') diff --git a/config/code_ocean.yml.ci b/config/code_ocean.yml.ci index 8b8d2a64..1c531f9e 100644 --- a/config/code_ocean.yml.ci +++ b/config/code_ocean.yml.ci @@ -10,6 +10,7 @@ test: prometheus_exporter: enabled: false runner_management: + enabled: true strategy: poseidon url: https://runners.example.org unused_runner_expiration_time: 180 diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 6b694123..0202bfc9 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -12,6 +12,7 @@ default: &default prometheus_exporter: enabled: false runner_management: + enabled: false strategy: poseidon url: https://runners.example.org unused_runner_expiration_time: 180 diff --git a/config/locales/de.yml b/config/locales/de.yml index abab5b43..ea5f34eb 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -284,7 +284,7 @@ de: docker_image: 'Wählen Sie ein Docker-Image aus der Liste oder fügen Sie ein neues hinzu, welches über DockerHub verfügbar ist.' exposed_ports: Während der Ausführung sind diese Ports für den Nutzer zugänglich. Die Portnummern müssen mit Komma, aber ohne Leerzeichen voneinander getrennt sein. errors: - not_synced_to_poseidon: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zu Poseidon synchronisiert. + not_synced_to_runner_management: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zum Runnermanagement synchronisiert. index: shell: Shell synchronize_all: diff --git a/config/locales/en.yml b/config/locales/en.yml index dd241006..b2995a27 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,7 +284,7 @@ en: docker_image: Pick a Docker image listed above or add a new one which is available via DockerHub. exposed_ports: During code execution these ports are accessible for the user. Port numbers must be separated by a comma but no space. errors: - not_synced_to_poseidon: The ExecutionEnvironment was created but not synced to Poseidon due to an error. + not_synced_to_runner_management: The execution environment was created but not synced to the runner management due to an error. index: shell: Shell synchronize_all: diff --git a/config/routes.rb b/config/routes.rb index 1608ac05..ca546cea 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -67,7 +67,7 @@ Rails.application.routes.draw do get :statistics end - post :synchronize_all_to_poseidon, on: :collection + post :sync_all_to_runner_management, on: :collection end post '/import_exercise' => 'exercises#import_exercise' diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index 0007caab..702311f4 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -5,7 +5,15 @@ class Runner::Strategy @execution_environment = environment end - def self.request_from_management + def self.config + raise NotImplementedError + end + + def self.sync_environment(_environment) + raise NotImplementedError + end + + def self.request_from_management(_environment) raise NotImplementedError end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index afa4261d..feb41a18 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -8,6 +8,11 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy @config ||= CodeOcean::Config.new(:docker).read(erb: true) end + def self.sync_environment(_environment) + # There is no dedicated sync mechanism yet + true + end + def self.request_from_management(environment) container_id = JSON.parse(Faraday.get("#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}").body)['id'] container_id.presence || raise(Runner::Error::NotAvailable.new("DockerContainerPool didn't return a container id")) diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 6a87d92b..f61d1748 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -10,13 +10,28 @@ class Runner::Strategy::Poseidon < Runner::Strategy end end + def self.config + @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} + end + def self.sync_environment(environment) - environment.copy_to_poseidon + url = "#{config[:url]}/execution-environments/#{environment.id}" + response = Faraday.put(url, environment.to_json, HEADERS) + return true if [201, 204].include? response.status + + Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") + false + rescue Faraday::Error => e + Rails.logger.warn("Could not create execution environment because of Faraday error: #{e.inspect}") + false end def self.request_from_management(environment) - url = "#{Runner::BASE_URL}/runners" - body = {executionEnvironmentId: environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME} + url = "#{config[:url]}/runners" + body = { + executionEnvironmentId: environment.id, + inactivityTimeout: config[:unused_runner_expiration_time].seconds, + } response = Faraday.post(url, body.to_json, HEADERS) case response.status @@ -124,7 +139,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def runner_url - "#{Runner::BASE_URL}/runners/#{@allocation_id}" + "#{self.class.config[:url]}/runners/#{@allocation_id}" end class Connection < Runner::Connection diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index fa76b4dd..73b408b1 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -8,7 +8,7 @@ describe ExecutionEnvironmentsController do before do allow(controller).to receive(:current_user).and_return(user) - allow(controller).to receive(:copy_execution_environment_to_poseidon).and_return(nil) + allow(controller).to receive(:sync_to_runner_management).and_return(nil) end describe 'POST #create' do @@ -26,8 +26,8 @@ describe ExecutionEnvironmentsController do expect { perform_request.call }.to change(ExecutionEnvironment, :count).by(1) end - it 'registers the execution environment with Poseidon' do - expect(controller).to have_received(:copy_execution_environment_to_poseidon) + it 'registers the execution environment with the runner management' do + expect(controller).to have_received(:sync_to_runner_management) end expect_redirect(ExecutionEnvironment.last) @@ -40,8 +40,8 @@ describe ExecutionEnvironmentsController do expect_status(200) expect_template(:new) - it 'does not register the execution environment with Poseidon' do - expect(controller).not_to have_received(:copy_execution_environment_to_poseidon) + it 'does not register the execution environment with the runner management' do + expect(controller).not_to have_received(:sync_to_runner_management) end end end @@ -167,7 +167,7 @@ describe ExecutionEnvironmentsController do context 'with a valid execution environment' do before do allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) - allow(controller).to receive(:copy_execution_environment_to_poseidon).and_return(nil) + allow(controller).to receive(:sync_to_runner_management).and_return(nil) put :update, params: {execution_environment: FactoryBot.attributes_for(:ruby), id: execution_environment.id} end @@ -175,8 +175,8 @@ describe ExecutionEnvironmentsController do expect_assigns(execution_environment: ExecutionEnvironment) expect_redirect(:execution_environment) - it 'updates the execution environment at Poseidon' do - expect(controller).to have_received(:copy_execution_environment_to_poseidon) + it 'updates the execution environment at the runner management' do + expect(controller).to have_received(:sync_to_runner_management) end end @@ -187,25 +187,24 @@ describe ExecutionEnvironmentsController do expect_status(200) expect_template(:edit) - it 'does not update the execution environment at Poseidon' do - expect(controller).not_to have_received(:copy_execution_environment_to_poseidon) + it 'does not update the execution environment at the runner management' do + expect(controller).not_to have_received(:sync_to_runner_management) end end end - describe '#synchronize_all_to_poseidon' do + describe '#sync_all_to_runner_management' do let(:execution_environments) { FactoryBot.build_list(:ruby, 3) } - it 'copies all execution environments to Poseidon' do + it 'copies all execution environments to the runner management' do allow(ExecutionEnvironment).to receive(:all).and_return(execution_environments) execution_environments.each do |execution_environment| - allow(execution_environment).to receive(:copy_to_poseidon).and_return(true) + allow(Runner::Strategy::Poseidon).to receive(:sync_environment).with(execution_environment).and_return(true) + expect(Runner::Strategy::Poseidon).to receive(:sync_environment).with(execution_environment).once end - post :synchronize_all_to_poseidon - - expect(execution_environments).to all(have_received(:copy_to_poseidon).once) + post :sync_all_to_runner_management end end end diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 79c4935b..52b12389 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -114,13 +114,58 @@ describe Runner::Strategy::Poseidon do end end + describe '::sync_environment' do + let(:action) { -> { described_class.sync_environment(execution_environment) } } + let(:execution_environment) { FactoryBot.create(:ruby) } + + it 'makes the correct request to Poseidon' do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: 201)) + action.call + expect(Faraday).to have_received(:put) do |url, body, headers| + expect(url).to match(%r{execution-environments/#{execution_environment.id}\z}) + expect(body).to eq(execution_environment.to_json) + expect(headers).to include({'Content-Type' => 'application/json'}) + end + end + + shared_examples 'returns true when the api request was successful' do |status| + it "returns true on status #{status}" do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + expect(action.call).to be_truthy + end + end + + shared_examples 'returns false when the api request failed' do |status| + it "returns false on status #{status}" do + allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + expect(action.call).to be_falsey + end + end + + [201, 204].each do |status| + include_examples 'returns true when the api request was successful', status + end + + [400, 500].each do |status| + include_examples 'returns false when the api request failed', status + end + + it 'returns false if Faraday raises an error' do + allow(Faraday).to receive(:put).and_raise(Faraday::TimeoutError) + expect(action.call).to be_falsey + end + end + describe '::request_from_management' do let(:action) { -> { described_class.request_from_management(execution_environment) } } let!(:request_runner_stub) do WebMock - .stub_request(:post, "#{Runner::BASE_URL}/runners") + .stub_request(:post, "#{described_class.config[:url]}/runners") .with( - body: {executionEnvironmentId: execution_environment.id, inactivityTimeout: Runner::UNUSED_EXPIRATION_TIME}, + body: { + executionEnvironmentId: execution_environment.id, + inactivityTimeout: described_class.config[:unused_runner_expiration_time].seconds, + }, headers: {'Content-Type' => 'application/json'} ) .to_return(body: response_body, status: response_status) @@ -181,7 +226,7 @@ describe Runner::Strategy::Poseidon do let(:websocket_url) { 'ws://ws.example.com/path/to/websocket' } let!(:execute_command_stub) do WebMock - .stub_request(:post, "#{Runner::BASE_URL}/runners/#{runner_id}/execute") + .stub_request(:post, "#{described_class.config[:url]}/runners/#{runner_id}/execute") .with( body: {command: command, timeLimit: execution_environment.permitted_execution_time}, headers: {'Content-Type' => 'application/json'} @@ -235,7 +280,7 @@ describe Runner::Strategy::Poseidon do let(:action) { -> { poseidon.destroy_at_management } } let!(:destroy_stub) do WebMock - .stub_request(:delete, "#{Runner::BASE_URL}/runners/#{runner_id}") + .stub_request(:delete, "#{described_class.config[:url]}/runners/#{runner_id}") .to_return(body: response_body, status: response_status) end @@ -262,7 +307,7 @@ describe Runner::Strategy::Poseidon do let(:encoded_file_content) { Base64.strict_encode64(file.content) } let!(:copy_files_stub) do WebMock - .stub_request(:patch, "#{Runner::BASE_URL}/runners/#{runner_id}/files") + .stub_request(:patch, "#{described_class.config[:url]}/runners/#{runner_id}/files") .with( body: {copy: [{path: file.filepath, content: encoded_file_content}]}, headers: {'Content-Type' => 'application/json'} diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index 6441b896..d7b057ab 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -192,45 +192,4 @@ describe ExecutionEnvironment do end end end - - describe '#copy_to_poseidon' do - let(:execution_environment) { FactoryBot.create(:ruby) } - - it 'makes the correct request to Poseidon' do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: 201)) - execution_environment.copy_to_poseidon - expect(Faraday).to have_received(:put) do |url, body, headers| - expect(url).to match(%r{execution-environments/#{execution_environment.id}\z}) - expect(body).to eq(execution_environment.to_json) - expect(headers).to include({'Content-Type' => 'application/json'}) - end - end - - shared_examples 'returns true when the api request was successful' do |status| - it "returns true on status #{status}" do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) - expect(execution_environment.copy_to_poseidon).to be_truthy - end - end - - shared_examples 'returns false when the api request failed' do |status| - it "returns false on status #{status}" do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) - expect(execution_environment.copy_to_poseidon).to be_falsey - end - end - - [201, 204].each do |status| - include_examples 'returns true when the api request was successful', status - end - - [400, 500].each do |status| - include_examples 'returns false when the api request failed', status - end - - it 'returns false if Faraday raises an error' do - allow(Faraday).to receive(:put).and_raise(Faraday::TimeoutError) - expect(execution_environment.copy_to_poseidon).to be_falsey - end - end end diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 6105f819..0a496b24 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -30,8 +30,20 @@ describe Runner do describe '::strategy_class' do shared_examples 'uses the strategy defined in the constant' do |strategy, strategy_class| + let(:codeocean_config) { instance_double(CodeOcean::Config) } + let(:runner_management_config) { {runner_management: {enabled: true, strategy: strategy}} } + + before do + allow(CodeOcean::Config).to receive(:new).with(:code_ocean).and_return(codeocean_config) + allow(codeocean_config).to receive(:read).and_return(runner_management_config) + end + + after do + # Reset the memorized helper + described_class.remove_instance_variable :@strategy_class + end + it "uses #{strategy_class} as strategy class for constant #{strategy}" do - stub_const('Runner::STRATEGY_NAME', strategy) expect(described_class.strategy_class).to eq(strategy_class) end end @@ -41,7 +53,7 @@ describe Runner do docker_container_pool: Runner::Strategy::DockerContainerPool, } available_strategies.each do |strategy, strategy_class| - include_examples 'uses the strategy defined in the constant', strategy, strategy_class + it_behaves_like 'uses the strategy defined in the constant', strategy, strategy_class end end diff --git a/spec/policies/execution_environment_policy_spec.rb b/spec/policies/execution_environment_policy_spec.rb index 9d0b2539..588f76db 100644 --- a/spec/policies/execution_environment_policy_spec.rb +++ b/spec/policies/execution_environment_policy_spec.rb @@ -59,7 +59,7 @@ describe ExecutionEnvironmentPolicy do end end - permissions(:synchronize_all_to_poseidon?) do + permissions(:sync_all_to_runner_management?) do it 'grants access to the admin' do expect(policy).to permit(FactoryBot.build(:admin)) end From 1bf92d8c902fd7302e39037ce2a77dea2ed7e799 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 14 Sep 2021 17:01:46 +0300 Subject: [PATCH 053/156] Fix sentry error capturing in submissions controller --- app/controllers/submissions_controller.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index b2e0bbe9..fc340454 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -152,12 +152,14 @@ class SubmissionsController < ApplicationController else Rails.logger.info("Unknown command from client: #{event[:cmd]}") end - rescue JSON::ParserError + rescue JSON::ParserError => e Rails.logger.info("Data received from client is not valid json: #{data.inspect}") Sentry.set_extras(data: data) - rescue TypeError + Sentry.capture_exception(e) + rescue TypeError => e Rails.logger.info("JSON data received from client cannot be parsed as hash: #{data.inspect}") Sentry.set_extras(data: data) + Sentry.capture_exception(e) end end From f77e6d9df8a58bdba18a04527ae75be5ae0757e6 Mon Sep 17 00:00:00 2001 From: Felix Auringer <48409110+felixauringer@users.noreply.github.com> Date: Tue, 14 Sep 2021 18:02:22 +0300 Subject: [PATCH 054/156] Simplify code in runner model --- app/models/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 543452d7..861ba5ce 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -20,7 +20,7 @@ class Runner < ApplicationRecord end def self.for(user, exercise) - execution_environment = ExecutionEnvironment.find(exercise.execution_environment_id) + execution_environment = exercise.execution_environment runner = find_by(user: user, execution_environment: execution_environment) if runner.nil? From 5037a73f368e88033dcef229b155e9e6cea5fa98 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 11:47:00 +0200 Subject: [PATCH 055/156] Mock runner management settings for spec --- .../execution_environments_controller_spec.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index 73b408b1..eacb0fb5 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -196,6 +196,19 @@ describe ExecutionEnvironmentsController do describe '#sync_all_to_runner_management' do let(:execution_environments) { FactoryBot.build_list(:ruby, 3) } + let(:codeocean_config) { instance_double(CodeOcean::Config) } + let(:runner_management_config) { {runner_management: {enabled: true, strategy: :poseidon}} } + + before do + allow(CodeOcean::Config).to receive(:new).with(:code_ocean).and_return(codeocean_config) + allow(codeocean_config).to receive(:read).and_return(runner_management_config) + end + + after do + # Reset the memorized helper + Runner.remove_instance_variable :@strategy_class + end + it 'copies all execution environments to the runner management' do allow(ExecutionEnvironment).to receive(:all).and_return(execution_environments) From 4ad898ad8b4c18e41eaf47b410785723422d0e9c Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 12:04:14 +0200 Subject: [PATCH 056/156] Remove set_docker_client method from submissions_controller.rb --- app/controllers/submissions_controller.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index fc340454..be01f550 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -270,11 +270,6 @@ class SubmissionsController < ApplicationController end end - # def set_docker_client - # @docker_client = DockerClient.new(execution_environment: @submission.execution_environment) - # end - # private :set_docker_client - def set_file @file = @files.detect {|file| file.name_with_extension == sanitize_filename } head :not_found unless @file From fd9e243064878670e7f90bca3af2273f17b5bb01 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 12:26:29 +0200 Subject: [PATCH 057/156] Disable DCP if other strategy class is chosen --- app/views/admin/dashboard/show.html.slim | 4 ++-- lib/docker_client.rb | 6 +++--- lib/docker_container_pool.rb | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/views/admin/dashboard/show.html.slim b/app/views/admin/dashboard/show.html.slim index f6766d80..a4fd9102 100644 --- a/app/views/admin/dashboard/show.html.slim +++ b/app/views/admin/dashboard/show.html.slim @@ -13,14 +13,14 @@ div.mb-4 = "CodeOcean Release:" pre = Sentry.configuration.release -- if DockerContainerPool.config[:active] +- if DockerContainerPool.active? div.mb-4 = "DockerContainerPool Release:" pre = DockerContainerPool.dump_info['release'] h2 Docker -- if DockerContainerPool.config[:active] +- if DockerContainerPool.active? h3 = t('.current') .table-responsive table.table diff --git a/lib/docker_client.rb b/lib/docker_client.rb index 807f2725..cd48721a 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -230,7 +230,7 @@ class DockerClient Rails.logger.info("destroying container #{container}") # Checks only if container assignment is not nil and not whether the container itself is still present. - if container && !DockerContainerPool.config[:active] + if container && !DockerContainerPool.active? container.kill container.port_bindings.each_value {|port| PortPool.release(port) } begin @@ -355,7 +355,7 @@ container_execution_time: nil} exit_thread_if_alive @socket.close # if we use pooling and recylce the containers, put it back. otherwise, destroy it. - if DockerContainerPool.config[:active] && RECYCLE_CONTAINERS + if DockerContainerPool.active? && RECYCLE_CONTAINERS self.class.return_container(container, @execution_environment) else @@ -493,7 +493,7 @@ container_execution_time: nil} end # if we use pooling and recylce the containers, put it back. otherwise, destroy it. - if DockerContainerPool.config[:active] && RECYCLE_CONTAINERS + if DockerContainerPool.active? && RECYCLE_CONTAINERS self.class.return_container(container, @execution_environment) else self.class.destroy_container(container) diff --git a/lib/docker_container_pool.rb b/lib/docker_container_pool.rb index a290c82c..61de69aa 100644 --- a/lib/docker_container_pool.rb +++ b/lib/docker_container_pool.rb @@ -9,6 +9,11 @@ require 'concurrent/timer_task' # dump_info and quantities are still in use. class DockerContainerPool + def self.active? + # TODO: Refactor config and merge with code_ocean.yml + config[:active] && Runner.management_active? && Runner.strategy_class == Runner::Strategy::DockerContainerPool + end + def self.config # TODO: Why erb? @config ||= CodeOcean::Config.new(:docker).read(erb: true)[:pool] @@ -39,7 +44,7 @@ class DockerContainerPool def self.get_container(execution_environment) # if pooling is active, do pooling, otherwise just create an container and return it - if config[:active] + if active? begin container_id = JSON.parse(Faraday.get("#{config[:location]}/docker_container_pool/get_container/#{execution_environment.id}").body)['id'] Docker::Container.get(container_id) if container_id.present? From 13c378b980bd59622e640c4266d77b1eeb3a126c Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 14:50:34 +0200 Subject: [PATCH 058/156] Remove concurrent ruby gem no longer needed --- Gemfile | 1 - app/models/submission.rb | 2 -- lib/docker_client.rb | 1 - lib/docker_container_pool.rb | 3 --- 4 files changed, 7 deletions(-) diff --git a/Gemfile b/Gemfile index c75764ec..960e1146 100644 --- a/Gemfile +++ b/Gemfile @@ -46,7 +46,6 @@ gem 'webpacker' gem 'whenever', require: false # Error Tracing -gem 'concurrent-ruby' gem 'mnemosyne-ruby' gem 'newrelic_rpm' gem 'sentry-rails' diff --git a/app/models/submission.rb b/app/models/submission.rb index b9442084..6bb1cee6 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -5,8 +5,6 @@ class Submission < ApplicationRecord include Creation include ActionCableHelper - require 'concurrent/future' - CAUSES = %w[assess download file render run save submit test autosave requestComments remoteAssess remoteSubmit].freeze FILENAME_URL_PLACEHOLDER = '{filename}' diff --git a/lib/docker_client.rb b/lib/docker_client.rb index cd48721a..313bb226 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'concurrent' require 'pathname' class DockerClient diff --git a/lib/docker_container_pool.rb b/lib/docker_container_pool.rb index 61de69aa..a89bd461 100644 --- a/lib/docker_container_pool.rb +++ b/lib/docker_container_pool.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'concurrent/future' -require 'concurrent/timer_task' - # get_container, destroy_container was moved to lib/runner/strategy/docker_container_pool.rb. # return_container is not used anymore because runners are not shared between users anymore. # create_container is done by the DockerContainerPool. From 0c22e1392a39d21d3c72139a7069b7165d2e284f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 15:12:07 +0200 Subject: [PATCH 059/156] Remove outdated mnemosyne traces --- app/models/submission.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/models/submission.rb b/app/models/submission.rb index 6bb1cee6..ee277646 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -216,8 +216,6 @@ class Submission < ApplicationRecord end def score_file(output, file) - # Mnemosyne.trace 'custom.codeocean.collect_test_results', meta: { submission: id } do - # Mnemosyne.trace 'custom.codeocean.collect_test_results_block', meta: { file: file.id, submission: id } do assessor = Assessor.new(execution_environment: execution_environment) assessment = assessor.assess(output) passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score]).positive?) From 0cc1c7a396ca1f2a7bdbd6c3f90e81480ab5410a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 15:21:08 +0200 Subject: [PATCH 060/156] Combine runner waiting_time migrations --- db/migrate/20210519134938_create_runners.rb | 1 - .../20210611101330_remove_waiting_time_from_runners.rb | 7 ------- db/schema.rb | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 db/migrate/20210611101330_remove_waiting_time_from_runners.rb diff --git a/db/migrate/20210519134938_create_runners.rb b/db/migrate/20210519134938_create_runners.rb index 91ea4814..c8a1cd67 100644 --- a/db/migrate/20210519134938_create_runners.rb +++ b/db/migrate/20210519134938_create_runners.rb @@ -6,7 +6,6 @@ class CreateRunners < ActiveRecord::Migration[6.1] t.string :runner_id t.references :execution_environment t.references :user, polymorphic: true - t.float :waiting_time t.timestamps end diff --git a/db/migrate/20210611101330_remove_waiting_time_from_runners.rb b/db/migrate/20210611101330_remove_waiting_time_from_runners.rb deleted file mode 100644 index 5a81c62e..00000000 --- a/db/migrate/20210611101330_remove_waiting_time_from_runners.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class RemoveWaitingTimeFromRunners < ActiveRecord::Migration[6.1] - def change - remove_column :runners, :waiting_time - end -end diff --git a/db/schema.rb b/db/schema.rb index 2772b0ca..87fe8d77 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_06_11_101330) do +ActiveRecord::Schema.define(version: 2021_06_02_071834) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" From 8bd9a93944c2d1a116a0ce14db68239d67463c84 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 15:21:41 +0200 Subject: [PATCH 061/156] Add NOT NULL constraint on cpu_limit --- .../20210601095654_add_cpu_limit_to_execution_environment.rb | 2 +- db/schema.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb b/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb index c74aa69a..cdd4c2fa 100644 --- a/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb +++ b/db/migrate/20210601095654_add_cpu_limit_to_execution_environment.rb @@ -2,6 +2,6 @@ class AddCpuLimitToExecutionEnvironment < ActiveRecord::Migration[6.1] def change - add_column :execution_environments, :cpu_limit, :integer, default: 20 + add_column :execution_environments, :cpu_limit, :integer, null: false, default: 20 end end diff --git a/db/schema.rb b/db/schema.rb index 87fe8d77..9a53d05b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -112,7 +112,7 @@ ActiveRecord::Schema.define(version: 2021_06_02_071834) do t.integer "file_type_id" t.integer "memory_limit" t.boolean "network_enabled" - t.integer "cpu_limit", default: 20 + t.integer "cpu_limit", default: 20, null: false end create_table "exercise_collection_items", id: :serial, force: :cascade do |t| From ee1751debfd57a90257c8e694f7866c0268ca591 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 15:28:04 +0200 Subject: [PATCH 062/156] Fix rubocop offenses --- spec/lib/runner/strategy/poseidon_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index 52b12389..f8f60ac3 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -163,7 +163,7 @@ describe Runner::Strategy::Poseidon do .stub_request(:post, "#{described_class.config[:url]}/runners") .with( body: { - executionEnvironmentId: execution_environment.id, + executionEnvironmentId: execution_environment.id, inactivityTimeout: described_class.config[:unused_runner_expiration_time].seconds, }, headers: {'Content-Type' => 'application/json'} From cc17736bf5fc702c007d7cc1e263b35df68fd3c4 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 16:12:10 +0200 Subject: [PATCH 063/156] Add CPU limit to Execution Environment index --- app/views/execution_environments/index.html.slim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/execution_environments/index.html.slim b/app/views/execution_environments/index.html.slim index dd6deb0b..a78fd684 100644 --- a/app/views/execution_environments/index.html.slim +++ b/app/views/execution_environments/index.html.slim @@ -13,6 +13,7 @@ h1.d-inline-block = ExecutionEnvironment.model_name.human(count: 2) th = t('activerecord.attributes.execution_environment.user') th = t('activerecord.attributes.execution_environment.pool_size') th = t('activerecord.attributes.execution_environment.memory_limit') + th = t('activerecord.attributes.execution_environment.cpu_limit') th = t('activerecord.attributes.execution_environment.network_enabled') th = t('activerecord.attributes.execution_environment.permitted_execution_time') th colspan=5 = t('shared.actions') @@ -23,6 +24,7 @@ h1.d-inline-block = ExecutionEnvironment.model_name.human(count: 2) td = link_to_if(policy(execution_environment.author).show?, execution_environment.author, execution_environment.author) td = execution_environment.pool_size td = execution_environment.memory_limit + td = execution_environment.cpu_limit td = symbol_for(execution_environment.network_enabled) td = execution_environment.permitted_execution_time td = link_to(t('shared.show'), execution_environment) if policy(execution_environment).show? From 325720bd3b9e9f1c154025cb0aff0c7c8eb3ea02 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 23:59:38 +0200 Subject: [PATCH 064/156] Improve documentation in Runner::Connection --- lib/runner/connection.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index d2cb4a21..7ff5a82d 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -31,18 +31,21 @@ class Runner::Connection @exit_code = 1 end + # Register a callback based on the WebSocket connection state def on(event, &block) return unless EVENTS.include? event instance_variable_set(:"@#{event}_callback", block) end + # Send arbitrary data in the WebSocket connection def send(raw_data) encoded_message = encode(raw_data) Rails.logger.debug { "#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}" } @socket.send(encoded_message) end + # Close the WebSocket connection def close(status) return unless active? @@ -50,6 +53,7 @@ class Runner::Connection @socket.close end + # Check if the WebSocket connection is currently established def active? @status == :established end @@ -64,6 +68,10 @@ class Runner::Connection raise NotImplementedError end + # === WebSocket Callbacks + # These callbacks are executed based on events indicated by Faye WebSockets and are + # independent of the JSON specification that is used within the WebSocket once established. + def on_message(raw_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } event = decode(raw_event) @@ -101,6 +109,14 @@ class Runner::Connection @event_loop.stop end + # === Message Handlers + # Each message type indicated by the +type+ attribute in the JSON + # sent be the runner management has a dedicated method. + # Context:: All registered handlers are executed in the scope of + # the bindings they had where they were registered. + # Information not stored in the binding, such as the + # locale or call stack are not available during execution! + def handle_exit(event) @status = :terminated_by_management @exit_code = event[:data] From 6c5a5226b8d64b0ec57e4866ef52ae02f3db2b69 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 19 Sep 2021 23:59:53 +0200 Subject: [PATCH 065/156] Preserve locale during Runner::Connections --- lib/runner/connection.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 7ff5a82d..659d769f 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -12,16 +12,20 @@ class Runner::Connection attr_writer :status attr_reader :error - def initialize(url, strategy, event_loop) + def initialize(url, strategy, event_loop, locale = I18n.locale) @socket = Faye::WebSocket::Client.new(url, [], ping: 5) @strategy = strategy @status = :established @event_loop = event_loop + @locale = locale # For every event type of Faye WebSockets, the corresponding # RunnerConnection method starting with `on_` is called. %i[open message error close].each do |event_type| - @socket.on(event_type) {|event| __send__(:"on_#{event_type}", event) } + @socket.on(event_type) do |event| + # The initial locale when establishing the connection is used for all callbacks + I18n.with_locale(@locale) { __send__(:"on_#{event_type}", event) } + end end # This registers empty default callbacks. From 09b672eb08eb0949111e125ad885e97f6cdb6f5b Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 20 Sep 2021 11:30:14 +0200 Subject: [PATCH 066/156] DCP strategy: Use stdout for most test results --- lib/py_unit_adapter.rb | 1 + lib/runner/strategy/docker_container_pool.rb | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/py_unit_adapter.rb b/lib/py_unit_adapter.rb index 88f43758..8b3b2093 100644 --- a/lib/py_unit_adapter.rb +++ b/lib/py_unit_adapter.rb @@ -11,6 +11,7 @@ class PyUnitAdapter < TestingFrameworkAdapter end def parse_output(output) + # PyUnit is expected to print test results on Stderr! count = COUNT_REGEXP.match(output[:stderr]).captures.first.to_i failures_matches = FAILURES_REGEXP.match(output[:stderr]) failed = failures_matches ? failures_matches.captures.try(:first).to_i : 0 diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index feb41a18..bddf75af 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Runner::Strategy::DockerContainerPool < Runner::Strategy - attr_reader :container_id, :command, :execution_environment + attr_reader :container_id, :command def self.config # Since the docker configuration file contains code that must be executed, we use ERB templating. @@ -128,9 +128,14 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy # TODO: Can we use the actual exit code here? @exit_code = 0 close(:terminated_by_codeocean) - when /#{format(@strategy.execution_environment.test_command, class_name: '.*', filename: '.*', module_name: '.*')}/ - # TODO: Super dirty hack to redirect test output to stderr (remove attr_reader afterwards) + when /python3.*-m\s*unittest/ + # TODO: Super dirty hack to redirect test output to stderr + # This is only required for Python and the unittest module but must not be used with PyLint @stream = 'stderr' + when /\*\*\*\*\*\*\*\*\*\*\*\*\* Module/ + # Identification of PyLint output, change stream back to stdout and return event + @stream = 'stdout' + {'type' => @stream, 'data' => raw_event.data} when /#{@strategy.command}/ when /bash: cmd:canvasevent: command not found/ else From 44395b7792f7272e73830468fbb71dbee67e7dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= <22845248+mpass99@users.noreply.github.com> Date: Wed, 29 Sep 2021 14:25:26 +0200 Subject: [PATCH 067/156] Add ca file option for requests to Poseidon --- Gemfile.lock | 1 - config/code_ocean.yml.example | 1 + lib/runner/strategy/poseidon.rb | 15 ++++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f0efcde2..ba6772b4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -556,7 +556,6 @@ DEPENDENCIES capybara carrierwave charlock_holmes - concurrent-ruby database_cleaner docker-api eventmachine diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 0202bfc9..bd2cdf92 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -15,6 +15,7 @@ default: &default enabled: false strategy: poseidon url: https://runners.example.org + ca_file: /example/certificates/ca.crt unused_runner_expiration_time: 180 development: diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index f61d1748..aab92803 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -16,7 +16,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy def self.sync_environment(environment) url = "#{config[:url]}/execution-environments/#{environment.id}" - response = Faraday.put(url, environment.to_json, HEADERS) + connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} + response = connection.put url, environment.to_json, HEADERS return true if [201, 204].include? response.status Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") @@ -32,7 +33,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy executionEnvironmentId: environment.id, inactivityTimeout: config[:unused_runner_expiration_time].seconds, } - response = Faraday.post(url, body.to_json, HEADERS) + connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} + response = connection.post url, body.to_json, HEADERS case response.status when 200 @@ -91,7 +93,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end url = "#{runner_url}/files" body = {copy: copy} - response = Faraday.patch(url, body.to_json, HEADERS) + connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} + response = connection.patch url, body.to_json, HEADERS return if response.status == 204 Runner.destroy(@allocation_id) if response.status == 400 @@ -108,7 +111,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def destroy_at_management - response = Faraday.delete runner_url + connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} + response = connection.delete runner_url, nil, HEADERS self.class.handle_error response unless response.status == 204 rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") @@ -119,7 +123,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy def execute_command(command) url = "#{runner_url}/execute" body = {command: command, timeLimit: @execution_environment.permitted_execution_time} - response = Faraday.post(url, body.to_json, HEADERS) + connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} + response = connection.post url, body.to_json, HEADERS case response.status when 200 response_body = self.class.parse response From b51a45e9b115d1cbb21ac87e29f9881fe9b51ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= <22845248+mpass99@users.noreply.github.com> Date: Thu, 30 Sep 2021 13:17:50 +0200 Subject: [PATCH 068/156] Add token header option for requests to Poseidon --- config/code_ocean.yml.example | 1 + lib/runner/strategy/poseidon.rb | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index bd2cdf92..712db04a 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -16,6 +16,7 @@ default: &default strategy: poseidon url: https://runners.example.org ca_file: /example/certificates/ca.crt + token: SECRET unused_runner_expiration_time: 180 development: diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index aab92803..90699400 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class Runner::Strategy::Poseidon < Runner::Strategy - HEADERS = {'Content-Type' => 'application/json'}.freeze ERRORS = %w[NOMAD_UNREACHABLE NOMAD_OVERLOAD NOMAD_INTERNAL_SERVER_ERROR UNKNOWN].freeze ERRORS.each do |error| @@ -14,10 +13,14 @@ class Runner::Strategy::Poseidon < Runner::Strategy @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} end + def self.headers + @headers ||= {'Content-Type' => 'application/json', 'Poseidon-Token' => config[:token]} + end + def self.sync_environment(environment) url = "#{config[:url]}/execution-environments/#{environment.id}" connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} - response = connection.put url, environment.to_json, HEADERS + response = connection.put url, environment.to_json, headers return true if [201, 204].include? response.status Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") @@ -34,7 +37,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy inactivityTimeout: config[:unused_runner_expiration_time].seconds, } connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} - response = connection.post url, body.to_json, HEADERS + response = connection.post url, body.to_json, headers case response.status when 200 @@ -94,7 +97,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy url = "#{runner_url}/files" body = {copy: copy} connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.patch url, body.to_json, HEADERS + response = connection.patch url, body.to_json, self.class.headers return if response.status == 204 Runner.destroy(@allocation_id) if response.status == 400 @@ -112,7 +115,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy def destroy_at_management connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.delete runner_url, nil, HEADERS + response = connection.delete runner_url, nil, self.class.headers self.class.handle_error response unless response.status == 204 rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") @@ -124,7 +127,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy url = "#{runner_url}/execute" body = {command: command, timeLimit: @execution_environment.permitted_execution_time} connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.post url, body.to_json, HEADERS + response = connection.post url, body.to_json, self.class.headers case response.status when 200 response_body = self.class.parse response From 3fa6ba6c7215fb9ecba4a062010ea2719509e7fa Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 11:20:08 +0200 Subject: [PATCH 069/156] Use instance_double for Poseidon Strategy specs --- spec/lib/runner/strategy/poseidon_spec.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index f8f60ac3..b87bf462 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -108,7 +108,9 @@ describe Runner::Strategy::Poseidon do let(:response_status) { -1 } it 'raises an error' do - %i[post patch delete].each {|message| allow(Faraday).to receive(message).and_raise(Faraday::TimeoutError) } + faraday_connection = instance_double 'Faraday::Connection' + allow(Faraday).to receive(:new).and_return(faraday_connection) + %i[post patch delete].each {|message| allow(faraday_connection).to receive(message).and_raise(Faraday::TimeoutError) } expect { action.call }.to raise_error(Runner::Error::FaradayError) end end @@ -119,9 +121,11 @@ describe Runner::Strategy::Poseidon do let(:execution_environment) { FactoryBot.create(:ruby) } it 'makes the correct request to Poseidon' do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: 201)) + faraday_connection = instance_double 'Faraday::Connection' + allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: 201)) action.call - expect(Faraday).to have_received(:put) do |url, body, headers| + expect(faraday_connection).to have_received(:put) do |url, body, headers| expect(url).to match(%r{execution-environments/#{execution_environment.id}\z}) expect(body).to eq(execution_environment.to_json) expect(headers).to include({'Content-Type' => 'application/json'}) @@ -130,14 +134,18 @@ describe Runner::Strategy::Poseidon do shared_examples 'returns true when the api request was successful' do |status| it "returns true on status #{status}" do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + faraday_connection = instance_double 'Faraday::Connection' + allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: status)) expect(action.call).to be_truthy end end shared_examples 'returns false when the api request failed' do |status| it "returns false on status #{status}" do - allow(Faraday).to receive(:put).and_return(Faraday::Response.new(status: status)) + faraday_connection = instance_double 'Faraday::Connection' + allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: status)) expect(action.call).to be_falsey end end @@ -151,7 +159,9 @@ describe Runner::Strategy::Poseidon do end it 'returns false if Faraday raises an error' do - allow(Faraday).to receive(:put).and_raise(Faraday::TimeoutError) + faraday_connection = instance_double 'Faraday::Connection' + allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(faraday_connection).to receive(:put).and_raise(Faraday::TimeoutError) expect(action.call).to be_falsey end end From 82cab390ad8bbb0dae44a39df83dcc70ad150e06 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 12:11:26 +0200 Subject: [PATCH 070/156] Remove outdated run method from turtle.js --- app/assets/javascripts/turtle.js | 79 -------------------------------- 1 file changed, 79 deletions(-) diff --git a/app/assets/javascripts/turtle.js b/app/assets/javascripts/turtle.js index 1563d299..d3066e54 100644 --- a/app/assets/javascripts/turtle.js +++ b/app/assets/javascripts/turtle.js @@ -205,82 +205,3 @@ Turtle.prototype.css = function (key, value) { this.canvas.css(key, value); } }; - -function run(launchmsg) { - var i, turtlescreen, msg, result, cmd; - $('#assess').empty(); - - turtlescreen = new Turtle(); - - output = $('#output'); - output.empty(); - if (typeof pipeurl === 'undefined') { - if (wp_port === '443') { - pipeurl = 'wss://'+wp_hostname+'/pipe'; - } else { - pipeurl = 'ws://'+wp_hostname+':'+wp_port+'/pipe'; - } - } - saveFile(); - output.pipe = new WebSocket(pipeurl); - output.pipe.onopen = function () { - output.pipe.send(JSON.stringify(launchmsg)); - }; - output.pipe.onmessage = function (response) { - msg = JSON.parse(response.data); - if (msg.cmd === 'input') { - output.inputelem = $('',{'size':40}); - submit = $('',{'type':'submit'}); - submit.click(function (){ - text = output.inputelem.val(); - output.input.replaceWith($('', {text:text+'\n'})); - output.pipe.send(JSON.stringify({'cmd':'inputresult', - 'data':text})); - }); - output.inputelem.keydown(function(event){ - if(event.keyCode === 13){ - submit.click(); - } - }); - output.append($('', {text:msg.data})); - output.input = $('').append(output.inputelem).append(submit); - output.append(output.input); - output.inputelem.focus(); - } else if (msg.cmd === 'stop') { - if (launchmsg.cmd === 'runscript') { - if (msg.timedout) { - output.append('
Dein Programm hat zu lange gerechnet und wurde beendet.'); - } else { - output.append('
Dein Progamm wurde beendet'); - } - } - output.pipe.close(); - } else if (msg.cmd === 'passed') { - $('#assess').html("Herzlich Glückwunsch! Dein Programm funktioniert korrekt."); - } else if (msg.cmd === 'failed') { - $('#assess').html(msg.data); - } else if (msg.cmd === 'turtle') { - if (msg.action in turtlescreen) { - result = turtlescreen[msg.action].apply(turtlescreen, msg.args); - output.pipe.send(JSON.stringify({cmd:'result', 'result':result})); - } else { - output.pipe.send(JSON.stringify({cmd:'exception', exception:'AttributeError', - message:msg.action})); - } - } else if (msg.cmd === 'turtlebatch') { - for (i=0; i < msg.batch.length; i += 1) { - cmd = msg.batch[i]; - turtlescreen[cmd[0]].apply(turtlescreen, cmd[1]); - } - } else { - if(msg.stream === 'internal') { - output.append('
Interner Fehler (bitte melden):\n'); - } - else if (msg.stream === 'stderr') { - showConsole(); - $('#consoleradio').prop('checked', 'checked'); - } - output.append($('',{text:msg.data, 'class':msg.stream})); - } - }; -} From 601e1fab5c715964d90ca9636e1f36c2c5434a26 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 12:23:24 +0200 Subject: [PATCH 071/156] Remove all occurrences of server-sent events --- app/controllers/submissions_controller.rb | 16 ------ .../submissions_controller_spec.rb | 54 ------------------- spec/lib/docker_client_spec.rb | 2 +- 3 files changed, 1 insertion(+), 71 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index be01f550..3aaeb18d 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -315,22 +315,6 @@ class SubmissionsController < ApplicationController # end # end - def with_server_sent_events - response.headers['Content-Type'] = 'text/event-stream' - server_sent_event = SSE.new(response.stream) - server_sent_event.write(nil, event: 'start') - yield(server_sent_event) if block_given? - server_sent_event.write({code: 200}, event: 'close') - rescue StandardError => e - Sentry.capture_exception(e) - logger.error(e.message) - logger.error(e.backtrace.join("\n")) - server_sent_event.write({code: 500}, event: 'close') - ensure - server_sent_event.close - end - private :with_server_sent_events - def create_remote_evaluation_mapping user = @submission.user exercise_id = @submission.exercise_id diff --git a/spec/controllers/submissions_controller_spec.rb b/spec/controllers/submissions_controller_spec.rb index 61e21c98..a176fdc9 100644 --- a/spec/controllers/submissions_controller_spec.rb +++ b/spec/controllers/submissions_controller_spec.rb @@ -154,10 +154,6 @@ describe SubmissionsController do let(:filename) { submission.collect_files.detect(&:main_file?).name_with_extension } let(:perform_request) { get :run, params: {filename: filename, id: submission.id} } - before do - allow_any_instance_of(ActionController::Live::SSE).to receive(:write).at_least(3).times - end - context 'when no errors occur during execution' do before do allow_any_instance_of(DockerClient).to receive(:execute_run_command).with(submission, filename).and_return({}) @@ -229,54 +225,4 @@ describe SubmissionsController do pending('todo') end - - describe '#with_server_sent_events' do - let(:response) { ActionDispatch::TestResponse.new } - - before { allow(controller).to receive(:response).and_return(response) } - - context 'when no error occurs' do - after { controller.send(:with_server_sent_events) } - - it 'uses server-sent events' do - expect(ActionController::Live::SSE).to receive(:new).and_call_original - end - - it "writes a 'start' event" do - allow_any_instance_of(ActionController::Live::SSE).to receive(:write) - expect_any_instance_of(ActionController::Live::SSE).to receive(:write).with(nil, event: 'start') - end - - it "writes a 'close' event" do - allow_any_instance_of(ActionController::Live::SSE).to receive(:write) - expect_any_instance_of(ActionController::Live::SSE).to receive(:write).with({code: 200}, event: 'close') - end - - it 'closes the stream' do - expect_any_instance_of(ActionController::Live::SSE).to receive(:close).and_call_original - end - end - - context 'when an error occurs' do - after { controller.send(:with_server_sent_events) { raise } } - - it 'uses server-sent events' do - expect(ActionController::Live::SSE).to receive(:new).and_call_original - end - - it "writes a 'start' event" do - allow_any_instance_of(ActionController::Live::SSE).to receive(:write) - expect_any_instance_of(ActionController::Live::SSE).to receive(:write).with(nil, event: 'start') - end - - it "writes a 'close' event" do - allow_any_instance_of(ActionController::Live::SSE).to receive(:write) - expect_any_instance_of(ActionController::Live::SSE).to receive(:write).with({code: 500}, event: 'close') - end - - it 'closes the stream' do - expect_any_instance_of(ActionController::Live::SSE).to receive(:close).and_call_original - end - end - end end diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb index 14b81a9c..5b112416 100644 --- a/spec/lib/docker_client_spec.rb +++ b/spec/lib/docker_client_spec.rb @@ -364,7 +364,7 @@ describe DockerClient, docker: true do end it 'provides the command to be executed as input' do - pending('we are currently not using any input and for output server send events instead of attach.') + pending('we are currently not using attach but rather exec.') expect(container).to receive(:attach).with(stdin: kind_of(StringIO)) end From 1403fc03c4aae1383d21323b5705ab3a7f1a0936 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 12:24:05 +0200 Subject: [PATCH 072/156] Remove outdated lines from #download method --- app/controllers/submissions_controller.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 3aaeb18d..2ea68e4b 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -59,10 +59,6 @@ class SubmissionsController < ApplicationController def download raise Pundit::NotAuthorizedError if @embed_options[:disable_download] - # files = @submission.files.map{ } - # zipline( files, 'submission.zip') - # send_data(@file.content, filename: @file.name_with_extension) - id_file = create_remote_evaluation_mapping stringio = Zip::OutputStream.write_buffer do |zio| From cc98dc22298de0e5fe8e4fbee504af752f054563 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 15:12:45 +0200 Subject: [PATCH 073/156] Split WebSocket event in multiple lines before processing --- lib/runner/connection.rb | 24 ++++++++++++-------- lib/runner/strategy/docker_container_pool.rb | 10 ++++---- lib/runner/strategy/poseidon.rb | 4 ++-- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 659d769f..bca9cfea 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -78,16 +78,22 @@ class Runner::Connection def on_message(raw_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } - event = decode(raw_event) - return unless BACKEND_OUTPUT_SCHEMA.valid?(event) + # The WebSocket connection might group multiple lines. For further processing, we require all lines + # to be processed separately. Therefore, we split the lines by each newline character not part of an enclosed + # substring either in single or double quotes (e.g., within a JSON) + # Inspired by https://stackoverflow.com/questions/13040585/split-string-by-spaces-properly-accounting-for-quotes-and-backslashes-ruby + raw_event.data.scan(/(?:"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|[^"\n])+/).each do |event_data| + event = decode(event_data) + next unless BACKEND_OUTPUT_SCHEMA.valid?(event) - event = event.deep_symbolize_keys - message_type = event[:type].to_sym - if WEBSOCKET_MESSAGE_TYPES.include?(message_type) - __send__("handle_#{message_type}", event) - else - @error = Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") - close(:error) + event = event.deep_symbolize_keys + message_type = event[:type].to_sym + if WEBSOCKET_MESSAGE_TYPES.include?(message_type) + __send__("handle_#{message_type}", event) + else + @error = Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") + close(:error) + end end end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index bddf75af..aae831cd 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -121,9 +121,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy "#{data}\n" end - def decode(raw_event) - case raw_event.data - when /@#{@strategy.container_id[0..11]}/ + def decode(event_data) + case event_data + when /(@#{@strategy.container_id[0..11]}|#exit)/ # Assume correct termination for now and return exit code 0 # TODO: Can we use the actual exit code here? @exit_code = 0 @@ -135,11 +135,11 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy when /\*\*\*\*\*\*\*\*\*\*\*\*\* Module/ # Identification of PyLint output, change stream back to stdout and return event @stream = 'stdout' - {'type' => @stream, 'data' => raw_event.data} + {'type' => @stream, 'data' => event_data} when /#{@strategy.command}/ when /bash: cmd:canvasevent: command not found/ else - {'type' => @stream, 'data' => raw_event.data} + {'type' => @stream, 'data' => event_data} end end end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 90699400..7eeef0aa 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -151,8 +151,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end class Connection < Runner::Connection - def decode(raw_event) - JSON.parse(raw_event.data) + def decode(event_data) + JSON.parse(event_data) rescue JSON::ParserError => e @error = Runner::Error::UnexpectedResponse.new("The WebSocket message from Poseidon could not be decoded to JSON: #{e.inspect}") close(:error) From 0f925264945018d1e87cc6ac276919c4c0ec740f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 15:13:59 +0200 Subject: [PATCH 074/156] Remove outdated output callback from Runner::Connection --- lib/runner/connection.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index bca9cfea..36467f3b 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -5,7 +5,7 @@ require 'json_schemer' class Runner::Connection # These are events for which callbacks can be registered. - EVENTS = %i[start output exit stdout stderr].freeze + EVENTS = %i[start exit stdout stderr].freeze WEBSOCKET_MESSAGE_TYPES = %i[start stdout stderr error timeout exit].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) @@ -134,12 +134,10 @@ class Runner::Connection def handle_stdout(event) @stdout_callback.call event[:data] - @output_callback.call event[:data] end def handle_stderr(event) @stderr_callback.call event[:data] - @output_callback.call event[:data] end def handle_error(_event); end From f896d041f82dba44b7c7a896d07aee666e62369a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 10 Oct 2021 15:34:47 +0200 Subject: [PATCH 075/156] Restructure submissions_controller and remove outdated copy_comments method --- app/assets/javascripts/editor/submissions.js | 3 +- app/controllers/submissions_controller.rb | 194 ++++++++----------- db/migrate/20181129093207_drop_errors.rb | 2 +- 3 files changed, 85 insertions(+), 114 deletions(-) diff --git a/app/assets/javascripts/editor/submissions.js b/app/assets/javascripts/editor/submissions.js index af7c78bb..3cd26502 100644 --- a/app/assets/javascripts/editor/submissions.js +++ b/app/assets/javascripts/editor/submissions.js @@ -26,8 +26,7 @@ CodeOceanEditorSubmissions = { cause: $(initiator).data('cause') || $(initiator).prop('id'), exercise_id: $('#editor').data('exercise-id'), files_attributes: (filter || _.identity)(this.collectFiles()) - }, - annotations_arr: [] + } }, dataType: 'json', method: 'POST', diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 2ea68e4b..551775e6 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -7,55 +7,18 @@ class SubmissionsController < ApplicationController include SubmissionParameters include Tubesock::Hijack - before_action :set_submission, - only: %i[download download_file render_file run score extract_errors show statistics] + before_action :set_submission, only: %i[download download_file render_file run score show statistics] before_action :set_files, only: %i[download download_file render_file show run] before_action :set_file, only: %i[download_file render_file run] before_action :set_mime_type, only: %i[download_file render_file] skip_before_action :verify_authenticity_token, only: %i[download_file render_file] - def max_output_buffer_size - if @submission.cause == 'requestComments' - 5000 - else - 500 - end - end - - def authorize! - authorize(@submission || @submissions) - end - private :authorize! - def create @submission = Submission.new(submission_params) authorize! - copy_comments create_and_respond(object: @submission) end - def copy_comments - # copy each annotation and set the target_file.id - params[:annotations_arr]&.each do |annotation| - # comment = Comment.new(annotation[1].permit(:user_id, :file_id, :user_type, :row, :column, :text, :created_at, :updated_at)) - comment = Comment.new(user_id: annotation[1][:user_id], file_id: annotation[1][:file_id], - user_type: current_user.class.name, row: annotation[1][:row], column: annotation[1][:column], text: annotation[1][:text]) - source_file = CodeOcean::File.find(annotation[1][:file_id]) - - # retrieve target file - target_file = @submission.files.detect do |file| - # file_id has to be that of a the former iteration OR of the initial file (if this is the first run) - file.file_id == source_file.file_id || file.file_id == source_file.id # seems to be needed here: (check this): || file.file_id == source_file.id ; yes this is needed, for comments on templates as well as comments on files added by users. - end - - # save to assign an id - target_file.save! - - comment.file_id = target_file.id - comment.save! - end - end - def download raise Pundit::NotAuthorizedError if @embed_options[:disable_download] @@ -201,47 +164,6 @@ class SubmissionsController < ApplicationController save_run_output end - def extract_durations(error) - @container_execution_time = error.execution_duration - @waiting_for_container_time = error.waiting_duration - end - private :extract_durations - - def close_client_connection(client_socket) - # search for errors and save them as StructuredError (for scoring runs see submission.rb) - errors = extract_errors - send_hints(client_socket, errors) - kill_client_socket(client_socket) - end - - def kill_client_socket(client_socket) - client_socket.send_data JSON.dump({cmd: :exit}) - client_socket.close - end - - # save the output of this "run" as a "testrun" (scoring runs are saved in submission.rb) - def save_run_output - Testrun.create( - file: @file, - cause: 'run', - submission: @submission, - output: @output, - container_execution_time: @container_execution_time, - waiting_for_container_time: @waiting_for_container_time - ) - end - - def extract_errors - results = [] - if @output.present? - @submission.exercise.execution_environment.error_templates.each do |template| - pattern = Regexp.new(template.signature).freeze - results << StructuredError.create_from_template(template, @output, @submission) if pattern.match(@output) - end - end - results - end - def score hijack do |tubesock| return if @embed_options[:disable_run] @@ -257,38 +179,6 @@ class SubmissionsController < ApplicationController end end - def send_hints(tubesock, errors) - return if @embed_options[:disable_hints] - - errors = errors.to_a.uniq(&:hint) - errors.each do |error| - tubesock.send_data JSON.dump({cmd: 'hint', hint: error.hint, description: error.error_template.description}) - end - end - - def set_file - @file = @files.detect {|file| file.name_with_extension == sanitize_filename } - head :not_found unless @file - end - private :set_file - - def set_files - @files = @submission.collect_files.select(&:visible) - end - private :set_files - - def set_mime_type - @mime_type = Mime::Type.lookup_by_extension(@file.file_type.file_extension.gsub(/^\./, '')) - response.headers['Content-Type'] = @mime_type.to_s - end - private :set_mime_type - - def set_submission - @submission = Submission.find(params[:id]) - authorize! - end - private :set_submission - def show; end def statistics; end @@ -311,6 +201,24 @@ class SubmissionsController < ApplicationController # end # end + private + + def authorize! + authorize(@submission || @submissions) + end + + def close_client_connection(client_socket) + # search for errors and save them as StructuredError (for scoring runs see submission.rb) + errors = extract_errors + send_hints(client_socket, errors) + kill_client_socket(client_socket) + end + + def kill_client_socket(client_socket) + client_socket.send_data JSON.dump({cmd: :exit}) + client_socket.close + end + def create_remote_evaluation_mapping user = @submission.user exercise_id = @submission.exercise_id @@ -337,7 +245,71 @@ class SubmissionsController < ApplicationController path end + def extract_durations(error) + @container_execution_time = error.execution_duration + @waiting_for_container_time = error.waiting_duration + end + + def extract_errors + results = [] + if @output.present? + @submission.exercise.execution_environment.error_templates.each do |template| + pattern = Regexp.new(template.signature).freeze + results << StructuredError.create_from_template(template, @output, @submission) if pattern.match(@output) + end + end + results + end + + def max_output_buffer_size + if @submission.cause == 'requestComments' + 5000 + else + 500 + end + end + def sanitize_filename params[:filename].gsub(/\.json$/, '') end + + # save the output of this "run" as a "testrun" (scoring runs are saved in submission.rb) + def save_run_output + Testrun.create( + file: @file, + cause: 'run', + submission: @submission, + output: @output, + container_execution_time: @container_execution_time, + waiting_for_container_time: @waiting_for_container_time + ) + end + + def send_hints(tubesock, errors) + return if @embed_options[:disable_hints] + + errors = errors.to_a.uniq(&:hint) + errors.each do |error| + tubesock.send_data JSON.dump({cmd: 'hint', hint: error.hint, description: error.error_template.description}) + end + end + + def set_file + @file = @files.detect {|file| file.name_with_extension == sanitize_filename } + head :not_found unless @file + end + + def set_files + @files = @submission.collect_files.select(&:visible) + end + + def set_mime_type + @mime_type = Mime::Type.lookup_by_extension(@file.file_type.file_extension.gsub(/^\./, '')) + response.headers['Content-Type'] = @mime_type.to_s + end + + def set_submission + @submission = Submission.find(params[:id]) + authorize! + end end diff --git a/db/migrate/20181129093207_drop_errors.rb b/db/migrate/20181129093207_drop_errors.rb index 76306ec9..59418c38 100644 --- a/db/migrate/20181129093207_drop_errors.rb +++ b/db/migrate/20181129093207_drop_errors.rb @@ -34,7 +34,7 @@ class DropErrors < ActiveRecord::Migration[5.2] submissions_controller.instance_variable_set(:@raw_output, raw_output) submissions_controller.instance_variable_set(:@submission, submission) - submissions_controller.extract_errors + submissions_controller.send(:extract_errors) end drop_table :errors From 3240ea7076f18f756ec6f622ae951852fe65115b Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 09:44:34 +0200 Subject: [PATCH 076/156] Forward input as raw_event to runner * Also, rename #send to #send_data in order to prevent debugging issues in RubyMine --- app/controllers/submissions_controller.rb | 10 ++++++---- lib/runner/connection.rb | 2 +- lib/runner/strategy/docker_container_pool.rb | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 551775e6..55f04458 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -95,21 +95,23 @@ class SubmissionsController < ApplicationController runner_socket&.close(:terminated_by_client) end - client_socket.onmessage do |event| - event = JSON.parse(event).deep_symbolize_keys + client_socket.onmessage do |raw_event| + event = JSON.parse(raw_event).deep_symbolize_keys case event[:cmd].to_sym when :client_kill close_client_connection(client_socket) Rails.logger.debug('Client exited container.') - when :result + when :result, :canvasevent, :exception # The client cannot send something before the runner connection is established. if runner_socket.present? - runner_socket.send event[:data] + runner_socket.send_data raw_event else Rails.logger.info("Could not forward data from client because runner connection was not established yet: #{event[:data].inspect}") end else Rails.logger.info("Unknown command from client: #{event[:cmd]}") + Sentry.set_extras(event: event) + Sentry.capture_message("Unknown command from client: #{event[:cmd]}") end rescue JSON::ParserError => e Rails.logger.info("Data received from client is not valid json: #{data.inspect}") diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 36467f3b..c1dd99f5 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -43,7 +43,7 @@ class Runner::Connection end # Send arbitrary data in the WebSocket connection - def send(raw_data) + def send_data(raw_data) encoded_message = encode(raw_data) Rails.logger.debug { "#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}" } @socket.send(encoded_message) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index aae831cd..bf0fbe96 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -64,7 +64,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy socket = Connection.new(websocket_url, self, event_loop) begin Timeout.timeout(@execution_environment.permitted_execution_time) do - socket.send(command) + socket.send_data(command) yield(socket) event_loop.wait event_loop.stop From a074a5cb0dd4d1ba997a107407d338547cd8743d Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 09:47:17 +0200 Subject: [PATCH 077/156] Add buffering to output received from runner --- app/errors/runner/connection/buffer/error.rb | 7 ++ lib/runner/connection.rb | 36 ++++---- lib/runner/connection/buffer.rb | 96 ++++++++++++++++++++ lib/runner/strategy/docker_container_pool.rb | 3 + 4 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 app/errors/runner/connection/buffer/error.rb create mode 100644 lib/runner/connection/buffer.rb diff --git a/app/errors/runner/connection/buffer/error.rb b/app/errors/runner/connection/buffer/error.rb new file mode 100644 index 00000000..106d95fb --- /dev/null +++ b/app/errors/runner/connection/buffer/error.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class Runner::Connection::Buffer + class Error < ApplicationError + class NotEmpty < Error; end + end +end diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index c1dd99f5..addd22fd 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -18,6 +18,7 @@ class Runner::Connection @status = :established @event_loop = event_loop @locale = locale + @buffer = Buffer.new # For every event type of Faye WebSockets, the corresponding # RunnerConnection method starting with `on_` is called. @@ -78,25 +79,27 @@ class Runner::Connection def on_message(raw_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } - # The WebSocket connection might group multiple lines. For further processing, we require all lines - # to be processed separately. Therefore, we split the lines by each newline character not part of an enclosed - # substring either in single or double quotes (e.g., within a JSON) - # Inspired by https://stackoverflow.com/questions/13040585/split-string-by-spaces-properly-accounting-for-quotes-and-backslashes-ruby - raw_event.data.scan(/(?:"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|[^"\n])+/).each do |event_data| - event = decode(event_data) - next unless BACKEND_OUTPUT_SCHEMA.valid?(event) - - event = event.deep_symbolize_keys - message_type = event[:type].to_sym - if WEBSOCKET_MESSAGE_TYPES.include?(message_type) - __send__("handle_#{message_type}", event) - else - @error = Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") - close(:error) - end + @buffer.store raw_event.data + @buffer.events.each do |event_data| + forward_message event_data end end + def forward_message(event_data) + event = decode(event_data) + return unless BACKEND_OUTPUT_SCHEMA.valid?(event) + + event = event.deep_symbolize_keys + message_type = event[:type].to_sym + if WEBSOCKET_MESSAGE_TYPES.include?(message_type) + __send__("handle_#{message_type}", event) + else + @error = Runner::Error::UnexpectedResponse.new("Unknown WebSocket message type: #{message_type}") + close(:error) + end + end + private :forward_message + def on_open(_event) @start_callback.call end @@ -105,6 +108,7 @@ class Runner::Connection def on_close(_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } + forward_message @buffer.flush case @status when :timeout @error = Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') diff --git a/lib/runner/connection/buffer.rb b/lib/runner/connection/buffer.rb new file mode 100644 index 00000000..82dc1006 --- /dev/null +++ b/lib/runner/connection/buffer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +class Runner::Connection::Buffer + # The WebSocket connection might group multiple lines. For further processing, we require all lines + # to be processed separately. Therefore, we split the lines by each newline character not part of an enclosed + # substring either in single or double quotes (e.g., within a JSON) + # Inspired by https://stackoverflow.com/questions/13040585/split-string-by-spaces-properly-accounting-for-quotes-and-backslashes-ruby + SPLIT_INDIVIDUAL_LINES = Regexp.compile(/(?:"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|[^\n])+/) + + def initialize + @global_buffer = +'' + @buffering = false + @line_buffer = Queue.new + super + end + + def store(event_data) + # First, we append the new data to the existing `@global_buffer`. + # Either, the `@global_buffer` is empty and this is a NO OP. + # Or, the `@global_buffer` contains an incomplete string and thus requires the new part. + @global_buffer += event_data + # We process the full `@global_buffer`. Valid parts are removed from the buffer and + # the remaining invalid parts are still stored in `@global_buffer`. + @global_buffer = process_and_split @global_buffer + end + + def events + # Return all items from `@line_buffer` in an array (which is iterable) and clear the queue + Array.new(@line_buffer.size) { @line_buffer.pop } + end + + def flush + raise Error::NotEmpty unless @line_buffer.empty? + + remaining_buffer = @global_buffer + @buffering = false + @global_buffer = +'' + remaining_buffer + end + + private + + def process_and_split(message_parts, stop: false) + # We need a temporary buffer to operate on + buffer = +'' + message_parts.scan(SPLIT_INDIVIDUAL_LINES).each do |line| + # Same argumentation as above: We can always append (previous empty or invalid) + buffer += line + + if buffering_required_for? buffer + @buffering = true + # Check the existing substring `buffer` if it contains a valid message. + # The remaining buffer is stored for further processing. + buffer = process_and_split buffer, stop: true unless stop + else + add_to_line_buffer(buffer) + # Clear the current buffer. + buffer = +'' + end + end + # Return the remaining buffer which might become the `@global_buffer` + buffer + end + + def add_to_line_buffer(message) + @buffering = false + @global_buffer = +'' + @line_buffer.push message + end + + def buffering_required_for?(message) + # First, check if the message is very short and start with { + return true if message.size <= 5 && message.start_with?(/\s*{/) + + invalid_json = !valid_json?(message) + # Second, if we have the beginning of a valid command but an invalid JSON + return true if invalid_json && message.start_with?(/\s*{"cmd/) + # Third, global_buffer the message if it contains long messages (e.g., an image or turtle batch commands) + return true if invalid_json && (message.include?(' Date: Mon, 11 Oct 2021 09:48:11 +0200 Subject: [PATCH 078/156] Forward JSON from runner to client if possible --- app/controllers/submissions_controller.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 55f04458..3ee6857c 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -130,13 +130,13 @@ class SubmissionsController < ApplicationController client_socket.send_data JSON.dump({cmd: :status, status: :container_running}) runner_socket.on :stdout do |data| - json_data = JSON.dump({cmd: :write, stream: :stdout, data: data}) + json_data = prepare data, :stdout @output << json_data[0, max_output_buffer_size - @output.size] client_socket.send_data(json_data) end runner_socket.on :stderr do |data| - json_data = JSON.dump({cmd: :write, stream: :stderr, data: data}) + json_data = prepare data, :stderr @output << json_data[0, max_output_buffer_size - @output.size] client_socket.send_data(json_data) end @@ -271,6 +271,14 @@ class SubmissionsController < ApplicationController end end + def prepare(data, stream) + if valid_command? data + data + else + JSON.dump({cmd: :write, stream: stream, data: data}) + end + end + def sanitize_filename params[:filename].gsub(/\.json$/, '') end @@ -314,4 +322,11 @@ class SubmissionsController < ApplicationController @submission = Submission.find(params[:id]) authorize! end + + def valid_command?(data) + parsed = JSON.parse(data) + parsed.instance_of?(Hash) && parsed.key?('cmd') + rescue JSON::ParserError + false + end end From 788f6dba2014c6c27ef4f84b2a6d4839e7cff749 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 09:56:59 +0200 Subject: [PATCH 079/156] Specify TLS certificate for Faye::WebSocket::Client --- lib/runner/connection.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index addd22fd..002331d1 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -13,7 +13,11 @@ class Runner::Connection attr_reader :error def initialize(url, strategy, event_loop, locale = I18n.locale) - @socket = Faye::WebSocket::Client.new(url, [], ping: 5) + # The `ping` value is measured in seconds and specifies how often a Ping frame should be sent. + # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread + # The `tls` option is used to customize the validation of TLS connections. + # Passing `nil` as a `root_cert_file` is okay and done so for the DockerContainerPool. + @socket = Faye::WebSocket::Client.new(url, [], ping: 0.1, tls: {root_cert_file: Runner.strategy_class.config[:ca_file]}) @strategy = strategy @status = :established @event_loop = event_loop From 7e7b7ebdfaabb99d181469c266e08d1eafe27f85 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 15:16:17 +0200 Subject: [PATCH 080/156] Allow flushing the WebSocket connection * This will prevent the current thread from crashing when a single newline character is received. --- app/controllers/submissions_controller.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 3ee6857c..f0ebd2ec 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -96,7 +96,15 @@ class SubmissionsController < ApplicationController end client_socket.onmessage do |raw_event| - event = JSON.parse(raw_event).deep_symbolize_keys + event = if raw_event == "\n" + # Obviously, this is just flushing the current connection. + # We temporarily wrap it and then forward the original event intentionally. + {cmd: 'result'} + else + # We expect to receive a JSON + JSON.parse(raw_event).deep_symbolize_keys + end + case event[:cmd].to_sym when :client_kill close_client_connection(client_socket) @@ -114,12 +122,12 @@ class SubmissionsController < ApplicationController Sentry.capture_message("Unknown command from client: #{event[:cmd]}") end rescue JSON::ParserError => e - Rails.logger.info("Data received from client is not valid json: #{data.inspect}") - Sentry.set_extras(data: data) + Rails.logger.info("Data received from client is not valid json: #{raw_event.inspect}") + Sentry.set_extras(data: raw_event) Sentry.capture_exception(e) rescue TypeError => e - Rails.logger.info("JSON data received from client cannot be parsed as hash: #{data.inspect}") - Sentry.set_extras(data: data) + Rails.logger.info("JSON data received from client cannot be parsed as hash: #{raw_event.inspect}") + Sentry.set_extras(data: raw_event) Sentry.capture_exception(e) end end From 08f36a0a7aaee23a811bbf99a95a283abb5fee82 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 15:19:23 +0200 Subject: [PATCH 081/156] Destroy runner at management in case of errors --- lib/runner/connection.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 002331d1..5800045b 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -113,15 +113,23 @@ class Runner::Connection def on_close(_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } forward_message @buffer.flush + + # Depending on the status, we might want to destroy the runner at management. + # This ensures we get a new runner on the next request. + # All failing runs, those cancelled by the user or those hitting a timeout or error are subject to this mechanism. + case @status when :timeout + @strategy.destroy_at_management @error = Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') when :terminated_by_codeocean, :terminated_by_management @exit_callback.call @exit_code when :terminated_by_client, :error + @strategy.destroy_at_management else # :established # If the runner is killed by the DockerContainerPool after the maximum allowed time per user and # while the owning user is running an execution, the command execution stops and log output is incomplete. + @strategy.destroy_at_management @error = Runner::Error::Unknown.new('Execution terminated with an unknown reason') end @event_loop.stop From 58e923abd8887b4a016358ca5978da42be67e453 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 23:11:40 +0200 Subject: [PATCH 082/156] Add custom websocket header to strategy class --- lib/runner/connection.rb | 2 +- lib/runner/strategy.rb | 4 ++++ lib/runner/strategy/docker_container_pool.rb | 4 ++++ lib/runner/strategy/poseidon.rb | 7 +++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 5800045b..5619bf2d 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -17,7 +17,7 @@ class Runner::Connection # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread # The `tls` option is used to customize the validation of TLS connections. # Passing `nil` as a `root_cert_file` is okay and done so for the DockerContainerPool. - @socket = Faye::WebSocket::Client.new(url, [], ping: 0.1, tls: {root_cert_file: Runner.strategy_class.config[:ca_file]}) + @socket = Faye::WebSocket::Client.new(url, [], strategy.websocket_header.merge(ping: 0.1)) @strategy = strategy @status = :established @event_loop = event_loop diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index 702311f4..908be168 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -28,4 +28,8 @@ class Runner::Strategy def attach_to_execution(_command) raise NotImplementedError end + + def websocket_header + raise NotImplementedError + end end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 5ae8d62a..5d2fee7d 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -76,6 +76,10 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy socket end + def websocket_header + {} + end + private def container diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 7eeef0aa..2c186e1f 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -121,6 +121,13 @@ class Runner::Strategy::Poseidon < Runner::Strategy raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end + def websocket_header + { + tls: {root_cert_file: self.class.config[:ca_file]}, + headers: {'Poseidon-Token' => self.class.config[:token]}, + } + end + private def execute_command(command) From 1891cdd69c28de710f3576e8a7134871633f7144 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 11 Oct 2021 23:15:36 +0200 Subject: [PATCH 083/156] Add check whether buffer is empty --- lib/runner/connection.rb | 2 +- lib/runner/connection/buffer.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 5619bf2d..03368ab7 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -112,7 +112,7 @@ class Runner::Connection def on_close(_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } - forward_message @buffer.flush + forward_message @buffer.flush unless @buffer.empty? # Depending on the status, we might want to destroy the runner at management. # This ensures we get a new runner on the next request. diff --git a/lib/runner/connection/buffer.rb b/lib/runner/connection/buffer.rb index 82dc1006..c7dcba87 100644 --- a/lib/runner/connection/buffer.rb +++ b/lib/runner/connection/buffer.rb @@ -38,6 +38,10 @@ class Runner::Connection::Buffer remaining_buffer end + def empty? + @line_buffer.empty? && @global_buffer.empty? + end + private def process_and_split(message_parts, stop: false) From 345860c779e5edf544b3c6052eb244cc78706d71 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 13:15:52 +0200 Subject: [PATCH 084/156] Adapt output buffering to Poseidon and DCP * Refactor flushing of messages * Introduce two separate buffers for stdout and stderr --- app/controllers/submissions_controller.rb | 13 ++++------ lib/runner/connection.rb | 31 +++++++++++++---------- lib/runner/strategy/poseidon.rb | 2 +- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index f0ebd2ec..06456a77 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -96,14 +96,11 @@ class SubmissionsController < ApplicationController end client_socket.onmessage do |raw_event| - event = if raw_event == "\n" - # Obviously, this is just flushing the current connection. - # We temporarily wrap it and then forward the original event intentionally. - {cmd: 'result'} - else - # We expect to receive a JSON - JSON.parse(raw_event).deep_symbolize_keys - end + # Obviously, this is just flushing the current connection: Filtering. + next if raw_event == "\n" + + # Otherwise, we expect to receive a JSON: Parsing. + event = JSON.parse(raw_event).deep_symbolize_keys case event[:cmd].to_sym when :client_kill diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 03368ab7..ab3f5279 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -22,7 +22,8 @@ class Runner::Connection @status = :established @event_loop = event_loop @locale = locale - @buffer = Buffer.new + @stdout_buffer = Buffer.new + @stderr_buffer = Buffer.new # For every event type of Faye WebSockets, the corresponding # RunnerConnection method starting with `on_` is called. @@ -83,14 +84,7 @@ class Runner::Connection def on_message(raw_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } - @buffer.store raw_event.data - @buffer.events.each do |event_data| - forward_message event_data - end - end - - def forward_message(event_data) - event = decode(event_data) + event = decode(raw_event.data) return unless BACKEND_OUTPUT_SCHEMA.valid?(event) event = event.deep_symbolize_keys @@ -102,7 +96,6 @@ class Runner::Connection close(:error) end end - private :forward_message def on_open(_event) @start_callback.call @@ -112,7 +105,7 @@ class Runner::Connection def on_close(_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } - forward_message @buffer.flush unless @buffer.empty? + flush_buffers # Depending on the status, we might want to destroy the runner at management. # This ensures we get a new runner on the next request. @@ -149,11 +142,17 @@ class Runner::Connection end def handle_stdout(event) - @stdout_callback.call event[:data] + @stdout_buffer.store event[:data] + @stdout_buffer.events.each do |event_data| + @stdout_callback.call event_data + end end def handle_stderr(event) - @stderr_callback.call event[:data] + @stderr_buffer.store event[:data] + @stderr_buffer.events.each do |event_data| + @stderr_callback.call event_data + end end def handle_error(_event); end @@ -163,4 +162,10 @@ class Runner::Connection def handle_timeout(_event) @status = :timeout end + + def flush_buffers + @stdout_callback.call @stdout_buffer.flush unless @stdout_buffer.empty? + @stderr_callback.call @stderr_buffer.flush unless @stderr_buffer.empty? + end + private :flush_buffers end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 2c186e1f..10ce2a60 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -166,7 +166,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def encode(data) - data + "#{data}\n" end end end From 21e0571838496193cb6695fbb088251b6aead00b Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 13:22:38 +0200 Subject: [PATCH 085/156] Remove unnecessary post parameter from sync_all view --- app/views/execution_environments/index.html.slim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/execution_environments/index.html.slim b/app/views/execution_environments/index.html.slim index a78fd684..60c52d0f 100644 --- a/app/views/execution_environments/index.html.slim +++ b/app/views/execution_environments/index.html.slim @@ -1,7 +1,7 @@ h1.d-inline-block = ExecutionEnvironment.model_name.human(count: 2) - if Runner.management_active? - = button_to( { action: :sync_all_to_runner_management, method: :post }, { form_class: 'float-right mb-2', class: 'btn btn-success' }) + = button_to( { action: :sync_all_to_runner_management }, { form_class: 'float-right mb-2', class: 'btn btn-success' }) i.fa.fa-upload = t('execution_environments.index.synchronize_all.button') From 2377f8370cca20fc01ed0ccbc147f88cf1c52b8e Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 16:19:56 +0200 Subject: [PATCH 086/156] Clarify set_file and set_files in SubmissionsController --- app/controllers/submissions_controller.rb | 13 ++++++++----- app/models/submission.rb | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 06456a77..5454a72c 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -7,9 +7,9 @@ class SubmissionsController < ApplicationController include SubmissionParameters include Tubesock::Hijack - before_action :set_submission, only: %i[download download_file render_file run score show statistics] - before_action :set_files, only: %i[download download_file render_file show run] - before_action :set_file, only: %i[download_file render_file run] + before_action :set_submission, only: %i[download download_file render_file run score show statistics test] + before_action :set_files, only: %i[download show] + before_action :set_files_and_specific_file, only: %i[download_file render_file run test] before_action :set_mime_type, only: %i[download_file render_file] skip_before_action :verify_authenticity_token, only: %i[download_file render_file] @@ -130,7 +130,7 @@ class SubmissionsController < ApplicationController end @output = +'' - durations = @submission.run(sanitize_filename) do |socket| + durations = @submission.run(@file) do |socket| runner_socket = socket client_socket.send_data JSON.dump({cmd: :status, status: :container_running}) @@ -309,7 +309,10 @@ class SubmissionsController < ApplicationController end end - def set_file + def set_files_and_specific_file + # @files contains all visible files for the user + # @file contains the specific file requested for run / test / render / ... + set_files @file = @files.detect {|file| file.name_with_extension == sanitize_filename } head :not_found unless @file end diff --git a/app/models/submission.rb b/app/models/submission.rb index ee277646..29b803d1 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -175,7 +175,7 @@ class Submission < ApplicationRecord end def run(file, &block) - run_command = command_for execution_environment.run_command, file + run_command = command_for execution_environment.run_command, file.name_with_extension durations = {} prepared_runner do |runner, waiting_duration| durations[:execution_duration] = runner.attach_to_execution(run_command, &block) From 56a1d78793e4d005df0a9942f40a94aae4eba6f4 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 16:20:15 +0200 Subject: [PATCH 087/156] Use correct embed_option to disable_score --- app/controllers/submissions_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 5454a72c..ba07f0ee 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -173,7 +173,7 @@ class SubmissionsController < ApplicationController def score hijack do |tubesock| - return if @embed_options[:disable_run] + return if @embed_options[:disable_score] tubesock.send_data(JSON.dump(@submission.calculate_score)) # To enable hints when scoring a submission, uncomment the next line: From 5f9845627692f7d675c7f9049b5481821862454a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 16:21:13 +0200 Subject: [PATCH 088/156] Extract run_test_file from submission.rb --- app/models/submission.rb | 58 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/app/models/submission.rb b/app/models/submission.rb index 29b803d1..267811b0 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -141,33 +141,7 @@ class Submission < ApplicationRecord # If prepared_runner raises an error, no Testrun will be created. prepared_runner do |runner, waiting_duration| file_scores = collect_files.select(&:teacher_defined_assessment?).map do |file| - score_command = command_for execution_environment.test_command, file.name_with_extension - output = {file_role: file.role, waiting_for_container_time: waiting_duration} - stdout = +'' - stderr = +'' - begin - exit_code = 1 # default to error - execution_time = runner.attach_to_execution(score_command) do |socket| - socket.on :stderr do |data| - stderr << data - end - socket.on :stdout do |data| - stdout << data - end - socket.on :exit do |received_exit_code| - exit_code = received_exit_code - end - end - output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) - rescue Runner::Error::ExecutionTimeout => e - Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}" } - output.merge!(status: :timeout, container_execution_time: e.execution_duration) - rescue Runner::Error => e - Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}" } - output.merge!(status: :failed, container_execution_time: e.execution_duration) - ensure - output.merge!(stdout: stdout, stderr: stderr) - end + output = run_test_file file, runner, waiting_duration score_file(output, file) end end @@ -187,6 +161,36 @@ class Submission < ApplicationRecord durations end + def run_test_file(file, runner, waiting_duration) + score_command = command_for execution_environment.test_command, file.name_with_extension + output = {file_role: file.role, waiting_for_container_time: waiting_duration} + stdout = +'' + stderr = +'' + begin + exit_code = 1 # default to error + execution_time = runner.attach_to_execution(score_command) do |socket| + socket.on :stderr do |data| + stderr << data + end + socket.on :stdout do |data| + stdout << data + end + socket.on :exit do |received_exit_code| + exit_code = received_exit_code + end + end + output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) + rescue Runner::Error::ExecutionTimeout => e + Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}" } + output.merge!(status: :timeout, container_execution_time: e.execution_duration) + rescue Runner::Error => e + Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}" } + output.merge!(status: :failed, container_execution_time: e.execution_duration) + ensure + output.merge!(stdout: stdout, stderr: stderr) + end + end + private def prepared_runner From 7285978ea34fd13f5d821117258a0f90fc9451d5 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 16:22:10 +0200 Subject: [PATCH 089/156] Re-add test method for SubmissionsController --- app/controllers/submissions_controller.rb | 29 ++++++++++------------- app/models/submission.rb | 10 ++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index ba07f0ee..e084b7d0 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -190,23 +190,18 @@ class SubmissionsController < ApplicationController def statistics; end - # TODO: make this run, but with the test command - # TODO: add this method to the before action for set_submission again - # def test - # hijack do |tubesock| - # unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive? - # Thread.new do - # EventMachine.run - # ensure - # ActiveRecord::Base.connection_pool.release_connection - # end - # end - # output = @docker_client.execute_test_command(@submission, sanitize_filename) - # # tubesock is the socket to the client - # tubesock.send_data JSON.dump(output) - # tubesock.send_data JSON.dump('cmd' => 'exit') - # end - # end + def test + hijack do |tubesock| + return kill_client_socket(tubesock) if @embed_options[:disable_run] + + tubesock.send_data(JSON.dump(@submission.test(@file))) + rescue Runner::Error => e + tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted}) + Rails.logger.debug { "Runner error while testing submission #{@submission.id}: #{e.message}" } + ensure + kill_client_socket(tubesock) + end + end private diff --git a/app/models/submission.rb b/app/models/submission.rb index 267811b0..f48217fd 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -161,6 +161,16 @@ class Submission < ApplicationRecord durations end + def test(file) + prepared_runner do |runner, waiting_duration| + output = run_test_file file, runner, waiting_duration + score_file output, file + rescue Runner::Error => e + e.waiting_duration = waiting_duration + raise + end + end + def run_test_file(file, runner, waiting_duration) score_command = command_for execution_environment.test_command, file.name_with_extension output = {file_role: file.role, waiting_for_container_time: waiting_duration} From 352e5f432919a7fae8f5efc8dcbafab5b975075d Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 16:43:39 +0200 Subject: [PATCH 090/156] Clarify strategy creation in runner.rb with comment --- app/models/runner.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/runner.rb b/app/models/runner.rb index 861ba5ce..92650f2f 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -25,8 +25,10 @@ class Runner < ApplicationRecord runner = find_by(user: user, execution_environment: execution_environment) if runner.nil? runner = Runner.create(user: user, execution_environment: execution_environment) + # The `strategy` is added through the before_validation hook `:request_id`. raise Runner::Error::Unknown.new("Runner could not be saved: #{runner.errors.inspect}") unless runner.persisted? else + # This information is required but not persisted in the runner model. runner.strategy = strategy_class.new(runner.runner_id, runner.execution_environment) end From 90eeb3bb9cb6455857a2689d5e47c4d986808cf4 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 18:54:27 +0200 Subject: [PATCH 091/156] Move CodeOcean::FileNameValidator --- app/models/code_ocean/file.rb | 10 ---------- app/validators/code_ocean/file_name_validator.rb | 13 +++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 app/validators/code_ocean/file_name_validator.rb diff --git a/app/models/code_ocean/file.rb b/app/models/code_ocean/file.rb index 956d92a7..7bbba9b5 100644 --- a/app/models/code_ocean/file.rb +++ b/app/models/code_ocean/file.rb @@ -4,16 +4,6 @@ require File.expand_path('../../uploaders/file_uploader', __dir__) require File.expand_path('../../../lib/active_model/validations/boolean_presence_validator', __dir__) module CodeOcean - class FileNameValidator < ActiveModel::Validator - def validate(record) - existing_files = File.where(name: record.name, path: record.path, file_type_id: record.file_type_id, - context_id: record.context_id, context_type: record.context_type).to_a - if !existing_files.empty? && (!record.context.is_a?(Exercise) || record.context.new_record?) - record.errors[:base] << 'Duplicate' - end - end - end - class File < ApplicationRecord include DefaultValues diff --git a/app/validators/code_ocean/file_name_validator.rb b/app/validators/code_ocean/file_name_validator.rb new file mode 100644 index 00000000..f9161458 --- /dev/null +++ b/app/validators/code_ocean/file_name_validator.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module CodeOcean + class FileNameValidator < ActiveModel::Validator + def validate(record) + existing_files = File.where(name: record.name, path: record.path, file_type_id: record.file_type_id, + context_id: record.context_id, context_type: record.context_type).to_a + if !existing_files.empty? && (!record.context.is_a?(Exercise) || record.context.new_record?) + record.errors[:base] << 'Duplicate' + end + end + end +end From 064c55b711670de36e099bdb796910345856ab0e Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 18:54:44 +0200 Subject: [PATCH 092/156] Add new validator for all elements of an array --- app/validators/array_validator.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 app/validators/array_validator.rb diff --git a/app/validators/array_validator.rb b/app/validators/array_validator.rb new file mode 100644 index 00000000..f8e88414 --- /dev/null +++ b/app/validators/array_validator.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ArrayValidator < ActiveModel::EachValidator + # Taken from https://gist.github.com/justalever/73a1b36df8468ec101f54381996fb9d1 + + def validate_each(record, attribute, values) + Array(values).each do |value| + options.each do |key, args| + validator_options = {attributes: attribute} + validator_options.merge!(args) if args.is_a?(Hash) + + next if value.nil? && validator_options[:allow_nil] + next if value.blank? && validator_options[:allow_blank] + + validator_class_name = "#{key.to_s.camelize}Validator" + validator_class = begin + validator_class_name.constantize + rescue NameError + "ActiveModel::Validations::#{validator_class_name}".constantize + end + + validator = validator_class.new(validator_options) + validator.validate_each(record, attribute, value) + end + end + end +end From 06ef4430f5fc39e1bf887593f7950216b082956e Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 18:59:03 +0200 Subject: [PATCH 093/156] Change exposed_ports to array --- .../execution_environments_controller.rb | 11 +++++- app/models/execution_environment.rb | 12 ++++-- .../execution_environments/_form.html.slim | 6 +-- .../execution_environments/show.html.slim | 2 +- config/locales/de.yml | 3 +- config/locales/en.yml | 3 +- ..._exposed_ports_in_execution_environment.rb | 37 +++++++++++++++++++ ..._exposed_ports_in_execution_environment.rb | 17 --------- db/schema.rb | 2 +- lib/docker_client.rb | 2 +- spec/factories/execution_environment.rb | 2 +- spec/lib/docker_client_spec.rb | 2 +- spec/models/execution_environment_spec.rb | 16 ++++---- 13 files changed, 75 insertions(+), 40 deletions(-) create mode 100644 db/migrate/20210602071834_change_type_of_exposed_ports_in_execution_environment.rb delete mode 100644 db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 648f7140..68a84625 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -107,8 +107,15 @@ class ExecutionEnvironmentsController < ApplicationController def execution_environment_params if params[:execution_environment].present? - params[:execution_environment].permit(:docker_image, :exposed_ports, :editor_mode, :file_extension, :file_type_id, :help, :indent_size, :memory_limit, :cpu_limit, :name, :network_enabled, :permitted_execution_time, :pool_size, :run_command, :test_command, :testing_framework).merge( - user_id: current_user.id, user_type: current_user.class.name + exposed_ports = if params[:execution_environment][:exposed_ports_list].present? + # Transform the `exposed_ports_list` to `exposed_ports` array + params[:execution_environment].delete(:exposed_ports_list).scan(/\d+/) + else + [] + end + + params[:execution_environment].permit(:docker_image, :editor_mode, :file_extension, :file_type_id, :help, :indent_size, :memory_limit, :cpu_limit, :name, :network_enabled, :permitted_execution_time, :pool_size, :run_command, :test_command, :testing_framework).merge( + user_id: current_user.id, user_type: current_user.class.name, exposed_ports: exposed_ports ) end end diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index cee38252..8aafa19d 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -28,7 +28,8 @@ class ExecutionEnvironment < ApplicationRecord validates :pool_size, numericality: {only_integer: true}, presence: true validates :run_command, presence: true validates :cpu_limit, presence: true, numericality: {greater_than: 0, only_integer: true} - validates :exposed_ports, format: {with: /\A(([[:digit:]]{1,5},)*([[:digit:]]{1,5}))?\z/} + before_validation :clean_exposed_ports + validates :exposed_ports, array: {numericality: {greater_than_or_equal_to: 0, less_than: 65_536, only_integer: true}} def set_default_values set_default_values_if_present(permitted_execution_time: 60, pool_size: 0) @@ -47,14 +48,19 @@ class ExecutionEnvironment < ApplicationRecord cpuLimit: cpu_limit, memoryLimit: memory_limit, networkAccess: network_enabled, - exposedPorts: exposed_ports_list, + exposedPorts: exposed_ports, }.to_json end def exposed_ports_list - (exposed_ports || '').split(',').map(&:to_i) + exposed_ports.join(', ') end + def clean_exposed_ports + self.exposed_ports = exposed_ports.uniq.sort + end + private :clean_exposed_ports + def valid_test_setup? if test_command? ^ testing_framework? errors.add(:test_command, diff --git a/app/views/execution_environments/_form.html.slim b/app/views/execution_environments/_form.html.slim index 9447d1c6..e5853f90 100644 --- a/app/views/execution_environments/_form.html.slim +++ b/app/views/execution_environments/_form.html.slim @@ -14,9 +14,9 @@ = f.text_field(:docker_image, class: 'alternative-input form-control', disabled: true) .help-block.form-text == t('.hints.docker_image') .form-group - = f.label(:exposed_ports) - = f.text_field(:exposed_ports, class: 'form-control', placeholder: '3000,4000', pattern: '^((\d{1,5},)*(\d{1,5}))?$') - .help-block.form-text == t('.hints.exposed_ports') + = f.label(:exposed_ports_list) + = f.text_field(:exposed_ports_list, class: 'form-control', placeholder: '3000, 4000', pattern: '^(\s*(\d{1,5},\s*)*(\d{1,5}\s*))?$') + .help-block.form-text = t('.hints.exposed_ports_list') .form-group = f.label(:memory_limit) = f.number_field(:memory_limit, class: 'form-control', min: DockerClient::MINIMUM_MEMORY_LIMIT, value: f.object.memory_limit || DockerClient::DEFAULT_MEMORY_LIMIT) diff --git a/app/views/execution_environments/show.html.slim b/app/views/execution_environments/show.html.slim index 6d9fb32e..e1b15ec6 100644 --- a/app/views/execution_environments/show.html.slim +++ b/app/views/execution_environments/show.html.slim @@ -5,7 +5,7 @@ h1 = row(label: 'execution_environment.name', value: @execution_environment.name) = row(label: 'execution_environment.user', value: link_to_if(policy(@execution_environment.author).show?, @execution_environment.author, @execution_environment.author)) = row(label: 'execution_environment.file_type', value: @execution_environment.file_type.present? ? link_to(@execution_environment.file_type, @execution_environment.file_type) : nil) -- [:docker_image, :exposed_ports, :memory_limit, :cpu_limit, :network_enabled, :permitted_execution_time, :pool_size].each do |attribute| +- [:docker_image, :exposed_ports_list, :memory_limit, :cpu_limit, :network_enabled, :permitted_execution_time, :pool_size].each do |attribute| = row(label: "execution_environment.#{attribute}", value: @execution_environment.send(attribute)) - [:run_command, :test_command].each do |attribute| = row(label: "execution_environment.#{attribute}") do diff --git a/config/locales/de.yml b/config/locales/de.yml index ea5f34eb..a1ff2414 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -10,6 +10,7 @@ de: execution_environment: docker_image: Docker-Image exposed_ports: Zugängliche Ports + exposed_ports_list: Zugängliche Ports file_type: Standard-Dateityp file_type_id: Standard-Dateityp help: Hilfetext @@ -282,7 +283,7 @@ de: hints: command: filename wird automatisch durch den richtigen Dateinamen ersetzt. docker_image: 'Wählen Sie ein Docker-Image aus der Liste oder fügen Sie ein neues hinzu, welches über DockerHub verfügbar ist.' - exposed_ports: Während der Ausführung sind diese Ports für den Nutzer zugänglich. Die Portnummern müssen mit Komma, aber ohne Leerzeichen voneinander getrennt sein. + exposed_ports_list: Während der Ausführung sind diese Ports für den Nutzer zugänglich. Die Portnummern müssen nummerisch und mit Komma voneinander getrennt sein. errors: not_synced_to_runner_management: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zum Runnermanagement synchronisiert. index: diff --git a/config/locales/en.yml b/config/locales/en.yml index b2995a27..41722ab7 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -10,6 +10,7 @@ en: execution_environment: docker_image: Docker Image exposed_ports: Exposed Ports + exposed_ports_list: Exposed Ports file_type: Default File Type file_type_id: Default File Type help: Help Text @@ -282,7 +283,7 @@ en: hints: command: filename is automatically replaced with the correct filename. docker_image: Pick a Docker image listed above or add a new one which is available via DockerHub. - exposed_ports: During code execution these ports are accessible for the user. Port numbers must be separated by a comma but no space. + exposed_ports_list: During code execution these ports are accessible for the user. Port numbers must be numeric and separated by a comma. errors: not_synced_to_runner_management: The execution environment was created but not synced to the runner management due to an error. index: diff --git a/db/migrate/20210602071834_change_type_of_exposed_ports_in_execution_environment.rb b/db/migrate/20210602071834_change_type_of_exposed_ports_in_execution_environment.rb new file mode 100644 index 00000000..bac19565 --- /dev/null +++ b/db/migrate/20210602071834_change_type_of_exposed_ports_in_execution_environment.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class ChangeTypeOfExposedPortsInExecutionEnvironment < ActiveRecord::Migration[6.1] + # rubocop:disable Rails/SkipsModelValidations: + def up + rename_column :execution_environments, :exposed_ports, :exposed_ports_migration + add_column :execution_environments, :exposed_ports, :integer, array: true, default: [], nil: true + + ExecutionEnvironment.all.each do |execution_environment| + next if execution_environment.exposed_ports_migration.nil? + + cleaned = execution_environment.exposed_ports_migration.scan(/\d+/) + list = cleaned.map(&:to_i).uniq.sort + execution_environment.update_columns(exposed_ports: list) + end + + remove_column :execution_environments, :exposed_ports_migration + end + + def down + rename_column :execution_environments, :exposed_ports, :exposed_ports_migration + add_column :execution_environments, :exposed_ports, :string + + ExecutionEnvironment.all.each do |execution_environment| + next if execution_environment.exposed_ports_migration.empty? + + list = execution_environment.exposed_ports_migration + if list.empty? + execution_environment.update_columns(exposed_ports: nil) + else + execution_environment.update_columns(exposed_ports: list.join(',')) + end + end + remove_column :execution_environments, :exposed_ports_migration + end + # rubocop:enable Rails/SkipsModelValidations: +end diff --git a/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb b/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb deleted file mode 100644 index c50cc53f..00000000 --- a/db/migrate/20210602071834_clean_exposed_ports_in_execution_environment.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class CleanExposedPortsInExecutionEnvironment < ActiveRecord::Migration[6.1] - def change - ExecutionEnvironment.all.each do |execution_environment| - next if execution_environment.exposed_ports.nil? - - cleaned = execution_environment.exposed_ports.gsub(/[[:space:]]/, '') - list = cleaned.split(',').map(&:to_i).uniq - if list.empty? - execution_environment.update(exposed_ports: nil) - else - execution_environment.update(exposed_ports: list.join(',')) - end - end - end -end diff --git a/db/schema.rb b/db/schema.rb index 9a53d05b..59de10d1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -104,7 +104,6 @@ ActiveRecord::Schema.define(version: 2021_06_02_071834) do t.string "test_command", limit: 255 t.string "testing_framework", limit: 255 t.text "help" - t.string "exposed_ports", limit: 255 t.integer "permitted_execution_time" t.integer "user_id" t.string "user_type", limit: 255 @@ -113,6 +112,7 @@ ActiveRecord::Schema.define(version: 2021_06_02_071834) do t.integer "memory_limit" t.boolean "network_enabled" t.integer "cpu_limit", default: 20, null: false + t.integer "exposed_ports", default: [], array: true end create_table "exercise_collection_items", id: :serial, force: :cascade do |t| diff --git a/lib/docker_client.rb b/lib/docker_client.rb index 313bb226..ff9a9f45 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -448,7 +448,7 @@ container_execution_time: nil} end def self.mapped_ports(execution_environment) - (execution_environment.exposed_ports || '').gsub(/\s/, '').split(',').map do |port| + execution_environment.exposed_ports.map do |port| ["#{port}/tcp", [{'HostPort' => PortPool.available_port.to_s}]] end.to_h end diff --git a/spec/factories/execution_environment.rb b/spec/factories/execution_environment.rb index 9d145031..94ecfdd4 100644 --- a/spec/factories/execution_environment.rb +++ b/spec/factories/execution_environment.rb @@ -122,7 +122,7 @@ FactoryBot.define do default_cpu_limit docker_image { 'hklement/ubuntu-sinatra:latest' } file_type { association :dot_rb, user: user } - exposed_ports { '4567' } + exposed_ports { [4567] } help name { 'Sinatra' } network_enabled { true } diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb index 5b112416..54208a77 100644 --- a/spec/lib/docker_client_spec.rb +++ b/spec/lib/docker_client_spec.rb @@ -332,7 +332,7 @@ describe DockerClient, docker: true do describe '.mapped_ports' do context 'with exposed ports' do - before { execution_environment.exposed_ports = '3000' } + before { execution_environment.exposed_ports = [3000] } it 'returns a mapping' do expect(described_class.mapped_ports(execution_environment)).to be_a(Hash) diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index d7b057ab..6525869d 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -178,17 +178,17 @@ describe ExecutionEnvironment do end describe '#exposed_ports_list' do - it 'returns an empty array if no ports are exposed' do - execution_environment.exposed_ports = nil - expect(execution_environment.exposed_ports_list).to eq([]) + it 'returns an empty string if no ports are exposed' do + execution_environment.exposed_ports = [] + expect(execution_environment.exposed_ports_list).to eq('') end - it 'returns an array of integers representing the exposed ports' do - execution_environment.exposed_ports = '1,2,3' - expect(execution_environment.exposed_ports_list).to eq([1, 2, 3]) + it 'returns an string with comma-separated integers representing the exposed ports' do + execution_environment.exposed_ports = [1, 2, 3] + expect(execution_environment.exposed_ports_list).to eq('1, 2, 3') - execution_environment.exposed_ports_list.each do |port| - expect(execution_environment.exposed_ports).to include(port.to_s) + execution_environment.exposed_ports.each do |port| + expect(execution_environment.exposed_ports_list).to include(port.to_s) end end end From c676785d55790748f6e12c1bb3b28dc82451eca3 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 19:32:55 +0200 Subject: [PATCH 094/156] Fix order-dependent runner_spec.rb --- spec/models/runner_spec.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index 0a496b24..bd99f4bb 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -34,15 +34,12 @@ describe Runner do let(:runner_management_config) { {runner_management: {enabled: true, strategy: strategy}} } before do + # Ensure to reset the memorized helper + described_class.instance_variable_set :@strategy_class, nil allow(CodeOcean::Config).to receive(:new).with(:code_ocean).and_return(codeocean_config) allow(codeocean_config).to receive(:read).and_return(runner_management_config) end - after do - # Reset the memorized helper - described_class.remove_instance_variable :@strategy_class - end - it "uses #{strategy_class} as strategy class for constant #{strategy}" do expect(described_class.strategy_class).to eq(strategy_class) end From a1db30c2884e3394b9bedd04ca910a8ab7515123 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 22:42:17 +0200 Subject: [PATCH 095/156] Clarify EnvironmentNotFound error handling --- app/models/runner.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 92650f2f..3aa53a22 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -78,13 +78,17 @@ class Runner < ApplicationRecord self.runner_id = strategy_class.request_from_management(execution_environment) @strategy = strategy_class.new(runner_id, execution_environment) rescue Runner::Error::EnvironmentNotFound + # Whenever the environment could not be found by the runner management, we + # try to synchronize it and then forward a more specific error to our callee. if strategy_class.sync_environment(execution_environment) raise Runner::Error::EnvironmentNotFound.new( - "The execution environment with id #{execution_environment.id} was not found and was successfully synced with the runner management" + "The execution environment with id #{execution_environment.id} was not found yet by the runner management. "\ + 'It has been successfully synced now so that the next request should be successful.' ) else raise Runner::Error::EnvironmentNotFound.new( - "The execution environment with id #{execution_environment.id} was not found and could not be synced with the runner management" + "The execution environment with id #{execution_environment.id} was not found by the runner management."\ + 'In addition, it could not be synced so that this probably indicates a permanent error.' ) end end From 95f97bd66e27d6d3629ab4239b7f3726396bec0d Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 23:11:41 +0200 Subject: [PATCH 096/156] Add documentation to code_ocean.yml --- config/code_ocean.yml.ci | 2 ++ config/code_ocean.yml.example | 37 +++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/config/code_ocean.yml.ci b/config/code_ocean.yml.ci index 1c531f9e..92b4eb01 100644 --- a/config/code_ocean.yml.ci +++ b/config/code_ocean.yml.ci @@ -13,4 +13,6 @@ test: enabled: true strategy: poseidon url: https://runners.example.org + ca_file: /example/certificates/ca.crt + token: SECRET unused_runner_expiration_time: 180 diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 712db04a..049ac305 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -1,37 +1,70 @@ default: &default flowr: + # When enabled, flowr can assist learners with related search results from + # StackOverflow.com regarding exceptions that occurred during code execution. + # The search is initiated through the learners' browser and displayed in the output pane. enabled: false + # The number of search results to be displayed answers_per_query: 3 + code_pilot: + # When enabled, CodePilot can be used by learners to request individual help by a tutor + # through a video conferencing system. Optionally, it also provides access to recordings + # of previous sessions. Support for CodePilot is currently in beta. enabled: false + # The root URL of CodePilot url: //localhost:3000 + codeharbor: + # When enabled, CodeHarbor is integrated in the teachers' view and allows importing + # and exporting exercises from CodeOcean using the ProFormA XML format to CodeHarbor. enabled: false + # The root URL of CodeHarbor + url: https://codeharbor.openhpi.de + codeocean_events: + # When enabled, learner-specific events within the editor are stored and can be used + # as part of learning analytics. This setting enables the JavaScript event handlers. enabled: false + prometheus_exporter: + # When enabled, a dedicated endpoint using the Prometheus format is offered and might + # be used by a Prometheus-compatible monitoring system. Exported metrics include absolute + # counters of all relations with specific support for Request-for-Comments. enabled: false + runner_management: + # When enabled, CodeOcean delegates the handling and management of (containerized) runners + # to a dedicated runner management. Otherwise, code executions are performed locally using + # Docker and without pre-warming support (one container per execution). enabled: false + # The strategy to use. Possible values are: poseidon, docker_container_pool strategy: poseidon + # The root URL of the runner management to use url: https://runners.example.org + # The root certificate authority to trust for TLS connections to the runner management (Poseidon only) ca_file: /example/certificates/ca.crt + # The authorization token for connections to the runner management (Poseidon only) + # If TLS support is not enabled, this token is transmitted in clear text! token: SECRET + # The maximum time in seconds a runner may idle at the runner management before + # it is removed. Each interaction with the runner resets this time (Poseidon only) unused_runner_expiration_time: 180 + development: <<: *default flowr: enabled: true - answers_per_query: 3 codeharbor: enabled: true - url: https://codeharbor.openhpi.de + production: <<: *default prometheus_exporter: enabled: true + test: <<: *default From a6a477e3616930939c95d1dc495627712435dfbe Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 17 Oct 2021 23:36:50 +0200 Subject: [PATCH 097/156] Move error handling to else branch in execute_command --- lib/runner/strategy/poseidon.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 10ce2a60..1c8ed3b9 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -139,16 +139,13 @@ class Runner::Strategy::Poseidon < Runner::Strategy when 200 response_body = self.class.parse response websocket_url = response_body[:websocketUrl] - if websocket_url.present? - return websocket_url - else - raise Runner::Error::UnexpectedResponse.new('Poseidon did not send a WebSocket URL') - end + websocket_url.presence || raise(Runner::Error::UnexpectedResponse.new('Poseidon did not send a WebSocket URL')) when 400 Runner.destroy(@allocation_id) + self.class.handle_error response + else + self.class.handle_error response end - - self.class.handle_error response rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") end From 696cd6a236e18d5da66d60fcf7548c674471a7d6 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 00:06:45 +0200 Subject: [PATCH 098/156] Poseidon: Clean workspace between executions --- lib/runner/strategy/poseidon.rb | 5 ++++- spec/lib/runner/strategy/poseidon_spec.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 1c8ed3b9..7080ddd9 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -95,7 +95,10 @@ class Runner::Strategy::Poseidon < Runner::Strategy } end url = "#{runner_url}/files" - body = {copy: copy} + + # First, clean the workspace and second, copy all files to their location. + # This ensures that no artefacts from a previous submission remain in the workspace. + body = {copy: copy, delete: ['./']} connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} response = connection.patch url, body.to_json, self.class.headers return if response.status == 204 diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index b87bf462..f81670da 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -319,7 +319,7 @@ describe Runner::Strategy::Poseidon do WebMock .stub_request(:patch, "#{described_class.config[:url]}/runners/#{runner_id}/files") .with( - body: {copy: [{path: file.filepath, content: encoded_file_content}]}, + body: {copy: [{path: file.filepath, content: encoded_file_content}], delete: ['./']}, headers: {'Content-Type' => 'application/json'} ) .to_return(body: response_body, status: response_status) From 50b62b57034d7d15784fe3c35cdedc1e86a8deca Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 01:22:26 +0200 Subject: [PATCH 099/156] Move flush_buffers method in Runner::Connection --- lib/runner/connection.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index ab3f5279..1e523b59 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -78,6 +78,11 @@ class Runner::Connection raise NotImplementedError end + def flush_buffers + @stdout_callback.call @stdout_buffer.flush unless @stdout_buffer.empty? + @stderr_callback.call @stderr_buffer.flush unless @stderr_buffer.empty? + end + # === WebSocket Callbacks # These callbacks are executed based on events indicated by Faye WebSockets and are # independent of the JSON specification that is used within the WebSocket once established. @@ -162,10 +167,4 @@ class Runner::Connection def handle_timeout(_event) @status = :timeout end - - def flush_buffers - @stdout_callback.call @stdout_buffer.flush unless @stdout_buffer.empty? - @stderr_callback.call @stderr_buffer.flush unless @stderr_buffer.empty? - end - private :flush_buffers end From c7ddbd676c2a1d9e446f439de0d0b5b98da2bd0b Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 01:23:31 +0200 Subject: [PATCH 100/156] Do not forward custom exit handlers to frontend --- lib/runner/connection.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 1e523b59..0abb7b28 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -83,6 +83,16 @@ class Runner::Connection @stderr_callback.call @stderr_buffer.flush unless @stderr_buffer.empty? end + def ignored_sequence?(event_data) + case event_data + when "#exit\r", "{\"cmd\": \"exit\"}\r" + # Do not forward. We will wait for the confirmed exit sent by the runner management. + true + else + false + end + end + # === WebSocket Callbacks # These callbacks are executed based on events indicated by Faye WebSockets and are # independent of the JSON specification that is used within the WebSocket once established. @@ -149,14 +159,14 @@ class Runner::Connection def handle_stdout(event) @stdout_buffer.store event[:data] @stdout_buffer.events.each do |event_data| - @stdout_callback.call event_data + @stdout_callback.call event_data unless ignored_sequence? event_data end end def handle_stderr(event) @stderr_buffer.store event[:data] @stderr_buffer.events.each do |event_data| - @stderr_callback.call event_data + @stderr_callback.call event_data unless ignored_sequence? event_data end end From 04c896c7de83126e212293dba5d7e2faba94f0ff Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 01:24:01 +0200 Subject: [PATCH 101/156] DCP: Listen for Python exit handler --- lib/runner/strategy/docker_container_pool.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 5d2fee7d..baccbf7f 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -127,7 +127,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def decode(event_data) case event_data - when /(@#{@strategy.container_id[0..11]}|#exit)/ + when /(@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/ # TODO: The whole message line is kept back. If this contains the remaining buffer, this buffer is also lost. # Example: A Java program prints `{` and then exists (with `#exit`). The `event_data` processed here is `{#exit` From 68c8f1dbdf795e51552af0bc99bd32666da67fed Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 01:24:17 +0200 Subject: [PATCH 102/156] DCP: Set sticky bit for folder and secure delete --- lib/runner/strategy/docker_container_pool.rb | 2 +- spec/lib/runner/strategy/docker_container_pool_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index baccbf7f..8592b5dd 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -47,7 +47,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end end end - FileUtils.chmod_R('+rwX', local_workspace_path) + FileUtils.chmod_R('+rwtX', local_workspace_path) end def destroy_at_management diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index 9f75e241..01b418fd 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -106,7 +106,7 @@ describe Runner::Strategy::DockerContainerPool do end it 'sets permission bits on the workspace' do - expect(FileUtils).to receive(:chmod_R).with('+rwX', local_path) + expect(FileUtils).to receive(:chmod_R).with('+rwtX', local_path) container_pool.copy_files(files) end From 2ad4eb76259d3334e01c458e989f37c6712323a5 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 01:24:32 +0200 Subject: [PATCH 103/156] DCP: Escape command for RegEx --- lib/runner/strategy/docker_container_pool.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 8592b5dd..e1059d2e 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -143,7 +143,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy # Identification of PyLint output, change stream back to stdout and return event @stream = 'stdout' {'type' => @stream, 'data' => event_data} - when /#{@strategy.command}/ + when /#{Regexp.quote(@strategy.command)}/ when /bash: cmd:canvasevent: command not found/ else {'type' => @stream, 'data' => event_data} From e95ad5e26cf06e04e8af1ed7de159f6484b5d6e2 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 22:02:42 +0200 Subject: [PATCH 104/156] Add @!attribute comments to connection.rb --- lib/runner/connection.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 0abb7b28..50f41478 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -9,6 +9,10 @@ class Runner::Connection WEBSOCKET_MESSAGE_TYPES = %i[start stdout stderr error timeout exit].freeze BACKEND_OUTPUT_SCHEMA = JSONSchemer.schema(JSON.parse(File.read('lib/runner/backend-output.schema.json'))) + # @!attribute start_callback + # @!attribute exit_callback + # @!attribute stdout_callback + # @!attribute stderr_callback attr_writer :status attr_reader :error From e272fcd19c9ac4bdda2a985951bc30c80a107937 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 18 Oct 2021 22:24:09 +0200 Subject: [PATCH 105/156] Add more comments and error logging to connection.rb --- lib/runner/connection.rb | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 50f41478..ac13818f 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -45,7 +45,7 @@ class Runner::Connection @exit_code = 1 end - # Register a callback based on the WebSocket connection state + # Register a callback based on the event type received from runner management def on(event, &block) return unless EVENTS.include? event @@ -120,7 +120,12 @@ class Runner::Connection @start_callback.call end - def on_error(_event); end + def on_error(event) + # In case of an WebSocket error, the connection will be closed by Faye::WebSocket::Client automatically. + # Thus, no further handling is required here (the user will get notified). + Sentry.set_extras(event: event.inspect) + Sentry.capture_message("The WebSocket connection to #{@socket.url} was closed with an error.") + end def on_close(_event) Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } @@ -174,11 +179,21 @@ class Runner::Connection end end - def handle_error(_event); end + def handle_error(event) + # In case of a (Nomad) error during execution, the runner management will notify us with an error message here. + # This shouldn't happen to often and can be considered an internal server error by the runner management. + # More information is available in the logs of the runner management or the orchestrator (e.g., Nomad). + Sentry.set_extras(event: event.inspect) + Sentry.capture_message("An error occurred during code execution while being connected to #{@socket.url}.") + end - def handle_start(_event); end + def handle_start(_event) + # The execution just started as requested. This is an informal message and no further processing is required. + end def handle_timeout(_event) @status = :timeout + # The runner management stopped the execution as the permitted execution time was exceeded. + # We set the status here and wait for the connection to be closed (by the runner management). end end From 7e2039ebc2131f6da26ff5dfc4116961a2519812 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Tue, 19 Oct 2021 00:23:58 +0200 Subject: [PATCH 106/156] Fix bug that always showed the default value for CPU limit when editing the limit --- app/views/execution_environments/_form.html.slim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/execution_environments/_form.html.slim b/app/views/execution_environments/_form.html.slim index e5853f90..b2073d6f 100644 --- a/app/views/execution_environments/_form.html.slim +++ b/app/views/execution_environments/_form.html.slim @@ -22,7 +22,7 @@ = f.number_field(:memory_limit, class: 'form-control', min: DockerClient::MINIMUM_MEMORY_LIMIT, value: f.object.memory_limit || DockerClient::DEFAULT_MEMORY_LIMIT) .form-group = f.label(:cpu_limit) - = f.number_field(:cpu_limit, class: 'form-control', min: 1, step: 1, value: ExecutionEnvironment::DEFAULT_CPU_LIMIT) + = f.number_field(:cpu_limit, class: 'form-control', min: 1, step: 1, value: f.object.cpu_limit || ExecutionEnvironment::DEFAULT_CPU_LIMIT) .form-check.mb-3 label.form-check-label = f.check_box(:network_enabled, class: 'form-check-input') From 02a2673bf23908e5213c869241e1bfb9da32d1db Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Tue, 19 Oct 2021 00:32:54 +0200 Subject: [PATCH 107/156] Add hint for CPU limit --- app/views/execution_environments/_form.html.slim | 1 + config/locales/de.yml | 1 + config/locales/en.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/app/views/execution_environments/_form.html.slim b/app/views/execution_environments/_form.html.slim index b2073d6f..6b74eeae 100644 --- a/app/views/execution_environments/_form.html.slim +++ b/app/views/execution_environments/_form.html.slim @@ -23,6 +23,7 @@ .form-group = f.label(:cpu_limit) = f.number_field(:cpu_limit, class: 'form-control', min: 1, step: 1, value: f.object.cpu_limit || ExecutionEnvironment::DEFAULT_CPU_LIMIT) + .help-block.form-text = t('.hints.cpu_limit') .form-check.mb-3 label.form-check-label = f.check_box(:network_enabled, class: 'form-check-input') diff --git a/config/locales/de.yml b/config/locales/de.yml index a1ff2414..4cf94e06 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -284,6 +284,7 @@ de: command: filename wird automatisch durch den richtigen Dateinamen ersetzt. docker_image: 'Wählen Sie ein Docker-Image aus der Liste oder fügen Sie ein neues hinzu, welches über DockerHub verfügbar ist.' exposed_ports_list: Während der Ausführung sind diese Ports für den Nutzer zugänglich. Die Portnummern müssen nummerisch und mit Komma voneinander getrennt sein. + cpu_limit: Geben Sie die Mindestmenge an CPU-Anteilen an, die für jeden Runner reserviert werden soll, gemessen in MHz. errors: not_synced_to_runner_management: Die Ausführungsumgebung wurde erstellt, aber aufgrund eines Fehlers nicht zum Runnermanagement synchronisiert. index: diff --git a/config/locales/en.yml b/config/locales/en.yml index 41722ab7..f9c1cfd9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,6 +284,7 @@ en: command: filename is automatically replaced with the correct filename. docker_image: Pick a Docker image listed above or add a new one which is available via DockerHub. exposed_ports_list: During code execution these ports are accessible for the user. Port numbers must be numeric and separated by a comma. + cpu_limit: Specify the minimum amount of CPU shares to reserve for each runner, measured in MHz. errors: not_synced_to_runner_management: The execution environment was created but not synced to the runner management due to an error. index: From d87e23b9a3736f9465d2574ada33c9b4c07b4aa9 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Tue, 19 Oct 2021 22:53:27 +0200 Subject: [PATCH 108/156] Add `execute_command` method to runner.rb * This is now used by the score and test runs * This also re-enables the interactive shell for execution environments --- .../execution_environments_controller.rb | 5 ++- app/models/runner.rb | 43 +++++++++++++++++-- app/models/submission.rb | 33 +++----------- .../execution_environments_controller_spec.rb | 6 +-- spec/models/execution_environment_spec.rb | 8 ++-- spec/models/runner_spec.rb | 8 ++-- 6 files changed, 60 insertions(+), 43 deletions(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 68a84625..bd5e96d0 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -30,8 +30,9 @@ class ExecutionEnvironmentsController < ApplicationController end def execute_command - @docker_client = DockerClient.new(execution_environment: @execution_environment) - render(json: @docker_client.execute_arbitrary_command(params[:command])) + runner = Runner.for(current_user, @execution_environment) + output = runner.execute_command(params[:command]) + render(json: output) end def working_time_query diff --git a/app/models/runner.rb b/app/models/runner.rb index 3aa53a22..f9f30704 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -19,9 +19,7 @@ class Runner < ApplicationRecord @management_active ||= CodeOcean::Config.new(:code_ocean).read[:runner_management][:enabled] end - def self.for(user, exercise) - execution_environment = exercise.execution_environment - + def self.for(user, execution_environment) runner = find_by(user: user, execution_environment: execution_environment) if runner.nil? runner = Runner.create(user: user, execution_environment: execution_environment) @@ -62,6 +60,45 @@ class Runner < ApplicationRecord Time.zone.now - starting_time # execution duration end + def execute_command(command) + output = {} + stdout = +'' + stderr = +'' + try = 0 + begin + exit_code = 1 # default to error + execution_time = attach_to_execution(command) do |socket| + socket.on :stderr do |data| + stderr << data + end + socket.on :stdout do |data| + stdout << data + end + socket.on :exit do |received_exit_code| + exit_code = received_exit_code + end + end + output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) + rescue Runner::Error::ExecutionTimeout => e + Rails.logger.debug { "Running command `#{command}` timed out: #{e.message}" } + output.merge!(status: :timeout, container_execution_time: e.execution_duration) + rescue Runner::Error::RunnerNotFound => e + Rails.logger.debug { "Running command `#{command}` failed for the first time: #{e.message}" } + try += 1 + request_new_id + save + retry if try == 1 + + Rails.logger.debug { "Running command `#{command}` failed for the second time: #{e.message}" } + output.merge!(status: :failed, container_execution_time: e.execution_duration) + rescue Runner::Error => e + Rails.logger.debug { "Running command `#{command}` failed: #{e.message}" } + output.merge!(status: :failed, container_execution_time: e.execution_duration) + ensure + output.merge!(stdout: stdout, stderr: stderr) + end + end + def destroy_at_management @strategy.destroy_at_management end diff --git a/app/models/submission.rb b/app/models/submission.rb index f48217fd..ea67a351 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -172,33 +172,10 @@ class Submission < ApplicationRecord end def run_test_file(file, runner, waiting_duration) - score_command = command_for execution_environment.test_command, file.name_with_extension - output = {file_role: file.role, waiting_for_container_time: waiting_duration} - stdout = +'' - stderr = +'' - begin - exit_code = 1 # default to error - execution_time = runner.attach_to_execution(score_command) do |socket| - socket.on :stderr do |data| - stderr << data - end - socket.on :stdout do |data| - stdout << data - end - socket.on :exit do |received_exit_code| - exit_code = received_exit_code - end - end - output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed) - rescue Runner::Error::ExecutionTimeout => e - Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} timed out: #{e.message}" } - output.merge!(status: :timeout, container_execution_time: e.execution_duration) - rescue Runner::Error => e - Rails.logger.debug { "Running tests in #{file.name_with_extension} for submission #{id} failed: #{e.message}" } - output.merge!(status: :failed, container_execution_time: e.execution_duration) - ensure - output.merge!(stdout: stdout, stderr: stderr) - end + test_command = command_for execution_environment.test_command, file.name_with_extension + result = {file_role: file.role, waiting_for_container_time: waiting_duration} + output = runner.execute_command(test_command) + result.merge(output) end private @@ -206,7 +183,7 @@ class Submission < ApplicationRecord def prepared_runner request_time = Time.zone.now begin - runner = Runner.for(user, exercise) + runner = Runner.for(user, exercise.execution_environment) runner.copy_files(collect_files) rescue Runner::Error => e e.waiting_duration = Time.zone.now - request_time diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index eacb0fb5..e01b300d 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -75,12 +75,12 @@ describe ExecutionEnvironmentsController do let(:command) { 'which ruby' } before do - allow(DockerClient).to receive(:new).with(execution_environment: execution_environment).and_call_original - allow_any_instance_of(DockerClient).to receive(:execute_arbitrary_command).with(command) + runner = instance_double 'runner' + allow(Runner).to receive(:for).with(user, execution_environment).and_return runner + allow(runner).to receive(:execute_command).and_return({}) post :execute_command, params: {command: command, id: execution_environment.id} end - expect_assigns(docker_client: DockerClient) expect_assigns(execution_environment: :execution_environment) expect_json expect_status(200) diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index 6525869d..e4dba06c 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -149,10 +149,12 @@ describe ExecutionEnvironment do before { allow(DockerClient).to receive(:find_image_by_tag).and_return(Object.new) } - it 'instantiates a Docker client' do - expect(DockerClient).to receive(:new).with(execution_environment: execution_environment).and_call_original - allow_any_instance_of(DockerClient).to receive(:execute_arbitrary_command).and_return({}) + it 'instantiates a Runner' do + runner = instance_double 'runner' + allow(Runner).to receive(:for).with(execution_environment.author, execution_environment).and_return runner + allow(runner).to receive(:execute_command).and_return({}) working_docker_image? + expect(runner).to have_received(:execute_command).once end it 'executes the validation command' do diff --git a/spec/models/runner_spec.rb b/spec/models/runner_spec.rb index bd99f4bb..c5d7bf90 100644 --- a/spec/models/runner_spec.rb +++ b/spec/models/runner_spec.rb @@ -247,7 +247,7 @@ describe Runner do before { allow(strategy_class).to receive(:request_from_management).and_return(nil) } it 'raises an error' do - expect { described_class.for(user, exercise) }.to raise_error(Runner::Error::Unknown, /could not be saved/) + expect { described_class.for(user, exercise.execution_environment) }.to raise_error(Runner::Error::Unknown, /could not be saved/) end end @@ -255,12 +255,12 @@ describe Runner do let!(:existing_runner) { FactoryBot.create(:runner, user: user, execution_environment: exercise.execution_environment) } it 'returns the existing runner' do - new_runner = described_class.for(user, exercise) + new_runner = described_class.for(user, exercise.execution_environment) expect(new_runner).to eq(existing_runner) end it 'sets the strategy' do - runner = described_class.for(user, exercise) + runner = described_class.for(user, exercise.execution_environment) expect(runner.strategy).to be_present end end @@ -269,7 +269,7 @@ describe Runner do before { allow(strategy_class).to receive(:request_from_management).and_return(runner_id) } it 'returns a new runner' do - runner = described_class.for(user, exercise) + runner = described_class.for(user, exercise.execution_environment) expect(runner).to be_valid end end From 9cc4394296660be76ebda22b82d0fd90c85304d6 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Tue, 19 Oct 2021 23:20:34 +0200 Subject: [PATCH 109/156] Allow editing an Execution Environment with active runner management --- app/models/execution_environment.rb | 6 +++--- app/models/runner.rb | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 8aafa19d..863138ad 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -76,10 +76,10 @@ class ExecutionEnvironment < ApplicationRecord private :validate_docker_image? def working_docker_image? - DockerClient.pull(docker_image) if DockerClient.find_image_by_tag(docker_image).present? - output = DockerClient.new(execution_environment: self).execute_arbitrary_command(VALIDATION_COMMAND) + runner = Runner.for(author, self) + output = runner.execute_command(VALIDATION_COMMAND, raise_exception: true) errors.add(:docker_image, "error: #{output[:stderr]}") if output[:stderr].present? - rescue DockerClient::Error => e + rescue Runner::Error => e errors.add(:docker_image, "error: #{e}") end private :working_docker_image? diff --git a/app/models/runner.rb b/app/models/runner.rb index f9f30704..fbe317cd 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -60,7 +60,7 @@ class Runner < ApplicationRecord Time.zone.now - starting_time # execution duration end - def execute_command(command) + def execute_command(command, raise_exception: false) output = {} stdout = +'' stderr = +'' @@ -95,6 +95,9 @@ class Runner < ApplicationRecord Rails.logger.debug { "Running command `#{command}` failed: #{e.message}" } output.merge!(status: :failed, container_execution_time: e.execution_duration) ensure + # We forward the exception if requested + raise e if raise_exception && defined?(e) && e.present? + output.merge!(stdout: stdout, stderr: stderr) end end From 1dfee31079ee21b321a098f99721f8212fd3ba86 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Tue, 19 Oct 2021 23:52:59 +0200 Subject: [PATCH 110/156] Fix order-dependent execution_environments_controller_spec.rb --- spec/controllers/execution_environments_controller_spec.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index e01b300d..68f4ed45 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -200,15 +200,12 @@ describe ExecutionEnvironmentsController do let(:runner_management_config) { {runner_management: {enabled: true, strategy: :poseidon}} } before do + # Ensure to reset the memorized helper + Runner.instance_variable_set :@strategy_class, nil allow(CodeOcean::Config).to receive(:new).with(:code_ocean).and_return(codeocean_config) allow(codeocean_config).to receive(:read).and_return(runner_management_config) end - after do - # Reset the memorized helper - Runner.remove_instance_variable :@strategy_class - end - it 'copies all execution environments to the runner management' do allow(ExecutionEnvironment).to receive(:all).and_return(execution_environments) From 87c53023362dffc2afc4be5810001ed518d63e80 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Thu, 21 Oct 2021 10:06:30 +0200 Subject: [PATCH 111/156] Switch logging to milliseconds and add more details * By design, most logging happens in an `ensure` block. This ensures that no return value is modified unexpectedly. --- lib/runner/connection.rb | 9 ++++++--- lib/runner/strategy/docker_container_pool.rb | 16 ++++++++++++++-- lib/runner/strategy/poseidon.rb | 16 +++++++++++++++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index ac13818f..02c65922 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -17,6 +17,7 @@ class Runner::Connection attr_reader :error def initialize(url, strategy, event_loop, locale = I18n.locale) + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Opening connection to #{url}" } # The `ping` value is measured in seconds and specifies how often a Ping frame should be sent. # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread # The `tls` option is used to customize the validation of TLS connections. @@ -55,7 +56,7 @@ class Runner::Connection # Send arbitrary data in the WebSocket connection def send_data(raw_data) encoded_message = encode(raw_data) - Rails.logger.debug { "#{Time.zone.now.getutc}: Sending to #{@socket.url}: #{encoded_message.inspect}" } + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Sending to #{@socket.url}: #{encoded_message.inspect}" } @socket.send(encoded_message) end @@ -102,7 +103,7 @@ class Runner::Connection # independent of the JSON specification that is used within the WebSocket once established. def on_message(raw_event) - Rails.logger.debug { "#{Time.zone.now.getutc}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Receiving from #{@socket.url}: #{raw_event.data.inspect}" } event = decode(raw_event.data) return unless BACKEND_OUTPUT_SCHEMA.valid?(event) @@ -117,6 +118,7 @@ class Runner::Connection end def on_open(_event) + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Established connection to #{@socket.url}" } @start_callback.call end @@ -128,7 +130,7 @@ class Runner::Connection end def on_close(_event) - Rails.logger.debug { "#{Time.zone.now.getutc}: Closing connection to #{@socket.url} with status: #{@status}" } + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Closing connection to #{@socket.url} with status: #{@status}" } flush_buffers # Depending on the status, we might want to destroy the runner at management. @@ -149,6 +151,7 @@ class Runner::Connection @strategy.destroy_at_management @error = Runner::Error::Unknown.new('Execution terminated with an unknown reason') end + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Closed connection to #{@socket.url} with status: #{@status}" } @event_loop.stop end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index e1059d2e..3e0f2e3b 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -14,12 +14,17 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.request_from_management(environment) - container_id = JSON.parse(Faraday.get("#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}").body)['id'] + url = "#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}" + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } + response = Faraday.get url + container_id = JSON.parse(response.body)['id'] container_id.presence || raise(Runner::Error::NotAvailable.new("DockerContainerPool didn't return a container id")) rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") rescue JSON::ParserError => e raise Runner::Error::UnexpectedResponse.new("DockerContainerPool returned invalid JSON: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished new runner request" } end def initialize(runner_id, _environment) @@ -28,6 +33,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def copy_files(files) + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Sending files to #{local_workspace_path}" } FileUtils.mkdir_p(local_workspace_path) clean_workspace files.each do |file| @@ -48,12 +54,18 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end end FileUtils.chmod_R('+rwtX', local_workspace_path) + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished copying files" } end def destroy_at_management - Faraday.get("#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}") + url = "#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}" + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{url}" } + Faraday.get(url) rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } end def attach_to_execution(command, event_loop) diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 7080ddd9..5dc649ff 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -36,6 +36,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy executionEnvironmentId: environment.id, inactivityTimeout: config[:unused_runner_expiration_time].seconds, } + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} response = connection.post url, body.to_json, headers @@ -51,6 +52,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished new runner request" } end def self.handle_error(response) @@ -88,13 +91,15 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def copy_files(files) + url = "#{runner_url}/files" + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Sending files to #{url}" } + copy = files.map do |file| { path: file.filepath, content: Base64.strict_encode64(file.content), } end - url = "#{runner_url}/files" # First, clean the workspace and second, copy all files to their location. # This ensures that no artefacts from a previous submission remain in the workspace. @@ -107,6 +112,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy self.class.handle_error response rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished copying files" } end def attach_to_execution(command, event_loop) @@ -117,11 +124,14 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def destroy_at_management + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{runner_url}" } connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} response = connection.delete runner_url, nil, self.class.headers self.class.handle_error response unless response.status == 204 rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } end def websocket_header @@ -136,8 +146,10 @@ class Runner::Strategy::Poseidon < Runner::Strategy def execute_command(command) url = "#{runner_url}/execute" body = {command: command, timeLimit: @execution_environment.permitted_execution_time} + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Preparing command execution at #{url}: #{command}" } connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} response = connection.post url, body.to_json, self.class.headers + case response.status when 200 response_body = self.class.parse response @@ -151,6 +163,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished command execution preparation" } end def runner_url From 8a4bd84d04a2236cbcbdf4a11e881a3e622f504a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Thu, 21 Oct 2021 10:07:19 +0200 Subject: [PATCH 112/156] Add note about using hostnames on IPv6 systems --- config/code_ocean.yml.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 049ac305..3c8cb672 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -41,6 +41,8 @@ default: &default # The strategy to use. Possible values are: poseidon, docker_container_pool strategy: poseidon # The root URL of the runner management to use + # If a hostname is specified and the target host is reachable via IPv6, the WebSocket + # connection might not use the IPv6-to-IPv4 fallback but rather fail unexpectedly. url: https://runners.example.org # The root certificate authority to trust for TLS connections to the runner management (Poseidon only) ca_file: /example/certificates/ca.crt From 0db6f2093324b1811231e07d7a10a4daf8087575 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 09:55:12 +0200 Subject: [PATCH 113/156] Move MemoryLimit to Execution Environment --- app/models/execution_environment.rb | 4 +++- app/views/execution_environments/_form.html.slim | 2 +- ...150317083739_add_memory_limit_to_execution_environments.rb | 2 +- lib/docker_client.rb | 2 -- spec/factories/execution_environment.rb | 2 +- spec/models/execution_environment_spec.rb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 863138ad..3bb86331 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -8,6 +8,8 @@ class ExecutionEnvironment < ApplicationRecord VALIDATION_COMMAND = 'whoami' DEFAULT_CPU_LIMIT = 20 + DEFAULT_MEMORY_LIMIT = 256 + MINIMUM_MEMORY_LIMIT = 4 after_initialize :set_default_values @@ -21,7 +23,7 @@ class ExecutionEnvironment < ApplicationRecord validate :working_docker_image?, if: :validate_docker_image? validates :docker_image, presence: true validates :memory_limit, - numericality: {greater_than_or_equal_to: DockerClient::MINIMUM_MEMORY_LIMIT, only_integer: true}, presence: true + numericality: {greater_than_or_equal_to: MINIMUM_MEMORY_LIMIT, only_integer: true}, presence: true validates :network_enabled, boolean_presence: true validates :name, presence: true validates :permitted_execution_time, numericality: {only_integer: true}, presence: true diff --git a/app/views/execution_environments/_form.html.slim b/app/views/execution_environments/_form.html.slim index 6b74eeae..2a1122e8 100644 --- a/app/views/execution_environments/_form.html.slim +++ b/app/views/execution_environments/_form.html.slim @@ -19,7 +19,7 @@ .help-block.form-text = t('.hints.exposed_ports_list') .form-group = f.label(:memory_limit) - = f.number_field(:memory_limit, class: 'form-control', min: DockerClient::MINIMUM_MEMORY_LIMIT, value: f.object.memory_limit || DockerClient::DEFAULT_MEMORY_LIMIT) + = f.number_field(:memory_limit, class: 'form-control', min: ExecutionEnvironment::MINIMUM_MEMORY_LIMIT, value: f.object.memory_limit || ExecutionEnvironment::DEFAULT_MEMORY_LIMIT) .form-group = f.label(:cpu_limit) = f.number_field(:cpu_limit, class: 'form-control', min: 1, step: 1, value: f.object.cpu_limit || ExecutionEnvironment::DEFAULT_CPU_LIMIT) diff --git a/db/migrate/20150317083739_add_memory_limit_to_execution_environments.rb b/db/migrate/20150317083739_add_memory_limit_to_execution_environments.rb index 290f6c1b..05d4a7b7 100644 --- a/db/migrate/20150317083739_add_memory_limit_to_execution_environments.rb +++ b/db/migrate/20150317083739_add_memory_limit_to_execution_environments.rb @@ -6,7 +6,7 @@ class AddMemoryLimitToExecutionEnvironments < ActiveRecord::Migration[4.2] reversible do |direction| direction.up do - ExecutionEnvironment.update(memory_limit: DockerClient::DEFAULT_MEMORY_LIMIT) + ExecutionEnvironment.update(memory_limit: ExecutionEnvironment::DEFAULT_MEMORY_LIMIT) end end end diff --git a/lib/docker_client.rb b/lib/docker_client.rb index ff9a9f45..0517ff69 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -8,10 +8,8 @@ class DockerClient end CONTAINER_WORKSPACE_PATH = '/workspace' # '/home/python/workspace' #'/tmp/workspace' - DEFAULT_MEMORY_LIMIT = 256 # Ralf: I suggest to replace this with the environment variable. Ask Hauke why this is not the case! LOCAL_WORKSPACE_ROOT = File.expand_path(config[:workspace_root]) - MINIMUM_MEMORY_LIMIT = 4 RECYCLE_CONTAINERS = false RETRY_COUNT = 2 MINIMUM_CONTAINER_LIFETIME = 10.minutes diff --git a/spec/factories/execution_environment.rb b/spec/factories/execution_environment.rb index 94ecfdd4..c13c2292 100644 --- a/spec/factories/execution_environment.rb +++ b/spec/factories/execution_environment.rb @@ -152,7 +152,7 @@ FactoryBot.define do end trait :default_memory_limit do - memory_limit { DockerClient::DEFAULT_MEMORY_LIMIT } + memory_limit { ExecutionEnvironment::DEFAULT_MEMORY_LIMIT } end trait :default_cpu_limit do diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index e4dba06c..09fd1c0a 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -16,7 +16,7 @@ describe ExecutionEnvironment do end it 'validates the minimum value of the memory limit' do - execution_environment.update(memory_limit: DockerClient::MINIMUM_MEMORY_LIMIT / 2) + execution_environment.update(memory_limit: ExecutionEnvironment::MINIMUM_MEMORY_LIMIT / 2) expect(execution_environment.errors[:memory_limit]).to be_present end From 541afa92f3c2427107c99eccec3dcf3b0eb3a745 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 10:23:28 +0200 Subject: [PATCH 114/156] Remove ws_client_protocol option * The correct setting will be determined automatically --- .../javascripts/editor/{execution.js.erb => execution.js} | 6 +++--- config/docker.yml.erb.ci | 4 ---- config/docker.yml.erb.example | 3 --- 3 files changed, 3 insertions(+), 10 deletions(-) rename app/assets/javascripts/editor/{execution.js.erb => execution.js} (88%) diff --git a/app/assets/javascripts/editor/execution.js.erb b/app/assets/javascripts/editor/execution.js similarity index 88% rename from app/assets/javascripts/editor/execution.js.erb rename to app/assets/javascripts/editor/execution.js index eb0ff55a..8ac6df4d 100644 --- a/app/assets/javascripts/editor/execution.js.erb +++ b/app/assets/javascripts/editor/execution.js @@ -2,11 +2,11 @@ CodeOceanEditorWebsocket = { websocket: null, createSocketUrl: function(url) { - var sockURL = new URL(url, window.location); + const sockURL = new URL(url, window.location); // not needed any longer, we put it directly into the url: sockURL.pathname = url; - // sanitize socket protocol string, strip trailing slash and other malicious chars if they are there - sockURL.protocol = '<%= DockerClient.config['ws_client_protocol']&.match(/(\w+):*\/*/)&.to_a&.at(1) %>:'; + // replace `http` with `ws` for the WebSocket connection. This also works with `https` and `wss`. + sockURL.protocol = sockURL.protocol.replace("http", "ws"); // strip anchor if it is in the url sockURL.hash = ''; diff --git a/config/docker.yml.erb.ci b/config/docker.yml.erb.ci index e878ca2f..12415d0f 100644 --- a/config/docker.yml.erb.ci +++ b/config/docker.yml.erb.ci @@ -10,7 +10,6 @@ development: <<: *default host: tcp://127.0.0.1:2376 ws_host: ws://127.0.0.1:2376 #url to connect rails server to docker host - ws_client_protocol: 'ws:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> pool: location: http://localhost:7100 @@ -35,7 +34,6 @@ production: timeout: 60 workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> ws_host: ws://localhost:4243 #url to connect rails server to docker host - ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) staging: <<: *default @@ -50,10 +48,8 @@ staging: timeout: 60 workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> ws_host: ws://localhost:4243 #url to connect rails server to docker host - ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) test: <<: *default host: tcp://127.0.0.1:2376 workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> - ws_client_protocol: 'ws:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) diff --git a/config/docker.yml.erb.example b/config/docker.yml.erb.example index 2e5d2955..bb41db98 100644 --- a/config/docker.yml.erb.example +++ b/config/docker.yml.erb.example @@ -10,7 +10,6 @@ development: <<: *default host: tcp://127.0.0.1:2376 ws_host: ws://127.0.0.1:2376 #url to connect rails server to docker host - ws_client_protocol: 'ws:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> pool: active: true @@ -35,7 +34,6 @@ production: timeout: 60 workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> ws_host: ws://localhost:4243 #url to connect rails server to docker host - ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) staging: <<: *default @@ -50,7 +48,6 @@ staging: timeout: 60 workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> ws_host: ws://localhost:4243 #url to connect rails server to docker host - ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production) test: <<: *default From 2b98905acb4b604152ca625f3630ccdef8b1911b Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 11:21:24 +0200 Subject: [PATCH 115/156] Remove usage of DockerClient from execution_environments_controller.rb --- app/controllers/execution_environments_controller.rb | 10 +++++----- lib/runner/strategy.rb | 4 ++++ lib/runner/strategy/docker_container_pool.rb | 7 +++++++ lib/runner/strategy/poseidon.rb | 7 +++++++ .../execution_environments_controller_spec.rb | 2 ++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index bd5e96d0..5b966fb1 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -133,12 +133,12 @@ class ExecutionEnvironmentsController < ApplicationController end def set_docker_images - DockerClient.check_availability! - @docker_images = DockerClient.image_tags.sort - rescue DockerClient::Error => e - @docker_images = [] + @docker_images ||= ExecutionEnvironment.pluck(:docker_image) + @docker_images += Runner.strategy_class.available_images + rescue Runner::Error::InternalServerError => e flash[:warning] = e.message - Sentry.capture_exception(e) + ensure + @docker_images = @docker_images.sort.uniq end private :set_docker_images diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index 908be168..418c679e 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -9,6 +9,10 @@ class Runner::Strategy raise NotImplementedError end + def self.available_images + raise NotImplementedError + end + def self.sync_environment(_environment) raise NotImplementedError end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 3e0f2e3b..272278a3 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -8,6 +8,13 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy @config ||= CodeOcean::Config.new(:docker).read(erb: true) end + def self.available_images + DockerClient.check_availability! + DockerClient.image_tags + rescue DockerClient::Error => e + raise Runner::Error::InternalServerError.new(e.message) + end + def self.sync_environment(_environment) # There is no dedicated sync mechanism yet true diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 5dc649ff..d3c27bd2 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -13,6 +13,13 @@ class Runner::Strategy::Poseidon < Runner::Strategy @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} end + def self.available_images + # Images are pulled when needed for a new execution environment + # and cleaned up automatically if no longer in use. + # Hence, there is no additional image that we need to return + [] + end + def self.headers @headers ||= {'Content-Type' => 'application/json', 'Poseidon-Token' => config[:token]} end diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index 68f4ed45..cfa94fb3 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -114,6 +114,7 @@ describe ExecutionEnvironmentsController do let(:docker_images) { [1, 2, 3] } before do + allow(Runner).to receive(:strategy_class).and_return Runner::Strategy::DockerContainerPool allow(DockerClient).to receive(:check_availability!).at_least(:once) allow(DockerClient).to receive(:image_tags).and_return(docker_images) controller.send(:set_docker_images) @@ -126,6 +127,7 @@ describe ExecutionEnvironmentsController do let(:error_message) { 'Docker is unavailable' } before do + allow(Runner).to receive(:strategy_class).and_return Runner::Strategy::DockerContainerPool allow(DockerClient).to receive(:check_availability!).at_least(:once).and_raise(DockerClient::Error.new(error_message)) controller.send(:set_docker_images) end From e33af5760dedd2bb416cf7ffa142b93a69502370 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 11:39:19 +0200 Subject: [PATCH 116/156] Delegate initialization to Runner::Strategy --- config/application.rb | 3 +++ config/initializers/docker.rb | 5 ----- lib/runner/strategy.rb | 4 ++++ lib/runner/strategy/docker_container_pool.rb | 4 ++++ lib/runner/strategy/poseidon.rb | 5 +++++ 5 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 config/initializers/docker.rb diff --git a/config/application.rb b/config/application.rb index 91c8782d..c481b600 100644 --- a/config/application.rb +++ b/config/application.rb @@ -49,6 +49,9 @@ module CodeOcean config.after_initialize do # Initialize the counters according to the db Prometheus::Controller.initialize_metrics + + # Initialize the runner environment + Runner.strategy_class.initialize_environment end end end diff --git a/config/initializers/docker.rb b/config/initializers/docker.rb deleted file mode 100644 index 518d2d46..00000000 --- a/config/initializers/docker.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -require 'docker_client' - -DockerClient.initialize_environment unless Rails.env.test? && `which docker`.blank? diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index 418c679e..9a19b77b 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -9,6 +9,10 @@ class Runner::Strategy raise NotImplementedError end + def self.initialize_environment + raise NotImplementedError + end + def self.available_images raise NotImplementedError end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 272278a3..056da47d 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -8,6 +8,10 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy @config ||= CodeOcean::Config.new(:docker).read(erb: true) end + def self.initialize_environment + DockerClient.initialize_environment unless Rails.env.test? && `which docker`.blank? + end + def self.available_images DockerClient.check_availability! DockerClient.image_tags diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index d3c27bd2..15f7d8e9 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -13,6 +13,11 @@ class Runner::Strategy::Poseidon < Runner::Strategy @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} end + def self.initialize_environment + # There is no additional initialization required for Poseidon + nil + end + def self.available_images # Images are pulled when needed for a new execution environment # and cleaned up automatically if no longer in use. From f3b4be3006d6fab69e32c8d4d896c814dd5980df Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 11:39:58 +0200 Subject: [PATCH 117/156] Fix deprecation warning for raise_on_missing_translations --- config/environments/development.rb | 2 +- config/environments/staging.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/environments/development.rb b/config/environments/development.rb index a49ea5a8..a67669de 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -66,7 +66,7 @@ Rails.application.configure do config.assets.quiet = true # Raises error for missing translations. - config.action_view.raise_on_missing_translations = true + config.i18n.raise_on_missing_translations = true # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 2ad25a24..73832c26 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -32,7 +32,7 @@ Rails.application.configure do config.assets.raise_runtime_errors = true # Raise errors for missing translations. - config.action_view.raise_on_missing_translations = true + config.i18n.raise_on_missing_translations = true # Enable Rack::Cache to put a simple HTTP cache in front of your application # Add `rack-cache` to your Gemfile before enabling this. From 6d1b388e3c41dfe4f97c71eb9b717cab6b5411ff Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 12:36:01 +0200 Subject: [PATCH 118/156] Reorder methods in strategy classes --- lib/runner/connection.rb | 2 +- lib/runner/strategy.rb | 28 +++-- lib/runner/strategy/docker_container_pool.rb | 56 +++++---- lib/runner/strategy/poseidon.rb | 122 ++++++++++--------- 4 files changed, 116 insertions(+), 92 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 02c65922..7a58bd8f 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -22,7 +22,7 @@ class Runner::Connection # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread # The `tls` option is used to customize the validation of TLS connections. # Passing `nil` as a `root_cert_file` is okay and done so for the DockerContainerPool. - @socket = Faye::WebSocket::Client.new(url, [], strategy.websocket_header.merge(ping: 0.1)) + @socket = Faye::WebSocket::Client.new(url, [], strategy.class.websocket_header.merge(ping: 0.1)) @strategy = strategy @status = :established @event_loop = event_loop diff --git a/lib/runner/strategy.rb b/lib/runner/strategy.rb index 9a19b77b..ce58a947 100644 --- a/lib/runner/strategy.rb +++ b/lib/runner/strategy.rb @@ -5,18 +5,10 @@ class Runner::Strategy @execution_environment = environment end - def self.config - raise NotImplementedError - end - def self.initialize_environment raise NotImplementedError end - def self.available_images - raise NotImplementedError - end - def self.sync_environment(_environment) raise NotImplementedError end @@ -33,11 +25,27 @@ class Runner::Strategy raise NotImplementedError end - def attach_to_execution(_command) + def attach_to_execution(_command, _event_loop) raise NotImplementedError end - def websocket_header + def self.available_images + raise NotImplementedError + end + + def self.config + raise NotImplementedError + end + + def self.release + raise NotImplementedError + end + + def self.pool_size + raise NotImplementedError + end + + def self.websocket_header raise NotImplementedError end end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 056da47d..df47fac0 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -3,22 +3,15 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy attr_reader :container_id, :command - def self.config - # Since the docker configuration file contains code that must be executed, we use ERB templating. - @config ||= CodeOcean::Config.new(:docker).read(erb: true) + def initialize(runner_id, _environment) + super + @container_id = runner_id end def self.initialize_environment DockerClient.initialize_environment unless Rails.env.test? && `which docker`.blank? end - def self.available_images - DockerClient.check_availability! - DockerClient.image_tags - rescue DockerClient::Error => e - raise Runner::Error::InternalServerError.new(e.message) - end - def self.sync_environment(_environment) # There is no dedicated sync mechanism yet true @@ -38,9 +31,14 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished new runner request" } end - def initialize(runner_id, _environment) - super - @container_id = runner_id + def destroy_at_management + url = "#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}" + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{url}" } + Faraday.get(url) + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } end def copy_files(files) @@ -69,16 +67,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished copying files" } end - def destroy_at_management - url = "#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}" - Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{url}" } - Faraday.get(url) - rescue Faraday::Error => e - raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") - ensure - Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } - end - def attach_to_execution(command, event_loop) @command = command query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' @@ -99,7 +87,27 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy socket end - def websocket_header + def self.available_images + DockerClient.check_availability! + DockerClient.image_tags + rescue DockerClient::Error => e + raise Runner::Error::InternalServerError.new(e.message) + end + + def self.config + # Since the docker configuration file contains code that must be executed, we use ERB templating. + @config ||= CodeOcean::Config.new(:docker).read(erb: true) + end + + def self.release + nil + end + + def self.pool_size + {} + end + + def self.websocket_header {} end diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 15f7d8e9..88562759 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -9,8 +9,9 @@ class Runner::Strategy::Poseidon < Runner::Strategy end end - def self.config - @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} + def initialize(runner_id, _environment) + super + @allocation_id = runner_id end def self.initialize_environment @@ -18,17 +19,6 @@ class Runner::Strategy::Poseidon < Runner::Strategy nil end - def self.available_images - # Images are pulled when needed for a new execution environment - # and cleaned up automatically if no longer in use. - # Hence, there is no additional image that we need to return - [] - end - - def self.headers - @headers ||= {'Content-Type' => 'application/json', 'Poseidon-Token' => config[:token]} - end - def self.sync_environment(environment) url = "#{config[:url]}/execution-environments/#{environment.id}" connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} @@ -68,38 +58,15 @@ class Runner::Strategy::Poseidon < Runner::Strategy Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished new runner request" } end - def self.handle_error(response) - case response.status - when 400 - response_body = parse response - raise Runner::Error::BadRequest.new(response_body[:message]) - when 401 - raise Runner::Error::Unauthorized.new('Authentication with Poseidon failed') - when 404 - raise Runner::Error::RunnerNotFound.new - when 500 - response_body = parse response - error_code = response_body[:errorCode] - if error_code == error_nomad_overload - raise Runner::Error::NotAvailable.new("Poseidon has no runner available (#{error_code}): #{response_body[:message]}") - else - raise Runner::Error::InternalServerError.new("Poseidon sent #{response_body[:errorCode]}: #{response_body[:message]}") - end - else - raise Runner::Error::UnexpectedResponse.new("Poseidon sent unexpected response status code #{response.status}") - end - end - - def self.parse(response) - JSON.parse(response.body).deep_symbolize_keys - rescue JSON::ParserError => e - # Poseidon should not send invalid json - raise Runner::Error::UnexpectedResponse.new("Error parsing response from Poseidon: #{e.message}") - end - - def initialize(runner_id, _environment) - super - @allocation_id = runner_id + def destroy_at_management + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{runner_url}" } + connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} + response = connection.delete runner_url, nil, self.class.headers + self.class.handle_error response unless response.status == 204 + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } end def copy_files(files) @@ -135,24 +102,65 @@ class Runner::Strategy::Poseidon < Runner::Strategy socket end - def destroy_at_management - Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{runner_url}" } - connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.delete runner_url, nil, self.class.headers - self.class.handle_error response unless response.status == 204 - rescue Faraday::Error => e - raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") - ensure - Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished destroying runner" } + def self.available_images + # Images are pulled when needed for a new execution environment + # and cleaned up automatically if no longer in use. + # Hence, there is no additional image that we need to return + [] end - def websocket_header + def self.config + @config ||= CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} + end + + def self.release + nil + end + + def self.pool_size + {} + end + + def self.websocket_header { - tls: {root_cert_file: self.class.config[:ca_file]}, - headers: {'Poseidon-Token' => self.class.config[:token]}, + tls: {root_cert_file: config[:ca_file]}, + headers: {'Poseidon-Token' => config[:token]}, } end + def self.handle_error(response) + case response.status + when 400 + response_body = parse response + raise Runner::Error::BadRequest.new(response_body[:message]) + when 401 + raise Runner::Error::Unauthorized.new('Authentication with Poseidon failed') + when 404 + raise Runner::Error::RunnerNotFound.new + when 500 + response_body = parse response + error_code = response_body[:errorCode] + if error_code == error_nomad_overload + raise Runner::Error::NotAvailable.new("Poseidon has no runner available (#{error_code}): #{response_body[:message]}") + else + raise Runner::Error::InternalServerError.new("Poseidon sent #{response_body[:errorCode]}: #{response_body[:message]}") + end + else + raise Runner::Error::UnexpectedResponse.new("Poseidon sent unexpected response status code #{response.status}") + end + end + + def self.headers + @headers ||= {'Content-Type' => 'application/json', 'Poseidon-Token' => config[:token]} + end + + def self.parse(response) + JSON.parse(response.body).deep_symbolize_keys + rescue JSON::ParserError => e + # Poseidon should not send invalid json + raise Runner::Error::UnexpectedResponse.new("Error parsing response from Poseidon: #{e.message}") + end + private def execute_command(command) From ada438b23001b64c72741cd2bc7f523995c8ce16 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 12:59:55 +0200 Subject: [PATCH 119/156] Add release and pool_size methods to DCP --- app/helpers/admin/dashboard_helper.rb | 9 ++++++++- lib/runner/strategy/docker_container_pool.rb | 17 +++++++++++++++-- spec/helpers/admin/dashboard_helper_spec.rb | 8 ++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/app/helpers/admin/dashboard_helper.rb b/app/helpers/admin/dashboard_helper.rb index 791b430d..1b209fb8 100644 --- a/app/helpers/admin/dashboard_helper.rb +++ b/app/helpers/admin/dashboard_helper.rb @@ -7,8 +7,15 @@ module Admin end def docker_data + pool_size = begin + Runner.strategy_class.pool_size + rescue Runner::Error => e + Rails.logger.debug { "Runner error while fetching current pool size: #{e.message}" } + [] + end + ExecutionEnvironment.order(:id).select(:id, :pool_size).map do |execution_environment| - execution_environment.attributes.merge(quantity: DockerContainerPool.quantities[execution_environment.id]) + execution_environment.attributes.merge(quantity: pool_size[execution_environment.id]) end end end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index df47fac0..fdfe945a 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -100,11 +100,24 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.release - nil + url = "#{config[:pool][:location]}/docker_container_pool/dump_info" + response = Faraday.get(url) + JSON.parse(response.body)['release'] + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + rescue JSON::ParserError => e + raise Runner::Error::UnexpectedResponse.new("DockerContainerPool returned invalid JSON: #{e.inspect}") end def self.pool_size - {} + url = "#{config[:pool][:location]}/docker_container_pool/quantities" + response = Faraday.get(url) + pool_size = JSON.parse(response.body) + pool_size.transform_keys(&:to_i) + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + rescue JSON::ParserError => e + raise Runner::Error::UnexpectedResponse.new("DockerContainerPool returned invalid JSON: #{e.inspect}") end def self.websocket_header diff --git a/spec/helpers/admin/dashboard_helper_spec.rb b/spec/helpers/admin/dashboard_helper_spec.rb index a0594a13..22307561 100644 --- a/spec/helpers/admin/dashboard_helper_spec.rb +++ b/spec/helpers/admin/dashboard_helper_spec.rb @@ -10,7 +10,12 @@ describe Admin::DashboardHelper do end describe '#docker_data' do - before { FactoryBot.create(:ruby) } + before do + FactoryBot.create(:ruby) + dcp = instance_double 'docker_container_pool' + allow(Runner).to receive(:strategy_class).and_return dcp + allow(dcp).to receive(:pool_size).and_return([]) + end it 'contains an entry for every execution environment' do expect(docker_data.length).to eq(ExecutionEnvironment.count) @@ -21,7 +26,6 @@ describe Admin::DashboardHelper do end it 'contains the number of available containers for every execution environment' do - expect(DockerContainerPool).to receive(:quantities).exactly(ExecutionEnvironment.count).times.and_call_original expect(docker_data.first).to include(:quantity) end end From af93603ba3452a8adf2dd925ebeb7cfd48ff7382 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 13:00:35 +0200 Subject: [PATCH 120/156] Use strategy release in admin dashboard --- app/views/admin/dashboard/show.html.slim | 9 +++++---- config/locales/de.yml | 4 ++-- config/locales/en.yml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/views/admin/dashboard/show.html.slim b/app/views/admin/dashboard/show.html.slim index a4fd9102..9bf07936 100644 --- a/app/views/admin/dashboard/show.html.slim +++ b/app/views/admin/dashboard/show.html.slim @@ -13,14 +13,15 @@ div.mb-4 = "CodeOcean Release:" pre = Sentry.configuration.release -- if DockerContainerPool.active? +- if Runner.management_active? div.mb-4 - = "DockerContainerPool Release:" - pre = DockerContainerPool.dump_info['release'] + = Runner.strategy_class.name.demodulize + =< "Release:" + pre = Runner.strategy_class.release h2 Docker -- if DockerContainerPool.active? +- if Runner.management_active? h3 = t('.current') .table-responsive table.table diff --git a/config/locales/de.yml b/config/locales/de.yml index 4cf94e06..108da7e8 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -19,7 +19,7 @@ de: network_enabled: Netzwerkzugriff name: Name permitted_execution_time: Erlaubte Ausführungszeit (in Sekunden) - pool_size: Docker-Container-Pool-Größe + pool_size: Prewarming-Pool-Größe run_command: Ausführungsbefehl test_command: Testbefehl testing_framework: Testing-Framework @@ -240,7 +240,7 @@ de: show: current: Aktuelle Verfügbarkeit history: Verfügbarkeitsverlauf - inactive: Der Container-Pool ist nicht aktiv. + inactive: Es ist kein Runner Management aktiv. quantity: Verfügbare Container application: not_authorized: Sie Sind nicht berechtigt, diese Aktion auszuführen. diff --git a/config/locales/en.yml b/config/locales/en.yml index f9c1cfd9..44d0e8df 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -19,7 +19,7 @@ en: name: Name network_enabled: Network Enabled permitted_execution_time: Permitted Execution Time (in Seconds) - pool_size: Docker Container Pool Size + pool_size: Prewarming Pool Size run_command: Run Command test_command: Test Command testing_framework: Testing Framework @@ -240,7 +240,7 @@ en: show: current: Current Availability history: Availability History - inactive: Container pooling is not enabled. + inactive: No runner management is currently enabled. quantity: Available Containers application: not_authorized: You are not authorized to perform this action. From 288c7693f78be4cd15a5b2bf856e259b8e3eab60 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 13:01:00 +0200 Subject: [PATCH 121/156] Remove dump_docker method for admins --- app/controllers/admin/dashboard_controller.rb | 8 -------- app/policies/admin/dashboard_policy.rb | 3 --- 2 files changed, 11 deletions(-) diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index f509bd53..a8ce65ea 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -15,13 +15,5 @@ module Admin format.json { render(json: dashboard_data) } end end - - def dump_docker - authorize(self) - respond_to do |format| - format.html { render(json: DockerContainerPool.dump_info) } - format.json { render(json: DockerContainerPool.dump_info) } - end - end end end diff --git a/app/policies/admin/dashboard_policy.rb b/app/policies/admin/dashboard_policy.rb index d8c6ef49..d63d2954 100644 --- a/app/policies/admin/dashboard_policy.rb +++ b/app/policies/admin/dashboard_policy.rb @@ -2,8 +2,5 @@ module Admin class DashboardPolicy < AdminOnlyPolicy - def dump_docker? - admin? - end end end From 28c74bc9a59aac6e6650ce7fd9b3da79c350c635 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 24 Oct 2021 13:01:17 +0200 Subject: [PATCH 122/156] Improve memoization of @strategy_class --- app/models/runner.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index fbe317cd..4931db8e 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -11,8 +11,10 @@ class Runner < ApplicationRecord attr_accessor :strategy def self.strategy_class - strategy_name = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] - @strategy_class ||= "runner/strategy/#{strategy_name}".camelize.constantize + @strategy_class ||= begin + strategy_name = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] + "runner/strategy/#{strategy_name}".camelize.constantize + end end def self.management_active? From 953643f05eea72463b4bc50ad783c19e2ba37f70 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 25 Oct 2021 17:30:39 +0200 Subject: [PATCH 123/156] [Spec] Use strings for image names --- spec/controllers/execution_environments_controller_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index cfa94fb3..08e1f67d 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -111,7 +111,7 @@ describe ExecutionEnvironmentsController do describe '#set_docker_images' do context 'when Docker is available' do - let(:docker_images) { [1, 2, 3] } + let(:docker_images) { %w[image:one image:two image:three] } before do allow(Runner).to receive(:strategy_class).and_return Runner::Strategy::DockerContainerPool From c1cff2914717911b11a5583174dc19224a0e26a7 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 25 Oct 2021 17:31:01 +0200 Subject: [PATCH 124/156] [Spec] Initialize environment for Docker testing --- spec/lib/docker_client_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb index 54208a77..b4289488 100644 --- a/spec/lib/docker_client_spec.rb +++ b/spec/lib/docker_client_spec.rb @@ -14,6 +14,7 @@ describe DockerClient, docker: true do let(:workspace_path) { WORKSPACE_PATH } before do + described_class.initialize_environment allow(described_class).to receive(:container_creation_options).and_wrap_original do |original_method, *args, &block| result = original_method.call(*args, &block) result['NanoCPUs'] = 2 * 1_000_000_000 # CPU quota in units of 10^-9 CPUs. From e8c686ce756c99e532297781b39186a170ba685f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 25 Oct 2021 19:19:32 +0200 Subject: [PATCH 125/156] [Spec] Clean seed_secs with truncation --- spec/db/seeds_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/db/seeds_spec.rb b/spec/db/seeds_spec.rb index 4e3944bc..7c2191db 100644 --- a/spec/db/seeds_spec.rb +++ b/spec/db/seeds_spec.rb @@ -18,7 +18,7 @@ describe 'seeds' do } end - describe 'execute db:seed' do + describe 'execute db:seed', cleaning_strategy: :truncation do it 'collects the test results' do expect { seed }.not_to raise_error end From 25b007dfda1dcb321f59623b5d188881adf12bb4 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 25 Oct 2021 21:13:30 +0200 Subject: [PATCH 126/156] [Spec] Fix return value of image_tags for mocked DockerClient --- spec/support/docker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/docker.rb b/spec/support/docker.rb index d4c35e0d..5d86d951 100644 --- a/spec/support/docker.rb +++ b/spec/support/docker.rb @@ -9,7 +9,7 @@ RSpec.configure do |config| allow(DockerClient).to receive(:check_availability!).and_return(true) allow(DockerClient).to receive(:create_container).and_return(CONTAINER) allow(DockerClient).to receive(:find_image_by_tag).and_return(IMAGE) - allow(DockerClient).to receive(:image_tags).and_return([IMAGE]) + allow(DockerClient).to receive(:image_tags).and_return([IMAGE.info['RepoTags']].flatten) allow(DockerClient).to receive(:local_workspace_path).and_return(Dir.mktmpdir) allow_any_instance_of(DockerClient).to receive(:send_command).and_return({}) allow_any_instance_of(ExecutionEnvironment).to receive(:working_docker_image?) From 9d833e37b32021479825a7beda54d41a6e23cc26 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 00:13:29 +0200 Subject: [PATCH 127/156] Use Net::HTTP::Persistent for requests to Poseidon --- Gemfile | 1 + Gemfile.lock | 6 +++++- lib/runner/strategy/poseidon.rb | 21 +++++++++++---------- spec/lib/runner/strategy/poseidon_spec.rb | 13 ++++++------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Gemfile b/Gemfile index 960e1146..43aa0d60 100644 --- a/Gemfile +++ b/Gemfile @@ -20,6 +20,7 @@ gem 'json_schemer' gem 'js-routes' gem 'kramdown' gem 'mimemagic' +gem 'net-http-persistent' gem 'nokogiri' gem 'pagedown-bootstrap-rails' gem 'pg' diff --git a/Gemfile.lock b/Gemfile.lock index ba6772b4..6432b7f6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -140,6 +140,7 @@ GEM chronic (0.10.2) coderay (1.1.3) concurrent-ruby (1.1.9) + connection_pool (2.2.5) crack (0.4.5) rexml crass (1.0.6) @@ -279,6 +280,8 @@ GEM multi_xml (0.6.0) multipart-post (2.1.1) nested_form (0.3.2) + net-http-persistent (4.0.0) + connection_pool (~> 2.2) netrc (0.11.0) newrelic_rpm (8.1.0) nio4r (2.5.8) @@ -574,6 +577,7 @@ DEPENDENCIES listen mimemagic mnemosyne-ruby + net-http-persistent newrelic_rpm nokogiri nyan-cat-formatter @@ -620,4 +624,4 @@ DEPENDENCIES whenever BUNDLED WITH - 2.2.23 + 2.2.29 diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 88562759..781a4b2a 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -21,8 +21,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy def self.sync_environment(environment) url = "#{config[:url]}/execution-environments/#{environment.id}" - connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} - response = connection.put url, environment.to_json, headers + response = http_connection.put url, environment.to_json return true if [201, 204].include? response.status Rails.logger.warn("Could not create execution environment in Poseidon, got response: #{response.as_json}") @@ -39,8 +38,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy inactivityTimeout: config[:unused_runner_expiration_time].seconds, } Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } - connection = Faraday.new nil, ssl: {ca_file: config[:ca_file]} - response = connection.post url, body.to_json, headers + response = http_connection.post url, body.to_json case response.status when 200 @@ -60,8 +58,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy def destroy_at_management Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{runner_url}" } - connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.delete runner_url, nil, self.class.headers + response = self.class.http_connection.delete runner_url self.class.handle_error response unless response.status == 204 rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to Poseidon failed: #{e.inspect}") @@ -83,8 +80,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy # First, clean the workspace and second, copy all files to their location. # This ensures that no artefacts from a previous submission remain in the workspace. body = {copy: copy, delete: ['./']} - connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.patch url, body.to_json, self.class.headers + response = self.class.http_connection.patch url, body.to_json return if response.status == 204 Runner.destroy(@allocation_id) if response.status == 400 @@ -154,6 +150,12 @@ class Runner::Strategy::Poseidon < Runner::Strategy @headers ||= {'Content-Type' => 'application/json', 'Poseidon-Token' => config[:token]} end + def self.http_connection + @http_connection ||= Faraday.new(ssl: {ca_file: config[:ca_file]}, headers: headers) do |faraday| + faraday.adapter :net_http_persistent + end + end + def self.parse(response) JSON.parse(response.body).deep_symbolize_keys rescue JSON::ParserError => e @@ -167,8 +169,7 @@ class Runner::Strategy::Poseidon < Runner::Strategy url = "#{runner_url}/execute" body = {command: command, timeLimit: @execution_environment.permitted_execution_time} Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Preparing command execution at #{url}: #{command}" } - connection = Faraday.new nil, ssl: {ca_file: self.class.config[:ca_file]} - response = connection.post url, body.to_json, self.class.headers + response = self.class.http_connection.post url, body.to_json case response.status when 200 diff --git a/spec/lib/runner/strategy/poseidon_spec.rb b/spec/lib/runner/strategy/poseidon_spec.rb index f81670da..cec90a31 100644 --- a/spec/lib/runner/strategy/poseidon_spec.rb +++ b/spec/lib/runner/strategy/poseidon_spec.rb @@ -109,7 +109,7 @@ describe Runner::Strategy::Poseidon do it 'raises an error' do faraday_connection = instance_double 'Faraday::Connection' - allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(described_class).to receive(:http_connection).and_return(faraday_connection) %i[post patch delete].each {|message| allow(faraday_connection).to receive(message).and_raise(Faraday::TimeoutError) } expect { action.call }.to raise_error(Runner::Error::FaradayError) end @@ -122,20 +122,19 @@ describe Runner::Strategy::Poseidon do it 'makes the correct request to Poseidon' do faraday_connection = instance_double 'Faraday::Connection' - allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(described_class).to receive(:http_connection).and_return(faraday_connection) allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: 201)) action.call - expect(faraday_connection).to have_received(:put) do |url, body, headers| + expect(faraday_connection).to have_received(:put) do |url, body| expect(url).to match(%r{execution-environments/#{execution_environment.id}\z}) expect(body).to eq(execution_environment.to_json) - expect(headers).to include({'Content-Type' => 'application/json'}) end end shared_examples 'returns true when the api request was successful' do |status| it "returns true on status #{status}" do faraday_connection = instance_double 'Faraday::Connection' - allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(described_class).to receive(:http_connection).and_return(faraday_connection) allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: status)) expect(action.call).to be_truthy end @@ -144,7 +143,7 @@ describe Runner::Strategy::Poseidon do shared_examples 'returns false when the api request failed' do |status| it "returns false on status #{status}" do faraday_connection = instance_double 'Faraday::Connection' - allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(described_class).to receive(:http_connection).and_return(faraday_connection) allow(faraday_connection).to receive(:put).and_return(Faraday::Response.new(status: status)) expect(action.call).to be_falsey end @@ -160,7 +159,7 @@ describe Runner::Strategy::Poseidon do it 'returns false if Faraday raises an error' do faraday_connection = instance_double 'Faraday::Connection' - allow(Faraday).to receive(:new).and_return(faraday_connection) + allow(described_class).to receive(:http_connection).and_return(faraday_connection) allow(faraday_connection).to receive(:put).and_raise(Faraday::TimeoutError) expect(action.call).to be_falsey end From 2551ea709b8009465bc5538beb20c2fec05b2b60 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 20:05:28 +0200 Subject: [PATCH 128/156] Remove 'previous' DockerContainerPool implementation --- lib/docker_client.rb | 41 +++--------------- lib/docker_container_mixin.rb | 2 +- lib/docker_container_pool.rb | 78 ---------------------------------- spec/lib/docker_client_spec.rb | 12 +++--- 4 files changed, 12 insertions(+), 121 deletions(-) delete mode 100644 lib/docker_container_pool.rb diff --git a/lib/docker_client.rb b/lib/docker_client.rb index 0517ff69..b134fa1e 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -119,13 +119,11 @@ class DockerClient container.start_time = Time.zone.now container.status = :created container.execution_environment = execution_environment - container.re_use = true container.docker_client = new(execution_environment: execution_environment) Thread.new do timeout = Random.rand(MINIMUM_CONTAINER_LIFETIME..MAXIMUM_CONTAINER_LIFETIME) # seconds sleep(timeout) - container.re_use = false if container.status == :executing Thread.new do timeout = SELF_DESTROY_GRACE_PERIOD.to_i @@ -227,7 +225,7 @@ class DockerClient Rails.logger.info("destroying container #{container}") # Checks only if container assignment is not nil and not whether the container itself is still present. - if container && !DockerContainerPool.active? + if container container.kill container.port_bindings.each_value {|port| PortPool.release(port) } begin @@ -240,8 +238,6 @@ class DockerClient # Checks only if container assignment is not nil and not whether the container itself is still present. container&.delete(force: true, v: true) - elsif container - DockerContainerPool.destroy_container(container) end rescue Docker::Error::NotFoundError => e Rails.logger.error("destroy_container: Rescued from Docker::Error::NotFoundError: #{e}") @@ -261,7 +257,7 @@ class DockerClient def execute_command(command, before_execution_block, output_consuming_block) # tries ||= 0 container_request_time = Time.zone.now - @container = DockerContainerPool.get_container(@execution_environment) + @container = self.class.create_container(@execution_environment) waiting_for_container_time = Time.zone.now - container_request_time if @container @container.status = :executing @@ -285,7 +281,7 @@ container_execution_time: nil} # called when the user clicks the "Run" button def open_websocket_connection(command, before_execution_block, _output_consuming_block) - @container = DockerContainerPool.get_container(@execution_environment) + @container = self.class.create_container(@execution_environment) if @container @container.status = :executing # do not use try here, directly call the passed proc and rescue from the error in order to log the problem. @@ -351,13 +347,7 @@ container_execution_time: nil} # exit the timeout thread if it is still alive exit_thread_if_alive @socket.close - # if we use pooling and recylce the containers, put it back. otherwise, destroy it. - if DockerContainerPool.active? && RECYCLE_CONTAINERS - self.class.return_container(container, - @execution_environment) - else - self.class.destroy_container(container) - end + self.class.destroy_container(container) end def kill_container(container) @@ -413,7 +403,6 @@ container_execution_time: nil} end def self.initialize_environment - # TODO: Move to DockerContainerPool raise Error.new('Docker configuration missing!') unless config[:connection_timeout] && config[:workspace_root] Docker.url = config[:host] if config[:host] @@ -455,21 +444,6 @@ container_execution_time: nil} `docker pull #{docker_image}` if docker_image end - def self.return_container(container, execution_environment) - Rails.logger.debug { "returning container #{container}" } - begin - clean_container_workspace(container) - rescue Docker::Error::NotFoundError => e - # FIXME: Create new container? - Rails.logger.info("return_container: Rescued from Docker::Error::NotFoundError: #{e}") - Rails.logger.info('Nothing is done here additionally. The container will be exchanged upon its next retrieval.') - end - DockerContainerPool.return_container(container, execution_environment) - container.status = :available - end - - # private :return_container - def send_command(command, container) result = {status: :failed, stdout: '', stderr: ''} output = nil @@ -489,12 +463,7 @@ container_execution_time: nil} result = {status: (output[2])&.zero? ? :ok : :failed, stdout: output[0].join.force_encoding('utf-8'), stderr: output[1].join.force_encoding('utf-8')} end - # if we use pooling and recylce the containers, put it back. otherwise, destroy it. - if DockerContainerPool.active? && RECYCLE_CONTAINERS - self.class.return_container(container, @execution_environment) - else - self.class.destroy_container(container) - end + self.class.destroy_container(container) result rescue Timeout::Error Rails.logger.info("got timeout error for container #{container}") diff --git a/lib/docker_container_mixin.rb b/lib/docker_container_mixin.rb index 83b77865..d2785263 100644 --- a/lib/docker_container_mixin.rb +++ b/lib/docker_container_mixin.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module DockerContainerMixin - attr_accessor :start_time, :status, :re_use, :execution_environment, :docker_client + attr_accessor :start_time, :status, :execution_environment, :docker_client def binds host_config['Binds'] diff --git a/lib/docker_container_pool.rb b/lib/docker_container_pool.rb deleted file mode 100644 index a89bd461..00000000 --- a/lib/docker_container_pool.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -# get_container, destroy_container was moved to lib/runner/strategy/docker_container_pool.rb. -# return_container is not used anymore because runners are not shared between users anymore. -# create_container is done by the DockerContainerPool. -# dump_info and quantities are still in use. - -class DockerContainerPool - def self.active? - # TODO: Refactor config and merge with code_ocean.yml - config[:active] && Runner.management_active? && Runner.strategy_class == Runner::Strategy::DockerContainerPool - end - - def self.config - # TODO: Why erb? - @config ||= CodeOcean::Config.new(:docker).read(erb: true)[:pool] - end - - def self.create_container(execution_environment) - Rails.logger.info("trying to create container for execution environment: #{execution_environment}") - container = DockerClient.create_container(execution_environment) - container.status = 'available' # FIXME: String vs Symbol usage? - # Rails.logger.debug('created container ' + container.to_s + ' for execution environment ' + execution_environment.to_s) - container - rescue StandardError => e - Sentry.set_extras({container: container.inspect, execution_environment: execution_environment.inspect, - config: config.inspect}) - Sentry.capture_exception(e) - nil - end - - # not in use because DockerClient::RECYCLE_CONTAINERS == false - def self.return_container(container, execution_environment) - Faraday.get("#{config[:location]}/docker_container_pool/return_container/#{container.id}") - rescue StandardError => e - Sentry.set_extras({container: container.inspect, execution_environment: execution_environment.inspect, - config: config.inspect}) - Sentry.capture_exception(e) - nil - end - - def self.get_container(execution_environment) - # if pooling is active, do pooling, otherwise just create an container and return it - if active? - begin - container_id = JSON.parse(Faraday.get("#{config[:location]}/docker_container_pool/get_container/#{execution_environment.id}").body)['id'] - Docker::Container.get(container_id) if container_id.present? - rescue StandardError => e - Sentry.set_extras({container_id: container_id.inspect, execution_environment: execution_environment.inspect, - config: config.inspect}) - Sentry.capture_exception(e) - nil - end - else - create_container(execution_environment) - end - end - - def self.destroy_container(container) - Faraday.get("#{config[:location]}/docker_container_pool/destroy_container/#{container.id}") - end - - def self.quantities - response = JSON.parse(Faraday.get("#{config[:location]}/docker_container_pool/quantities").body) - response.transform_keys(&:to_i) - rescue StandardError => e - Sentry.set_extras({response: response.inspect}) - Sentry.capture_exception(e) - [] - end - - def self.dump_info - JSON.parse(Faraday.get("#{config[:location]}/docker_container_pool/dump_info").body) - rescue StandardError => e - Sentry.capture_exception(e) - nil - end -end diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb index b4289488..45dfdfca 100644 --- a/spec/lib/docker_client_spec.rb +++ b/spec/lib/docker_client_spec.rb @@ -204,8 +204,8 @@ describe DockerClient, docker: true do describe '#execute_arbitrary_command' do let(:execute_arbitrary_command) { docker_client.execute_arbitrary_command(command) } - it 'takes a container from the pool' do - expect(DockerContainerPool).to receive(:get_container).and_call_original + it 'creates a new container' do + expect(described_class).to receive(:create_container).and_call_original execute_arbitrary_command end @@ -248,8 +248,8 @@ describe DockerClient, docker: true do after { docker_client.send(:execute_run_command, submission, filename) } - it 'takes a container from the pool' do - expect(DockerContainerPool).to receive(:get_container).with(submission.execution_environment).and_call_original + it 'creates a new container' do + expect(described_class).to receive(:create_container).with(submission.execution_environment).and_call_original end it 'creates the workspace files' do @@ -268,8 +268,8 @@ describe DockerClient, docker: true do after { docker_client.send(:execute_test_command, submission, filename) } - it 'takes a container from the pool' do - expect(DockerContainerPool).to receive(:get_container).with(submission.execution_environment).and_call_original + it 'creates a new container' do + expect(described_class).to receive(:create_container).with(submission.execution_environment).and_call_original end it 'creates the workspace files' do From b13a3b084d3f38778e51b2e079aaffafbc4fd98c Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 22:32:15 +0200 Subject: [PATCH 129/156] Use new available_images routes from DCP --- lib/runner/strategy/docker_container_pool.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index fdfe945a..977fe02d 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -88,10 +88,16 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.available_images - DockerClient.check_availability! - DockerClient.image_tags - rescue DockerClient::Error => e - raise Runner::Error::InternalServerError.new(e.message) + url = "#{config[:pool][:location]}/docker_container_pool/available_images" + response = Faraday.get(url) + json = JSON.parse(response.body) + return json if response.success? + + raise Runner::Error::InternalServerError.new("DockerContainerPool returned: #{json['error']}") + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + rescue JSON::ParserError => e + raise Runner::Error::UnexpectedResponse.new("DockerContainerPool returned invalid JSON: #{e.inspect}") end def self.config From 20064b07154bc1411f2b5745dd5b98120b7dc1ec Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 22:32:55 +0200 Subject: [PATCH 130/156] DockerClient: Cleanup usage of config --- lib/docker_client.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/docker_client.rb b/lib/docker_client.rb index b134fa1e..af8667f3 100644 --- a/lib/docker_client.rb +++ b/lib/docker_client.rb @@ -55,7 +55,6 @@ class DockerClient { 'Image' => find_image_by_tag(execution_environment.docker_image).info['RepoTags'].first, 'NetworkDisabled' => !execution_environment.network_enabled?, - # DockerClient.config['allowed_cpus'] 'OpenStdin' => true, 'StdinOnce' => true, # required to expose standard streams over websocket @@ -83,7 +82,7 @@ class DockerClient # Headers are required by Docker headers = {'Origin' => 'http://localhost'} - socket_url = "#{DockerClient.config['ws_host']}/v1.27/containers/#{@container.id}/attach/ws?#{query_params}" + socket_url = "#{self.class.config['ws_host']}/v1.27/containers/#{@container.id}/attach/ws?#{query_params}" # The ping value is measured in seconds and specifies how often a Ping frame should be sent. # Internally, Faye::WebSocket uses EventMachine and the ping value is used to wake the EventMachine thread socket = Faye::WebSocket::Client.new(socket_url, [], headers: headers, ping: 0.1) From 01ec9343cf55e2266f090bdb56b0b7e9d662363f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 22:33:35 +0200 Subject: [PATCH 131/156] Remove usage of DockerClient --- lib/runner/strategy/docker_container_pool.rb | 4 +- .../execution_environments_controller_spec.rb | 10 +--- .../submissions_controller_spec.rb | 2 - spec/db/seeds_spec.rb | 1 + spec/features/factories_spec.rb | 2 +- spec/lib/docker_client_spec.rb | 48 ++++++++++++++----- spec/lib/docker_container_mixin_spec.rb | 10 ++-- spec/models/execution_environment_spec.rb | 21 ++++---- spec/support/docker.rb | 27 ----------- 9 files changed, 60 insertions(+), 65 deletions(-) delete mode 100644 spec/support/docker.rb diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 977fe02d..72b76d92 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -9,7 +9,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.initialize_environment - DockerClient.initialize_environment unless Rails.env.test? && `which docker`.blank? + raise Error.new('Docker configuration missing!') unless config[:host] && config[:workspace_root] + + FileUtils.mkdir_p(File.expand_path(config[:workspace_root])) end def self.sync_environment(_environment) diff --git a/spec/controllers/execution_environments_controller_spec.rb b/spec/controllers/execution_environments_controller_spec.rb index 08e1f67d..3d4a3896 100644 --- a/spec/controllers/execution_environments_controller_spec.rb +++ b/spec/controllers/execution_environments_controller_spec.rb @@ -12,8 +12,6 @@ describe ExecutionEnvironmentsController do end describe 'POST #create' do - before { allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) } - context 'with a valid execution environment' do let(:perform_request) { proc { post :create, params: {execution_environment: FactoryBot.build(:ruby).attributes} } } @@ -61,7 +59,6 @@ describe ExecutionEnvironmentsController do describe 'GET #edit' do before do - allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) get :edit, params: {id: execution_environment.id} end @@ -99,7 +96,6 @@ describe ExecutionEnvironmentsController do describe 'GET #new' do before do - allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) get :new end @@ -115,8 +111,7 @@ describe ExecutionEnvironmentsController do before do allow(Runner).to receive(:strategy_class).and_return Runner::Strategy::DockerContainerPool - allow(DockerClient).to receive(:check_availability!).at_least(:once) - allow(DockerClient).to receive(:image_tags).and_return(docker_images) + allow(Runner::Strategy::DockerContainerPool).to receive(:available_images).and_return(docker_images) controller.send(:set_docker_images) end @@ -128,7 +123,7 @@ describe ExecutionEnvironmentsController do before do allow(Runner).to receive(:strategy_class).and_return Runner::Strategy::DockerContainerPool - allow(DockerClient).to receive(:check_availability!).at_least(:once).and_raise(DockerClient::Error.new(error_message)) + allow(Runner::Strategy::DockerContainerPool).to receive(:available_images).and_raise(Runner::Error::InternalServerError.new(error_message)) controller.send(:set_docker_images) end @@ -168,7 +163,6 @@ describe ExecutionEnvironmentsController do describe 'PUT #update' do context 'with a valid execution environment' do before do - allow(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) allow(controller).to receive(:sync_to_runner_management).and_return(nil) put :update, params: {execution_environment: FactoryBot.attributes_for(:ruby), id: execution_environment.id} end diff --git a/spec/controllers/submissions_controller_spec.rb b/spec/controllers/submissions_controller_spec.rb index a176fdc9..b92f92d8 100644 --- a/spec/controllers/submissions_controller_spec.rb +++ b/spec/controllers/submissions_controller_spec.rb @@ -156,7 +156,6 @@ describe SubmissionsController do context 'when no errors occur during execution' do before do - allow_any_instance_of(DockerClient).to receive(:execute_run_command).with(submission, filename).and_return({}) perform_request end @@ -219,7 +218,6 @@ describe SubmissionsController do let(:output) { {} } before do - allow_any_instance_of(DockerClient).to receive(:execute_test_command).with(submission, filename) get :test, params: {filename: filename, id: submission.id} end diff --git a/spec/db/seeds_spec.rb b/spec/db/seeds_spec.rb index 7c2191db..07ab8e4f 100644 --- a/spec/db/seeds_spec.rb +++ b/spec/db/seeds_spec.rb @@ -16,6 +16,7 @@ describe 'seeds' do allow(ActiveRecord::Base).to receive(:establish_connection).with(:development) { ActiveRecord::Base.establish_connection(:test) } + allow_any_instance_of(ExecutionEnvironment).to receive(:working_docker_image?).and_return true end describe 'execute db:seed', cleaning_strategy: :truncation do diff --git a/spec/features/factories_spec.rb b/spec/features/factories_spec.rb index 4b65cc3d..21f4fbd1 100644 --- a/spec/features/factories_spec.rb +++ b/spec/features/factories_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe 'Factories' do - it 'are all valid', docker: true, permitted_execution_time: 30 do + it 'are all valid', permitted_execution_time: 30 do expect { FactoryBot.lint }.not_to raise_error end end diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb index 45dfdfca..70f56eea 100644 --- a/spec/lib/docker_client_spec.rb +++ b/spec/lib/docker_client_spec.rb @@ -5,7 +5,7 @@ require 'seeds_helper' WORKSPACE_PATH = Rails.root.join('tmp', 'files', Rails.env, 'code_ocean_test') -describe DockerClient, docker: true do +describe DockerClient do let(:command) { 'whoami' } let(:docker_client) { described_class.new(execution_environment: FactoryBot.build(:java), user: FactoryBot.build(:admin)) } let(:execution_environment) { FactoryBot.build(:java) } @@ -71,13 +71,15 @@ describe DockerClient, docker: true do it 'uses the correct Docker image' do expect(described_class).to receive(:find_image_by_tag).with(execution_environment.docker_image).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end it 'creates a unique directory' do expect(described_class).to receive(:generate_local_workspace_path).and_call_original expect(FileUtils).to receive(:mkdir).with(kind_of(String)).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end it 'creates a container' do @@ -91,22 +93,26 @@ describe DockerClient, docker: true do result end expect(Docker::Container).to receive(:create).with(kind_of(Hash)).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end it 'starts the container' do expect_any_instance_of(Docker::Container).to receive(:start).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end it 'configures mapped directories' do expect(described_class).to receive(:mapped_directories).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end it 'configures mapped ports' do expect(described_class).to receive(:mapped_ports).with(execution_environment).and_call_original - create_container + container = create_container + described_class.destroy_container(container) end context 'when an error occurs' do @@ -118,7 +124,9 @@ describe DockerClient, docker: true do end it 'retries to create a container' do - expect(create_container).to be_a(Docker::Container) + container = create_container + expect(container).to be_a(Docker::Container) + described_class.destroy_container(container) end end @@ -162,6 +170,7 @@ describe DockerClient, docker: true do end describe '#create_workspace_file' do + let(:container) { Docker::Container.send(:new, Docker::Connection.new('http://example.org', {}), 'id' => SecureRandom.hex) } let(:file) { FactoryBot.build(:file, content: 'puts 42') } let(:file_path) { File.join(workspace_path, file.name_with_extension) } @@ -170,7 +179,7 @@ describe DockerClient, docker: true do it 'creates a file' do expect(described_class).to receive(:local_workspace_path).at_least(:once).and_return(workspace_path) FileUtils.mkdir_p(workspace_path) - docker_client.send(:create_workspace_file, container: CONTAINER, file: file) + docker_client.send(:create_workspace_file, container: container, file: file) expect(File.exist?(file_path)).to be true expect(File.new(file_path, 'r').read).to eq(file.content) end @@ -197,13 +206,15 @@ describe DockerClient, docker: true do end it 'deletes the container' do - expect(container).to receive(:delete).with(force: true, v: true) + expect(container).to receive(:delete).with(force: true, v: true).and_call_original end end describe '#execute_arbitrary_command' do let(:execute_arbitrary_command) { docker_client.execute_arbitrary_command(command) } + after { described_class.destroy_container(docker_client.container) } + it 'creates a new container' do expect(described_class).to receive(:create_container).and_call_original execute_arbitrary_command @@ -246,7 +257,10 @@ describe DockerClient, docker: true do describe '#execute_run_command' do let(:filename) { submission.exercise.files.detect {|file| file.role == 'main_file' }.name_with_extension } - after { docker_client.send(:execute_run_command, submission, filename) } + after do + docker_client.send(:execute_run_command, submission, filename) + described_class.destroy_container(docker_client.container) + end it 'creates a new container' do expect(described_class).to receive(:create_container).with(submission.execution_environment).and_call_original @@ -266,7 +280,10 @@ describe DockerClient, docker: true do describe '#execute_test_command' do let(:filename) { submission.exercise.files.detect {|file| file.role == 'teacher_defined_test' || file.role == 'teacher_defined_linter' }.name_with_extension } - after { docker_client.send(:execute_test_command, submission, filename) } + after do + docker_client.send(:execute_test_command, submission, filename) + described_class.destroy_container(docker_client.container) + end it 'creates a new container' do expect(described_class).to receive(:create_container).with(submission.execution_environment).and_call_original @@ -314,6 +331,8 @@ describe DockerClient, docker: true do let(:container) { described_class.create_container(execution_environment) } let(:local_workspace_path) { described_class.local_workspace_path(container) } + after { described_class.destroy_container(container) } + it 'returns a path' do expect(local_workspace_path).to be_a(Pathname) end @@ -358,7 +377,10 @@ describe DockerClient, docker: true do let(:container) { described_class.create_container(execution_environment) } let(:send_command) { docker_client.send(:send_command, command, container, &block) } - after { send_command } + after do + send_command + described_class.destroy_container(container) + end it 'limits the execution time' do expect(Timeout).to receive(:timeout).at_least(:once).with(kind_of(Numeric)).and_call_original diff --git a/spec/lib/docker_container_mixin_spec.rb b/spec/lib/docker_container_mixin_spec.rb index 8faaf8da..6f9267e0 100644 --- a/spec/lib/docker_container_mixin_spec.rb +++ b/spec/lib/docker_container_mixin_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' describe DockerContainerMixin do + let(:container) { Docker::Container.send(:new, Docker::Connection.new('http://example.org', {}), 'id' => SecureRandom.hex) } + describe '#binds' do let(:binds) { [] } @@ -11,8 +13,8 @@ describe DockerContainerMixin do end it 'returns the correct information' do - allow(CONTAINER).to receive(:json).and_return('HostConfig' => {'Binds' => binds}) - expect(CONTAINER.binds).to eq(binds) + allow(container).to receive(:json).and_return('HostConfig' => {'Binds' => binds}) + expect(container.binds).to eq(binds) end end @@ -25,8 +27,8 @@ describe DockerContainerMixin do end it 'returns the correct information' do - allow(CONTAINER).to receive(:json).and_return('HostConfig' => {'PortBindings' => port_bindings}) - expect(CONTAINER.port_bindings).to eq(port => port) + allow(container).to receive(:json).and_return('HostConfig' => {'PortBindings' => port_bindings}) + expect(container.port_bindings).to eq(port => port) end end end diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index 09fd1c0a..2d7b1e48 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -5,10 +5,11 @@ require 'rails_helper' describe ExecutionEnvironment do let(:execution_environment) { described_class.create.tap {|execution_environment| execution_environment.update(network_enabled: nil) } } - it 'validates that the Docker image works', docker: true do + it 'validates that the Docker image works' do allow(execution_environment).to receive(:validate_docker_image?).and_return(true) - expect(execution_environment).to receive(:working_docker_image?) + allow(execution_environment).to receive(:working_docker_image?).and_return(true) execution_environment.update(docker_image: FactoryBot.attributes_for(:ruby)[:docker_image]) + expect(execution_environment).to have_received(:working_docker_image?) end it 'validates the presence of a Docker image name' do @@ -144,27 +145,29 @@ describe ExecutionEnvironment do end end - describe '#working_docker_image?', docker: true do + describe '#working_docker_image?' do let(:working_docker_image?) { execution_environment.send(:working_docker_image?) } + let(:runner) { instance_double 'runner' } - before { allow(DockerClient).to receive(:find_image_by_tag).and_return(Object.new) } + before do + allow(Runner).to receive(:for).with(execution_environment.author, execution_environment).and_return runner + end it 'instantiates a Runner' do - runner = instance_double 'runner' - allow(Runner).to receive(:for).with(execution_environment.author, execution_environment).and_return runner allow(runner).to receive(:execute_command).and_return({}) working_docker_image? expect(runner).to have_received(:execute_command).once end it 'executes the validation command' do - allow_any_instance_of(DockerClient).to receive(:execute_arbitrary_command).with(ExecutionEnvironment::VALIDATION_COMMAND).and_return({}) + allow(runner).to receive(:execute_command).and_return({}) working_docker_image? + expect(runner).to have_received(:execute_command).with(ExecutionEnvironment::VALIDATION_COMMAND, raise_exception: true) end context 'when the command produces an error' do it 'adds an error' do - allow_any_instance_of(DockerClient).to receive(:execute_arbitrary_command).and_return(stderr: 'command not found') + allow(runner).to receive(:execute_command).and_return(stderr: 'command not found') working_docker_image? expect(execution_environment.errors[:docker_image]).to be_present end @@ -172,7 +175,7 @@ describe ExecutionEnvironment do context 'when the Docker client produces an error' do it 'adds an error' do - allow_any_instance_of(DockerClient).to receive(:execute_arbitrary_command).and_raise(DockerClient::Error) + allow(runner).to receive(:execute_command).and_raise(Runner::Error) working_docker_image? expect(execution_environment.errors[:docker_image]).to be_present end diff --git a/spec/support/docker.rb b/spec/support/docker.rb deleted file mode 100644 index 5d86d951..00000000 --- a/spec/support/docker.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -CONTAINER = Docker::Container.send(:new, Docker::Connection.new('http://example.org', {}), 'id' => SecureRandom.hex) -IMAGE = Docker::Image.new(Docker::Connection.new('http://example.org', {}), 'id' => SecureRandom.hex, 'RepoTags' => [FactoryBot.attributes_for(:ruby)[:docker_image]]) - -RSpec.configure do |config| - config.before(:each) do |example| - unless example.metadata[:docker] - allow(DockerClient).to receive(:check_availability!).and_return(true) - allow(DockerClient).to receive(:create_container).and_return(CONTAINER) - allow(DockerClient).to receive(:find_image_by_tag).and_return(IMAGE) - allow(DockerClient).to receive(:image_tags).and_return([IMAGE.info['RepoTags']].flatten) - allow(DockerClient).to receive(:local_workspace_path).and_return(Dir.mktmpdir) - allow_any_instance_of(DockerClient).to receive(:send_command).and_return({}) - allow_any_instance_of(ExecutionEnvironment).to receive(:working_docker_image?) - end - end - - config.after(:suite) do - examples = RSpec.world.filtered_examples.values.flatten - has_docker_tests = examples.any? {|example| example.metadata[:docker] } - next unless has_docker_tests - - FileUtils.rm_rf(Rails.root.join('tmp/files/test')) - `which docker && test -n "$(docker ps --all --quiet)" && docker rm --force $(docker ps --all --quiet)` - end -end From 5550183d7e698c5353e5408809e431b961dc8df9 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 29 Oct 2021 22:34:31 +0200 Subject: [PATCH 132/156] [Spec] openhpi/co_execenv_python:3.4 is no longer required --- .github/workflows/ci.yml | 2 -- .gitlab-ci.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cebb0fc..c40be66a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - co_execenv_python: - image: openhpi/co_execenv_python:3.4 co_execenv_java: image: openhpi/co_execenv_java:8 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 516bf58a..77c3b3ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,6 @@ rspec: - rake db:schema:load - rake db:migrate - docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_PASS}" - - docker pull openhpi/co_execenv_python:3.4 - docker pull openhpi/co_execenv_java:8 script: - rspec --format progress From 4f1a7cde27a69a94d95cb66bcee2061e8d772b21 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 01:12:16 +0200 Subject: [PATCH 133/156] Add null strategy for runners * This is the default strategy used when the runner management is disabled. It might be replaced with a generic Docker strategy in the future (without pooling). For now, it allows normal "operation" of CodeOcean without any runner management. However, as no runner system is configured, no command can be executed. --- app/models/runner.rb | 10 ++++--- lib/runner/strategy/null.rb | 57 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 lib/runner/strategy/null.rb diff --git a/app/models/runner.rb b/app/models/runner.rb index 4931db8e..21af9d96 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -11,10 +11,12 @@ class Runner < ApplicationRecord attr_accessor :strategy def self.strategy_class - @strategy_class ||= begin - strategy_name = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] - "runner/strategy/#{strategy_name}".camelize.constantize - end + @strategy_class ||= if Runner.management_active? + strategy_name = CodeOcean::Config.new(:code_ocean).read[:runner_management][:strategy] + "runner/strategy/#{strategy_name}".camelize.constantize + else + Runner::Strategy::Null + end end def self.management_active? diff --git a/lib/runner/strategy/null.rb b/lib/runner/strategy/null.rb new file mode 100644 index 00000000..38ff522f --- /dev/null +++ b/lib/runner/strategy/null.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +# This strategy allows normal operation of CodeOcean even when the runner management is disabled. +# However, as no command can be executed, all execution requests will fail. +class Runner::Strategy::Null < Runner::Strategy + def self.initialize_environment; end + + def self.sync_environment(_environment); end + + def self.request_from_management(_environment) + SecureRandom.uuid + end + + def destroy_at_management; end + + def copy_files(_files); end + + def attach_to_execution(command, event_loop) + socket = Connection.new(nil, self, event_loop) + # We don't want to return an error if the execution environment is changed + socket.status = :terminated_by_codeocean if command == ExecutionEnvironment::VALIDATION_COMMAND + yield(socket) + socket + end + + def self.available_images + [] + end + + def self.config; end + + def self.release + 'N/A' + end + + def self.pool_size + {} + end + + def self.websocket_header + {} + end + + class Connection < Runner::Connection + def decode(event_data) + event_data + end + + def encode(data) + data + end + + def active? + false + end + end +end From 1609bd2e0eb1e80314384f114caaca02d3a43c14 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 01:13:32 +0200 Subject: [PATCH 134/156] Change default of raise_exception for execute_command --- app/controllers/execution_environments_controller.rb | 4 ++-- app/models/execution_environment.rb | 2 +- app/models/runner.rb | 2 +- spec/models/execution_environment_spec.rb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 5b966fb1..9d714af6 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -31,8 +31,8 @@ class ExecutionEnvironmentsController < ApplicationController def execute_command runner = Runner.for(current_user, @execution_environment) - output = runner.execute_command(params[:command]) - render(json: output) + output = runner.execute_command(params[:command], raise_exception: false) + render json: output end def working_time_query diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 3bb86331..615f9ceb 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -79,7 +79,7 @@ class ExecutionEnvironment < ApplicationRecord def working_docker_image? runner = Runner.for(author, self) - output = runner.execute_command(VALIDATION_COMMAND, raise_exception: true) + output = runner.execute_command(VALIDATION_COMMAND) errors.add(:docker_image, "error: #{output[:stderr]}") if output[:stderr].present? rescue Runner::Error => e errors.add(:docker_image, "error: #{e}") diff --git a/app/models/runner.rb b/app/models/runner.rb index 21af9d96..63582b6a 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -64,7 +64,7 @@ class Runner < ApplicationRecord Time.zone.now - starting_time # execution duration end - def execute_command(command, raise_exception: false) + def execute_command(command, raise_exception: true) output = {} stdout = +'' stderr = +'' diff --git a/spec/models/execution_environment_spec.rb b/spec/models/execution_environment_spec.rb index 2d7b1e48..5b41f4b8 100644 --- a/spec/models/execution_environment_spec.rb +++ b/spec/models/execution_environment_spec.rb @@ -162,7 +162,7 @@ describe ExecutionEnvironment do it 'executes the validation command' do allow(runner).to receive(:execute_command).and_return({}) working_docker_image? - expect(runner).to have_received(:execute_command).with(ExecutionEnvironment::VALIDATION_COMMAND, raise_exception: true) + expect(runner).to have_received(:execute_command).with(ExecutionEnvironment::VALIDATION_COMMAND) end context 'when the command produces an error' do From 3c8017f23ee71da4e15914fbf1e066f69822d4d6 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 01:14:55 +0200 Subject: [PATCH 135/156] JS: Ensure to print status messages for score * If only one response is available, no array will be passed (but rather an Object). The impact of this has been tackled with the changes included --- app/assets/javascripts/editor/evaluation.js | 26 +++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/editor/evaluation.js b/app/assets/javascripts/editor/evaluation.js index 971e91ef..8f79ccf6 100644 --- a/app/assets/javascripts/editor/evaluation.js +++ b/app/assets/javascripts/editor/evaluation.js @@ -67,13 +67,31 @@ CodeOceanEditorEvaluation = { }, printScoringResults: function (response) { + response = (Array.isArray(response)) ? response : [response] + const test_results = response.filter(function(x) { + if (x === undefined || x === null) { + return false; + } + switch (x.file_role) { + case 'teacher_defined_test': + return true; + case 'teacher_defined_linter': + return true; + default: + return false; + } + }); + $('#results ul').first().html(''); - $('.test-count .number').html(response.filter(function(x) { return x && x.file_role === 'teacher_defined_test'; }).length); + $('.test-count .number').html(test_results.length); this.clearOutput(); - _.each(response, function (result, index) { - this.printOutput(result, false, index); - this.printScoringResult(result, index); + _.each(test_results, function (result, index) { + // based on https://stackoverflow.com/questions/8511281/check-if-a-value-is-an-object-in-javascript + if (result === Object(result)) { + this.printOutput(result, false, index); + this.printScoringResult(result, index); + } }.bind(this)); if (_.some(response, function (result) { From b62a7ad129562912dc79ee35950020bf041e32e4 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 11:23:10 +0200 Subject: [PATCH 136/156] Prevent non-existing runner_management config to be read --- app/models/runner.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index 63582b6a..c9772fc2 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -20,7 +20,14 @@ class Runner < ApplicationRecord end def self.management_active? - @management_active ||= CodeOcean::Config.new(:code_ocean).read[:runner_management][:enabled] + @management_active ||= begin + runner_management = CodeOcean::Config.new(:code_ocean).read[:runner_management] + if runner_management + runner_management[:enabled] + else + false + end + end end def self.for(user, execution_environment) From 570809bfe9d153395fdd3ed06c664f33c497bc8e Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 12:23:24 +0200 Subject: [PATCH 137/156] Allow whitespace for JSON exit --- lib/runner/connection.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 7a58bd8f..150c7cb5 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -90,7 +90,7 @@ class Runner::Connection def ignored_sequence?(event_data) case event_data - when "#exit\r", "{\"cmd\": \"exit\"}\r" + when /#exit\r/, /\s*{"cmd": "exit"}\r/ # Do not forward. We will wait for the confirmed exit sent by the runner management. true else From dfdec92c6e4a5e7f45e8a0e62ddf66a511a3f9f1 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 12:26:16 +0200 Subject: [PATCH 138/156] Use ping option only for DCP WebSocket * The Faye::WebSocket library will "buffer" some output of the connection and emit the `on :message` events in the order of the messages. However, when a ping is sent while the connection has already been closed, it will emit the `on :close` event immediately and drop all other messages (in that "buffer"). This is problematic for very short running executions that generate a long output (as this will be cut off without a proper exit message sent by Poseidon). --- lib/runner/connection.rb | 6 +----- lib/runner/strategy/docker_container_pool.rb | 6 +++++- lib/runner/strategy/poseidon.rb | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 150c7cb5..87f59aac 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -18,11 +18,7 @@ class Runner::Connection def initialize(url, strategy, event_loop, locale = I18n.locale) Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Opening connection to #{url}" } - # The `ping` value is measured in seconds and specifies how often a Ping frame should be sent. - # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread - # The `tls` option is used to customize the validation of TLS connections. - # Passing `nil` as a `root_cert_file` is okay and done so for the DockerContainerPool. - @socket = Faye::WebSocket::Client.new(url, [], strategy.class.websocket_header.merge(ping: 0.1)) + @socket = Faye::WebSocket::Client.new(url, [], strategy.class.websocket_header) @strategy = strategy @status = :established @event_loop = event_loop diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 72b76d92..fe45d23c 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -129,7 +129,11 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.websocket_header - {} + # The `ping` value is measured in seconds and specifies how often a Ping frame should be sent. + # Internally, Faye::WebSocket uses EventMachine and the `ping` value is used to wake the EventMachine thread + { + ping: 0.1, + } end private diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index 781a4b2a..cabda9f5 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -118,6 +118,8 @@ class Runner::Strategy::Poseidon < Runner::Strategy end def self.websocket_header + # The `tls` option is used to customize the validation of TLS connections. + # The `headers` option is used to pass the `Poseidon-Token` as part of the initial connection request. { tls: {root_cert_file: config[:ca_file]}, headers: {'Poseidon-Token' => config[:token]}, From 7bb2ef858829dd95c63e0246a5a639d711646295 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sat, 30 Oct 2021 14:22:45 +0200 Subject: [PATCH 139/156] DCP: Forward data before matching stdout termination --- lib/runner/strategy/docker_container_pool.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index fe45d23c..26cdcb17 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -183,14 +183,19 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def decode(event_data) case event_data - when /(@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/ - # TODO: The whole message line is kept back. If this contains the remaining buffer, this buffer is also lost. - # Example: A Java program prints `{` and then exists (with `#exit`). The `event_data` processed here is `{#exit` + when /(?.*)((root|python|user)@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/m + # The RegEx above is used to determine unwanted output which also indicates a program termination. + # If the RegEx matches, at least two capture groups will be created. + # The first (called `previous_data`) contains any data before the match (including multiple lines) + # while the second contains the unwanted output data. # Assume correct termination for now and return exit code 0 # TODO: Can we use the actual exit code here? @exit_code = 0 close(:terminated_by_codeocean) + + # The first capture group is forwarded + {'type' => @stream, 'data' => Regexp.last_match(:previous_data)} when /python3.*-m\s*unittest/ # TODO: Super dirty hack to redirect test output to stderr # This is only required for Python and the unittest module but must not be used with PyLint From 6209e25ee23444637b1b44fd21d9d798f0ae1259 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 01:00:59 +0200 Subject: [PATCH 140/156] DCP: Move pool location to code_ocean.yml --- config/code_ocean.yml.example | 4 ++-- config/docker.yml.erb.ci | 4 ---- config/docker.yml.erb.example | 4 ---- lib/runner/strategy/docker_container_pool.rb | 19 ++++++++++++------- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 3c8cb672..740b1598 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -49,8 +49,8 @@ default: &default # The authorization token for connections to the runner management (Poseidon only) # If TLS support is not enabled, this token is transmitted in clear text! token: SECRET - # The maximum time in seconds a runner may idle at the runner management before - # it is removed. Each interaction with the runner resets this time (Poseidon only) + # The maximum time in seconds a runner may idle at the runner management before + # it is removed. Each begin of an interaction with the runner resets this time unused_runner_expiration_time: 180 diff --git a/config/docker.yml.erb.ci b/config/docker.yml.erb.ci index 12415d0f..aaeceb14 100644 --- a/config/docker.yml.erb.ci +++ b/config/docker.yml.erb.ci @@ -3,7 +3,6 @@ default: &default connection_timeout: 3 pool: active: false - location: http://localhost:7100 ports: !ruby/range 4500..4600 development: @@ -12,7 +11,6 @@ development: ws_host: ws://127.0.0.1:2376 #url to connect rails server to docker host workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> pool: - location: http://localhost:7100 active: true refill: async: false @@ -26,7 +24,6 @@ production: host: unix:///var/run/docker.sock pool: active: true - location: http://localhost:3000 refill: async: false batch_size: 8 @@ -40,7 +37,6 @@ staging: host: unix:///var/run/docker.sock pool: active: true - location: http://localhost:3000 refill: async: false batch_size: 8 diff --git a/config/docker.yml.erb.example b/config/docker.yml.erb.example index bb41db98..aaeceb14 100644 --- a/config/docker.yml.erb.example +++ b/config/docker.yml.erb.example @@ -3,7 +3,6 @@ default: &default connection_timeout: 3 pool: active: false - location: http://localhost:7100 ports: !ruby/range 4500..4600 development: @@ -13,7 +12,6 @@ development: workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %> pool: active: true - location: http://localhost:7100 refill: async: false batch_size: 8 @@ -26,7 +24,6 @@ production: host: unix:///var/run/docker.sock pool: active: true - location: http://localhost:7100 refill: async: false batch_size: 8 @@ -40,7 +37,6 @@ staging: host: unix:///var/run/docker.sock pool: active: true - location: http://localhost:7100 refill: async: false batch_size: 8 diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 26cdcb17..f426ee2c 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -20,7 +20,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.request_from_management(environment) - url = "#{config[:pool][:location]}/docker_container_pool/get_container/#{environment.id}" + url = "#{config[:url]}/docker_container_pool/get_container/#{environment.id}" Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } response = Faraday.get url container_id = JSON.parse(response.body)['id'] @@ -34,7 +34,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def destroy_at_management - url = "#{self.class.config[:pool][:location]}/docker_container_pool/destroy_container/#{container.id}" + url = "#{self.class.config[:url]}/docker_container_pool/destroy_container/#{container.id}" Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{url}" } Faraday.get(url) rescue Faraday::Error => e @@ -90,7 +90,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.available_images - url = "#{config[:pool][:location]}/docker_container_pool/available_images" + url = "#{config[:url]}/docker_container_pool/available_images" response = Faraday.get(url) json = JSON.parse(response.body) return json if response.success? @@ -103,12 +103,17 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.config - # Since the docker configuration file contains code that must be executed, we use ERB templating. - @config ||= CodeOcean::Config.new(:docker).read(erb: true) + @config ||= begin + # Since the docker configuration file contains code that must be executed, we use ERB templating. + docker_config = CodeOcean::Config.new(:docker).read(erb: true) + codeocean_config = CodeOcean::Config.new(:code_ocean).read[:runner_management] || {} + # All keys in `docker_config` take precedence over those in `codeocean_config` + docker_config.merge codeocean_config + end end def self.release - url = "#{config[:pool][:location]}/docker_container_pool/dump_info" + url = "#{config[:url]}/docker_container_pool/dump_info" response = Faraday.get(url) JSON.parse(response.body)['release'] rescue Faraday::Error => e @@ -118,7 +123,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def self.pool_size - url = "#{config[:pool][:location]}/docker_container_pool/quantities" + url = "#{config[:url]}/docker_container_pool/quantities" response = Faraday.get(url) pool_size = JSON.parse(response.body) pool_size.transform_keys(&:to_i) From dcafbb9d46076d7525bd277082c973910faff4e1 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 01:01:50 +0200 Subject: [PATCH 141/156] DCP: Change HTTP verbs --- lib/runner/strategy/docker_container_pool.rb | 5 +++-- .../runner/strategy/docker_container_pool_spec.rb | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index f426ee2c..761bdba8 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -22,7 +22,8 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def self.request_from_management(environment) url = "#{config[:url]}/docker_container_pool/get_container/#{environment.id}" Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } - response = Faraday.get url + response = Faraday.post url, body + container_id = JSON.parse(response.body)['id'] container_id.presence || raise(Runner::Error::NotAvailable.new("DockerContainerPool didn't return a container id")) rescue Faraday::Error => e @@ -36,7 +37,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def destroy_at_management url = "#{self.class.config[:url]}/docker_container_pool/destroy_container/#{container.id}" Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Destroying runner at #{url}" } - Faraday.get(url) + Faraday.delete(url) rescue Faraday::Error => e raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") ensure diff --git a/spec/lib/runner/strategy/docker_container_pool_spec.rb b/spec/lib/runner/strategy/docker_container_pool_spec.rb index 01b418fd..2c4b4719 100644 --- a/spec/lib/runner/strategy/docker_container_pool_spec.rb +++ b/spec/lib/runner/strategy/docker_container_pool_spec.rb @@ -8,7 +8,7 @@ describe Runner::Strategy::DockerContainerPool do let(:execution_environment) { FactoryBot.create :ruby } let(:container_pool) { described_class.new(runner_id, execution_environment) } let(:docker_container_pool_url) { 'http://localhost:1234' } - let(:config) { {pool: {location: docker_container_pool_url}} } + let(:config) { {url: docker_container_pool_url, unused_runner_expiration_time: 180} } let(:container) { instance_double(Docker::Container) } before do @@ -17,9 +17,9 @@ describe Runner::Strategy::DockerContainerPool do end # All requests handle a Faraday error the same way. - shared_examples 'Faraday error handling' do + shared_examples 'Faraday error handling' do |http_verb| it 'raises a runner error' do - allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError) + allow(Faraday).to receive(http_verb).and_raise(Faraday::TimeoutError) expect { action.call }.to raise_error(Runner::Error::FaradayError) end end @@ -29,7 +29,7 @@ describe Runner::Strategy::DockerContainerPool do let(:response_body) { nil } let!(:request_runner_stub) do WebMock - .stub_request(:get, "#{docker_container_pool_url}/docker_container_pool/get_container/#{execution_environment.id}") + .stub_request(:post, "#{docker_container_pool_url}/docker_container_pool/get_container/#{execution_environment.id}") .to_return(body: response_body, status: 200) end @@ -63,14 +63,14 @@ describe Runner::Strategy::DockerContainerPool do end end - include_examples 'Faraday error handling' + include_examples 'Faraday error handling', :post end describe '#destroy_at_management' do let(:action) { -> { container_pool.destroy_at_management } } let!(:destroy_runner_stub) do WebMock - .stub_request(:get, "#{docker_container_pool_url}/docker_container_pool/destroy_container/#{runner_id}") + .stub_request(:delete, "#{docker_container_pool_url}/docker_container_pool/destroy_container/#{runner_id}") .to_return(body: nil, status: 200) end @@ -81,7 +81,7 @@ describe Runner::Strategy::DockerContainerPool do expect(destroy_runner_stub).to have_been_requested.once end - include_examples 'Faraday error handling' + include_examples 'Faraday error handling', :delete end describe '#copy_files' do From 537d8bfc95e7fd611d4016a164fa92fe598e869d Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 01:02:18 +0200 Subject: [PATCH 142/156] DCP: Add handling of inactivity timer --- lib/runner/strategy/docker_container_pool.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 761bdba8..aa42ecc9 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -21,6 +21,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def self.request_from_management(environment) url = "#{config[:url]}/docker_container_pool/get_container/#{environment.id}" + body = { + inactivity_timeout: config[:unused_runner_expiration_time].seconds, + } Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } response = Faraday.post url, body @@ -71,6 +74,8 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end def attach_to_execution(command, event_loop) + reset_inactivity_timer + @command = command query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{@container_id}/attach/ws?#{query_params}" @@ -177,6 +182,19 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy @local_workspace_path ||= Pathname.new(container.binds.first.split(':').first) end + def reset_inactivity_timer + url = "#{self.class.config[:url]}/docker_container_pool/reuse_container/#{container.id}" + body = { + inactivity_timeout: self.class.config[:unused_runner_expiration_time].seconds, + } + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Resetting inactivity timer at #{url}" } + Faraday.post url, body + rescue Faraday::Error => e + raise Runner::Error::FaradayError.new("Request to DockerContainerPool failed: #{e.inspect}") + ensure + Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Finished resetting inactivity timer" } + end + class Connection < Runner::Connection def initialize(*args) @stream = 'stdout' From 6a902c41db2eb16c63a11f742722fc75fc3e1f2a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 01:02:48 +0200 Subject: [PATCH 143/156] DCP: Refactor `container` method and usage --- lib/runner/strategy/docker_container_pool.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index aa42ecc9..2900a99f 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -78,7 +78,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy @command = command query_params = 'logs=0&stream=1&stderr=1&stdout=1&stdin=1' - websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{@container_id}/attach/ws?#{query_params}" + websocket_url = "#{self.class.config[:ws_host]}/v1.27/containers/#{container.id}/attach/ws?#{query_params}" socket = Connection.new(websocket_url, self, event_loop) begin @@ -150,12 +150,12 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy private def container - return @container if @container.present? + @container ||= begin + container = Docker::Container.get(@container_id) + raise Runner::Error::RunnerNotFound unless container.info['State']['Running'] - @container = Docker::Container.get(@container_id) - raise Runner::Error::RunnerNotFound unless @container.info['State']['Running'] - - @container + container + end rescue Docker::Error::NotFoundError raise Runner::Error::RunnerNotFound end From eaa06ee52804614d3c395753a4bbb7ab9794546f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 01:03:07 +0200 Subject: [PATCH 144/156] DCP: Prevent double deletion of runner --- lib/runner/connection.rb | 2 ++ lib/runner/strategy/docker_container_pool.rb | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/runner/connection.rb b/lib/runner/connection.rb index 87f59aac..6f6b12bc 100644 --- a/lib/runner/connection.rb +++ b/lib/runner/connection.rb @@ -135,6 +135,8 @@ class Runner::Connection case @status when :timeout + # The runner will destroyed. For the DockerContainerPool, this mechanism is necessary. + # However, it might not be required for Poseidon. @strategy.destroy_at_management @error = Runner::Error::ExecutionTimeout.new('Execution exceeded its time limit') when :terminated_by_codeocean, :terminated_by_management diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 2900a99f..d4e6e422 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -90,7 +90,6 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy end rescue Timeout::Error socket.close(:timeout) - destroy_at_management end socket end From de83843496acd8f1be3683df39d649affc5f8004 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 11:34:11 +0100 Subject: [PATCH 145/156] Combine no_output and exit_status messages --- app/controllers/submissions_controller.rb | 16 ++++++++++++---- config/locales/de.yml | 5 ++++- config/locales/en.yml | 5 ++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index e084b7d0..141ffc0b 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -147,10 +147,18 @@ class SubmissionsController < ApplicationController end runner_socket.on :exit do |exit_code| - if @output.empty? - client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.no_output', timestamp: l(Time.zone.now, format: :short))}\n"}) - end - client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{t('exercises.implement.exit', exit_code: exit_code)}\n"}) + exit_statement = + if @output.empty? && exit_code.zero? + t('exercises.implement.no_output_exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + elsif @output.empty? + t('exercises.implement.no_output_exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + elsif exit_code.zero? + t('exercises.implement.exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + else + t('exercises.implement.exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + end + client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{exit_statement}\n"}) + close_client_connection(client_socket) end end diff --git a/config/locales/de.yml b/config/locales/de.yml index 108da7e8..d137d478 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -406,7 +406,10 @@ de: hint: Hinweis no_files: Die Aufgabe umfasst noch keine sichtbaren Dateien. no_output: Die letzte Code-Ausführung terminierte am %{timestamp} ohne Ausgabe. - exit: Der Exit-Status war %{exit_code}. + no_output_exit_successful: Die letzte Code-Ausführung terminierte am %{timestamp} ohne Ausgabe and wurde erfolgreich beendet (Statuscode %{exit_code}). + no_output_exit_failure: Die letzte Code-Ausführung terminierte am %{timestamp} ohne Ausgabe und wurde mit einem Fehler beendet (Statuscode %{exit_code}). + exit_successful: Die letzte Code-Ausführung wurde erfolgreich beendet (Statuscode %{exit_code}). + exit_failure: Die letzte Code-Ausführung wurde mit einem Fehler beendet (Statuscode %{exit_code}). no_output_yet: Bisher existiert noch keine Ausgabe. output: Programm-Ausgabe passed_tests: Erfolgreiche Tests diff --git a/config/locales/en.yml b/config/locales/en.yml index 44d0e8df..05127ab7 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -406,7 +406,10 @@ en: hint: Hint no_files: The exercise does not comprise visible files yet. no_output: The last code run finished on %{timestamp} without any output. - exit: The exit status was %{exit_code}. + no_output_exit_successful: The last code run finished on %{timestamp} without any output and exited successfully (status code %{exit_code}). + no_output_exit_failure: The last code run finished on %{timestamp} without any output and exited with a failure (status code %{exit_code}). + exit_successful: The last code run exited successfully (status code %{exit_code}). + exit_failure: The last code run exited with a failure (status code %{exit_code}). no_output_yet: There is no output yet. output: Program Output passed_tests: Passed Tests From 56d219ad8e6583cc9fd9a909f3cb950b960c6155 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 11:50:07 +0100 Subject: [PATCH 146/156] [ci-skip] Improve comment for unused_runner_expiration_time --- config/code_ocean.yml.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/code_ocean.yml.example b/config/code_ocean.yml.example index 740b1598..df56be79 100644 --- a/config/code_ocean.yml.example +++ b/config/code_ocean.yml.example @@ -49,8 +49,9 @@ default: &default # The authorization token for connections to the runner management (Poseidon only) # If TLS support is not enabled, this token is transmitted in clear text! token: SECRET - # The maximum time in seconds a runner may idle at the runner management before - # it is removed. Each begin of an interaction with the runner resets this time + # The maximum time in seconds a runner may idle at the runner management before it is removed. + # Each begin of an interaction with the runner resets this time. Thus, this value should + # be truly greater than any permitted execution time of an execution environment. unused_runner_expiration_time: 180 From 447860892a7abef6c94ffd483f3111029d4846e2 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 12:39:06 +0100 Subject: [PATCH 147/156] Always remove `exposed_ports_list` if present --- app/controllers/execution_environments_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/execution_environments_controller.rb b/app/controllers/execution_environments_controller.rb index 9d714af6..7ae70014 100644 --- a/app/controllers/execution_environments_controller.rb +++ b/app/controllers/execution_environments_controller.rb @@ -108,7 +108,7 @@ class ExecutionEnvironmentsController < ApplicationController def execution_environment_params if params[:execution_environment].present? - exposed_ports = if params[:execution_environment][:exposed_ports_list].present? + exposed_ports = if params[:execution_environment][:exposed_ports_list] # Transform the `exposed_ports_list` to `exposed_ports` array params[:execution_environment].delete(:exposed_ports_list).scan(/\d+/) else From 475aa8c5125c958f2d6fba16bd177b2b95601d7d Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 12:39:56 +0100 Subject: [PATCH 148/156] DCP: Allow increasing the pool size when previously empty --- app/models/execution_environment.rb | 2 ++ lib/runner/strategy/docker_container_pool.rb | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/models/execution_environment.rb b/app/models/execution_environment.rb index 615f9ceb..247accc8 100644 --- a/app/models/execution_environment.rb +++ b/app/models/execution_environment.rb @@ -81,6 +81,8 @@ class ExecutionEnvironment < ApplicationRecord runner = Runner.for(author, self) output = runner.execute_command(VALIDATION_COMMAND) errors.add(:docker_image, "error: #{output[:stderr]}") if output[:stderr].present? + rescue Runner::Error::NotAvailable => e + Rails.logger.info("The Docker image could not be verified: #{e}") rescue Runner::Error => e errors.add(:docker_image, "error: #{e}") end diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index d4e6e422..5654b5c6 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -14,9 +14,17 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy FileUtils.mkdir_p(File.expand_path(config[:workspace_root])) end - def self.sync_environment(_environment) - # There is no dedicated sync mechanism yet - true + def self.sync_environment(environment) + # There is no dedicated sync mechanism yet. However, we need to emit a warning when the pool was previously + # empty for this execution environment. In this case the validation command probably was not executed. + return true unless environment.pool_size_previously_changed? + + case environment.pool_size_previously_was + when nil, 0 + false + else + true + end end def self.request_from_management(environment) From 1e7cf1c6228cfe541a1682a71e2c3aab16cf72d1 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 12:57:23 +0100 Subject: [PATCH 149/156] Prevent parallel execution of run and test during RfC creation * Otherwise, the output of both might be mixed and saved incorrectly for the RfC --- app/controllers/request_for_comments_controller.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/controllers/request_for_comments_controller.rb b/app/controllers/request_for_comments_controller.rb index 4592d57d..c673f842 100644 --- a/app/controllers/request_for_comments_controller.rb +++ b/app/controllers/request_for_comments_controller.rb @@ -117,12 +117,10 @@ class RequestForCommentsController < ApplicationController respond_to do |format| if @request_for_comment.save - # create thread here and execute tests. A run is triggered from the frontend and does not need to be handled here. - Thread.new do - switch_locale { @request_for_comment.submission.calculate_score } - ensure - ActiveRecord::Base.connection_pool.release_connection - end + # execute the tests here and wait until they finished. + # As the same runner is used for the score and test run, no parallelization is possible + # A run is triggered from the frontend and does not need to be handled here. + @request_for_comment.submission.calculate_score format.json { render :show, status: :created, location: @request_for_comment } else format.html { render :new } From 6ff14d6fc7dc4d2966ea5af4855db5127ba898d2 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 14:34:55 +0100 Subject: [PATCH 150/156] Connection Buffer: Replace \r in run and score output with \n --- app/assets/javascripts/editor/evaluation.js | 1 - lib/runner/connection/buffer.rb | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/assets/javascripts/editor/evaluation.js b/app/assets/javascripts/editor/evaluation.js index 8f79ccf6..d3f8ed14 100644 --- a/app/assets/javascripts/editor/evaluation.js +++ b/app/assets/javascripts/editor/evaluation.js @@ -175,7 +175,6 @@ CodeOceanEditorEvaluation = { if (!msg.data || msg.data === "\r") { return; } - msg.data = msg.data.replace(/(\r)/gm, "\n"); var stream = {}; stream[msg.stream] = msg.data; this.printOutput(stream, true, 0); diff --git a/lib/runner/connection/buffer.rb b/lib/runner/connection/buffer.rb index c7dcba87..cbc8b4c1 100644 --- a/lib/runner/connection/buffer.rb +++ b/lib/runner/connection/buffer.rb @@ -3,7 +3,8 @@ class Runner::Connection::Buffer # The WebSocket connection might group multiple lines. For further processing, we require all lines # to be processed separately. Therefore, we split the lines by each newline character not part of an enclosed - # substring either in single or double quotes (e.g., within a JSON) + # substring either in single or double quotes (e.g., within a JSON). Originally, each line break consists of `\r\n`. + # We keep the `\r` at the end of the line (keeping "empty" lines) and replace it after buffering. # Inspired by https://stackoverflow.com/questions/13040585/split-string-by-spaces-properly-accounting-for-quotes-and-backslashes-ruby SPLIT_INDIVIDUAL_LINES = Regexp.compile(/(?:"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|[^\n])+/) @@ -69,6 +70,9 @@ class Runner::Connection::Buffer def add_to_line_buffer(message) @buffering = false @global_buffer = +'' + # For our buffering, we identified line breaks with the `\n` and removed those temporarily. + # Thus, we now re-add the `\n` at the end of the string and remove the `\r` in the same time. + message = message.gsub(/\r$/, "\n") @line_buffer.push message end From bdfcb0da192bd1282e4e00157f47bb18a02dd568 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Sun, 31 Oct 2021 15:07:27 +0100 Subject: [PATCH 151/156] Reset previous exception if retrying command execution --- app/models/runner.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index c9772fc2..e0bebd5e 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -98,7 +98,12 @@ class Runner < ApplicationRecord try += 1 request_new_id save - retry if try == 1 + + if try == 1 + # Reset the variable. This is required to prevent raising an outdated exception after a successful second try + e = nil + retry + end Rails.logger.debug { "Running command `#{command}` failed for the second time: #{e.message}" } output.merge!(status: :failed, container_execution_time: e.execution_duration) From d16917261bb82ee84e97316091c15aa6c2ef6a84 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 1 Nov 2021 09:48:16 +0100 Subject: [PATCH 152/156] Prevent inactivityTimeout from being smaller than permitted_execution_time --- lib/runner/strategy/docker_container_pool.rb | 6 ++++-- lib/runner/strategy/poseidon.rb | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 5654b5c6..eea7c9cc 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -29,8 +29,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def self.request_from_management(environment) url = "#{config[:url]}/docker_container_pool/get_container/#{environment.id}" + inactivity_timeout = [config[:unused_runner_expiration_time], environment.permitted_execution_time].max body = { - inactivity_timeout: config[:unused_runner_expiration_time].seconds, + inactivity_timeout: inactivity_timeout.to_i.seconds, } Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } response = Faraday.post url, body @@ -191,8 +192,9 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def reset_inactivity_timer url = "#{self.class.config[:url]}/docker_container_pool/reuse_container/#{container.id}" + inactivity_timeout = [self.class.config[:unused_runner_expiration_time], @execution_environment.permitted_execution_time].max body = { - inactivity_timeout: self.class.config[:unused_runner_expiration_time].seconds, + inactivity_timeout: inactivity_timeout.to_i.seconds, } Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Resetting inactivity timer at #{url}" } Faraday.post url, body diff --git a/lib/runner/strategy/poseidon.rb b/lib/runner/strategy/poseidon.rb index cabda9f5..c84365bf 100644 --- a/lib/runner/strategy/poseidon.rb +++ b/lib/runner/strategy/poseidon.rb @@ -33,9 +33,10 @@ class Runner::Strategy::Poseidon < Runner::Strategy def self.request_from_management(environment) url = "#{config[:url]}/runners" + inactivity_timeout = [config[:unused_runner_expiration_time], environment.permitted_execution_time].max body = { executionEnvironmentId: environment.id, - inactivityTimeout: config[:unused_runner_expiration_time].seconds, + inactivityTimeout: inactivity_timeout.to_i.seconds, } Rails.logger.debug { "#{Time.zone.now.getutc.inspect}: Requesting new runner at #{url}" } response = http_connection.post url, body.to_json From 65fe1d902db4dbbab83c9ca3c3dd35d74f27a714 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 1 Nov 2021 12:44:13 +0100 Subject: [PATCH 153/156] DCP: Match java@hostname output --- lib/runner/strategy/docker_container_pool.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index eea7c9cc..491fa701 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -216,7 +216,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def decode(event_data) case event_data - when /(?.*)((root|python|user)@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/m + when /(?.*)((root|python|java|user)@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/m # The RegEx above is used to determine unwanted output which also indicates a program termination. # If the RegEx matches, at least two capture groups will be created. # The first (called `previous_data`) contains any data before the match (including multiple lines) From 2c10b48b7060a3742f2a4726c735b0cbc015a2f1 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 1 Nov 2021 13:31:11 +0100 Subject: [PATCH 154/156] Execute Command: Guard requesting new runner * If any exception is thrown, these will be caught now and handled appropriately --- app/models/runner.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/runner.rb b/app/models/runner.rb index e0bebd5e..d92b66bc 100644 --- a/app/models/runner.rb +++ b/app/models/runner.rb @@ -77,6 +77,11 @@ class Runner < ApplicationRecord stderr = +'' try = 0 begin + if try.nonzero? + request_new_id + save + end + exit_code = 1 # default to error execution_time = attach_to_execution(command) do |socket| socket.on :stderr do |data| @@ -96,8 +101,6 @@ class Runner < ApplicationRecord rescue Runner::Error::RunnerNotFound => e Rails.logger.debug { "Running command `#{command}` failed for the first time: #{e.message}" } try += 1 - request_new_id - save if try == 1 # Reset the variable. This is required to prevent raising an outdated exception after a successful second try From 328055e6e85fafd5622682d8dae27cb752471a80 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 1 Nov 2021 14:03:12 +0100 Subject: [PATCH 155/156] DCP: previous_data match should be non-greedy --- lib/runner/strategy/docker_container_pool.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runner/strategy/docker_container_pool.rb b/lib/runner/strategy/docker_container_pool.rb index 491fa701..42c66f66 100644 --- a/lib/runner/strategy/docker_container_pool.rb +++ b/lib/runner/strategy/docker_container_pool.rb @@ -216,7 +216,7 @@ class Runner::Strategy::DockerContainerPool < Runner::Strategy def decode(event_data) case event_data - when /(?.*)((root|python|java|user)@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/m + when /(?.*?)((root|python|java|user)@#{@strategy.container_id[0..11]}|#exit|{"cmd": "exit"})/m # The RegEx above is used to determine unwanted output which also indicates a program termination. # If the RegEx matches, at least two capture groups will be created. # The first (called `previous_data`) contains any data before the match (including multiple lines) From c3642b5d0c516e415c17346e7bc977424fa5591f Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 1 Nov 2021 17:52:44 +0100 Subject: [PATCH 156/156] Add an empty line before printing exit message * The empty line is only added if the output is not empty --- app/controllers/submissions_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 141ffc0b..063cffe5 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -153,9 +153,9 @@ class SubmissionsController < ApplicationController elsif @output.empty? t('exercises.implement.no_output_exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) elsif exit_code.zero? - t('exercises.implement.exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + "\n#{t('exercises.implement.exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)}" else - t('exercises.implement.exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code) + "\n#{t('exercises.implement.exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)}" end client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{exit_statement}\n"})