htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1242 from openHPI/refactor_testrun_table

Browse Source 
Refactor testrun table
This commit is contained in:
Sebastian Serth
2022-05-04 00:25:08 +02:00
committed by GitHub
parent 2ce9c4258a 4ef7da839d
commit e4384d7ddd
19 changed files with 507 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/javascripts/editor/editor.js.erb
View File

@@ -703,6 +703,8 @@ var CodeOceanEditor = {
this.showTimeoutMessage();
} else if (output.status === 'container_depleted') {
this.showContainerDepletedMessage();
} else if (output.status === 'out_of_memory') {
this.showOutOfMemoryMessage();
} else if (output.stderr) {
$.flash.danger({
icon: ['fa', 'fa-bug'],

2
app/assets/javascripts/editor/execution.js
View File

@@ -46,8 +46,6 @@ CodeOceanEditorWebsocket = {
this.websocket.on('turtlebatch', this.handleTurtlebatchCommand.bind(this));
this.websocket.on('render', this.renderWebsocketOutput.bind(this));
this.websocket.on('exit', this.handleExitCommand.bind(this));
this.websocket.on('timeout', this.showTimeoutMessage.bind(this));
this.websocket.on('out_of_memory', this.showOutOfMemoryMessage.bind(this));
this.websocket.on('status', this.showStatus.bind(this));
this.websocket.on('hint', this.showHint.bind(this));
},

2
app/controllers/execution_environments_controller.rb
View File

@@ -30,7 +30,7 @@ class ExecutionEnvironmentsController < ApplicationController
def execute_command
runner = Runner.for(current_user, @execution_environment)
output = runner.execute_command(params[:command], raise_exception: false)
render json: output
render json: output.except(:messages)
end
def working_time_query

139
app/controllers/submissions_controller.rb
View File

@@ -9,6 +9,7 @@ class SubmissionsController < ApplicationController
before_action :require_user!
before_action :set_submission, only: %i[download download_file render_file run score show statistics test]
before_action :set_testrun, only: %i[run score test]
before_action :set_files, only: %i[download show]
before_action :set_files_and_specific_file, only: %i[download_file render_file run test]
before_action :set_mime_type, only: %i[download_file render_file]
@@ -92,6 +93,7 @@ class SubmissionsController < ApplicationController
client_socket.onclose do |_event|
runner_socket&.close(:terminated_by_client)
# We do not update the @testrun[:status] by design, it would be missleading
end
client_socket.onmessage do |raw_event|
@@ -100,9 +102,17 @@ class SubmissionsController < ApplicationController
# Otherwise, we expect to receive a JSON: Parsing.
event = JSON.parse(raw_event).deep_symbolize_keys
event[:cmd] = event[:cmd].to_sym
event[:stream] = event[:stream].to_sym if event.key? :stream
case event[:cmd].to_sym
# We could store the received event. However, it is also echoed by the container
# and correctly identified as the original input. Therefore, we don't store
# it here to prevent duplicated events.
# @testrun[:messages].push(event)
case event[:cmd]
when :client_kill
@testrun[:status] = :terminated_by_client
close_client_connection(client_socket)
Rails.logger.debug('Client exited container.')
when :result, :canvasevent, :exception
@@ -128,62 +138,61 @@ class SubmissionsController < ApplicationController
end
end
@output = +''
durations = @submission.run(@file) do |socket|
@testrun[:output] = +''
durations = @submission.run(@file) do |socket, starting_time|
runner_socket = socket
@testrun[:starting_time] = starting_time
client_socket.send_data JSON.dump({cmd: :status, status: :container_running})
runner_socket.on :stdout do |data|
json_data = prepare data, :stdout
@output << json_data[0, max_output_buffer_size - @output.size]
client_socket.send_data(json_data)
message = retrieve_message_from_output data, :stdout
@testrun[:output] << message[:data][0, max_output_buffer_size - @testrun[:output].size] if message[:data]
send_and_store client_socket, message
end
runner_socket.on :stderr do |data|
json_data = prepare data, :stderr
@output << json_data[0, max_output_buffer_size - @output.size]
client_socket.send_data(json_data)
message = retrieve_message_from_output data, :stderr
@testrun[:output] << message[:data][0, max_output_buffer_size - @testrun[:output].size] if message[:data]
send_and_store client_socket, message
end
runner_socket.on :exit do |exit_code|
@exit_code = exit_code
@testrun[:exit_code] = exit_code
exit_statement =
if @output.empty? && exit_code.zero?
@status = :ok
if @testrun[:output].empty? && exit_code.zero?
@testrun[:status] = :ok
t('exercises.implement.no_output_exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)
elsif @output.empty?
@status = :failed
elsif @testrun[:output].empty?
@testrun[:status] = :failed
t('exercises.implement.no_output_exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)
elsif exit_code.zero?
@status = :ok
@testrun[:status] = :ok
"\n#{t('exercises.implement.exit_successful', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)}"
else
@status = :failed
@testrun[:status] = :failed
"\n#{t('exercises.implement.exit_failure', timestamp: l(Time.zone.now, format: :short), exit_code: exit_code)}"
end
client_socket.send_data JSON.dump({cmd: :write, stream: :stdout, data: "#{exit_statement}\n"})
send_and_store client_socket, {cmd: :write, stream: :stdout, data: "#{exit_statement}\n"}
if exit_code == 137
client_socket.send_data JSON.dump({cmd: :out_of_memory})
@status = :out_of_memory
send_and_store client_socket, {cmd: :status, status: :out_of_memory}
@testrun[:status] = :out_of_memory
end
close_client_connection(client_socket)
end
end
@container_execution_time = durations[:execution_duration]
@waiting_for_container_time = durations[:waiting_duration]
@testrun[:container_execution_time] = durations[:execution_duration]
@testrun[:waiting_for_container_time] = durations[:waiting_duration]
rescue Runner::Error::ExecutionTimeout => e
client_socket.send_data JSON.dump({cmd: :status, status: :timeout})
send_and_store client_socket, {cmd: :status, status: :timeout}
close_client_connection(client_socket)
Rails.logger.debug { "Running a submission timed out: #{e.message}" }
@output = "timeout: #{@output}"
@status = :timeout
@testrun[:output] = "timeout: #{@testrun[:output]}"
extract_durations(e)
rescue Runner::Error => e
client_socket.send_data JSON.dump({cmd: :status, status: :container_depleted})
send_and_store client_socket, {cmd: :status, status: :container_depleted}
close_client_connection(client_socket)
Rails.logger.debug { "Runner error while running a submission: #{e.message}" }
@status = :container_depleted
extract_durations(e)
ensure
save_testrun_output 'run'
@@ -195,17 +204,17 @@ class SubmissionsController < ApplicationController
switch_locale do
kill_client_socket(tubesock) if @embed_options[:disable_score]
# The score is stored separately, we can forward it to the client immediately
tubesock.send_data(JSON.dump(@submission.calculate_score))
# To enable hints when scoring a submission, uncomment the next line:
# send_hints(tubesock, StructuredError.where(submission: @submission))
kill_client_socket(tubesock)
rescue Runner::Error => e
tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted})
extract_durations(e)
send_and_store tubesock, {cmd: :status, status: :container_depleted}
kill_client_socket(tubesock)
Rails.logger.debug { "Runner error while scoring submission #{@submission.id}: #{e.message}" }
@passed = false
@status = :container_depleted
extract_durations(e)
@testrun[:passed] = false
save_testrun_output 'assess'
end
end
@@ -222,15 +231,15 @@ class SubmissionsController < ApplicationController
switch_locale do
kill_client_socket(tubesock) if @embed_options[:disable_run]
# The score is stored separately, we can forward it to the client immediately
tubesock.send_data(JSON.dump(@submission.test(@file)))
kill_client_socket(tubesock)
rescue Runner::Error => e
tubesock.send_data JSON.dump({cmd: :status, status: :container_depleted})
extract_durations(e)
send_and_store tubesock, {cmd: :status, status: :container_depleted}
kill_client_socket(tubesock)
Rails.logger.debug { "Runner error while testing submission #{@submission.id}: #{e.message}" }
@passed = false
@status = :container_depleted
extract_durations(e)
@testrun[:passed] = false
save_testrun_output 'assess'
end
end
@@ -251,6 +260,7 @@ class SubmissionsController < ApplicationController
end
def kill_client_socket(client_socket)
# We don't want to store this (arbitrary) exit command and redirect it ourselves
client_socket.send_data JSON.dump({cmd: :exit})
client_socket.close
end
@@ -279,21 +289,29 @@ class SubmissionsController < ApplicationController
end
def extract_durations(error)
@container_execution_time = error.execution_duration
@waiting_for_container_time = error.waiting_duration
@testrun[:starting_time] = error.starting_time
@testrun[:container_execution_time] = error.execution_duration
@testrun[:waiting_for_container_time] = error.waiting_duration
end
def extract_errors
results = []
if @output.present?
if @testrun[:output].present?
@submission.exercise.execution_environment.error_templates.each do |template|
pattern = Regexp.new(template.signature).freeze
results << StructuredError.create_from_template(template, @output, @submission) if pattern.match(@output)
results << StructuredError.create_from_template(template, @testrun[:output], @submission) if pattern.match(@testrun[:output])
end
end
results
end
def send_and_store(client_socket, message)
message[:timestamp] = ActiveSupport::Duration.build(Time.zone.now - @testrun[:starting_time])
@testrun[:messages].push message
@testrun[:status] = message[:status] if message[:status]
client_socket.send_data JSON.dump(message)
end
def max_output_buffer_size
if @submission.cause == 'requestComments'
5000
@@ -302,14 +320,6 @@ class SubmissionsController < ApplicationController
end
end
def prepare(data, stream)
if valid_command? data
data
else
JSON.dump({cmd: :write, stream: stream, data: data})
end
end
def sanitize_filename
params[:filename].gsub(/\.json$/, '')
end
@@ -318,15 +328,16 @@ class SubmissionsController < ApplicationController
def save_testrun_output(cause)
testrun = Testrun.create!(
file: @file,
passed: @passed,
passed: @testrun[:passed],
cause: cause,
submission: @submission,
exit_code: @exit_code, # might be nil, e.g., when the run did not finish
status: @status,
output: @output.presence, # TODO: Remove duplicated saving of the output after creating TestrunMessages
container_execution_time: @container_execution_time,
waiting_for_container_time: @waiting_for_container_time
exit_code: @testrun[:exit_code], # might be nil, e.g., when the run did not finish
status: @testrun[:status],
output: @testrun[:output].presence, # TODO: Remove duplicated saving of the output after creating TestrunMessages
container_execution_time: @testrun[:container_execution_time],
waiting_for_container_time: @testrun[:waiting_for_container_time]
)
TestrunMessage.create_for(testrun, @testrun[:messages])
TestrunExecutionEnvironment.create(testrun: testrun, execution_environment: @submission.used_execution_environment)
end
@@ -335,7 +346,7 @@ class SubmissionsController < ApplicationController
errors = errors.to_a.uniq(&:hint)
errors.each do |error|
tubesock.send_data JSON.dump({cmd: 'hint', hint: error.hint, description: error.error_template.description})
send_and_store tubesock, {cmd: :hint, hint: error.hint, description: error.error_template.description}
end
end
@@ -361,10 +372,26 @@ class SubmissionsController < ApplicationController
authorize!
end
def valid_command?(data)
def set_testrun
@testrun = {
messages: [],
exit_code: nil,
status: nil,
}
end
def retrieve_message_from_output(data, stream)
parsed = JSON.parse(data)
parsed.instance_of?(Hash) && parsed.key?('cmd')
if parsed.instance_of?(Hash) && parsed.key?('cmd')
parsed.symbolize_keys!
# Symbolize two values if present
parsed[:cmd] = parsed[:cmd].to_sym
parsed[:stream] = parsed[:stream].to_sym if parsed.key? :stream
parsed
else
{cmd: :write, stream: stream, data: data}
end
rescue JSON::ParserError
false
{cmd: :write, stream: stream, data: data}
end
end

2
app/errors/runner/error.rb
View File

@@ -2,7 +2,7 @@
class Runner
class Error < ApplicationError
attr_accessor :waiting_duration, :execution_duration
attr_accessor :waiting_duration, :execution_duration, :starting_time
class BadRequest < Error; end

3
app/models/application_record.rb
View File

@@ -8,7 +8,8 @@ class ApplicationRecord < ActiveRecord::Base
def strip_strings
# trim whitespace from beginning and end of string attributes
# except for the `content` of CodeOcean::Files
attribute_names.without('content').each do |name|
# and except the `log` of TestrunMessages or the `output` of Testruns
attribute_names.without('content', 'log', 'output').each do |name|
if send(name.to_sym).respond_to?(:strip)
send("#{name}=".to_sym, send(name).strip)
end

28
app/models/runner.rb
View File

@@ -62,10 +62,11 @@ class Runner < ApplicationRecord
# initializing its Runner::Connection with the given event loop. The Runner::Connection class ensures that
# this event loop is stopped after the socket was closed.
event_loop = Runner::EventLoop.new
socket = @strategy.attach_to_execution(command, event_loop, &block)
socket = @strategy.attach_to_execution(command, event_loop, starting_time, &block)
event_loop.wait
raise socket.error if socket.error.present?
rescue Runner::Error => e
e.starting_time = starting_time
e.execution_duration = Time.zone.now - starting_time
raise
end
@@ -74,30 +75,34 @@ class Runner < ApplicationRecord
end
def execute_command(command, raise_exception: true)
output = {}
stdout = +''
stderr = +''
output = {
stdout: +'',
stderr: +'',
messages: [],
exit_code: 1, # default to error
}
try = 0
exit_code = 1 # default to error
begin
if try.nonzero?
request_new_id
save
end
execution_time = attach_to_execution(command) do |socket|
execution_time = attach_to_execution(command) do |socket, starting_time|
socket.on :stderr do |data|
stderr << data
output[:stderr] << data
output[:messages].push({cmd: :write, stream: :stderr, log: data, timestamp: Time.zone.now - starting_time})
end
socket.on :stdout do |data|
stdout << data
output[:stdout] << data
output[:messages].push({cmd: :write, stream: :stdout, log: data, timestamp: Time.zone.now - starting_time})
end
socket.on :exit do |received_exit_code|
exit_code = received_exit_code
output[:exit_code] = received_exit_code
end
end
output.merge!(container_execution_time: execution_time, status: exit_code.zero? ? :ok : :failed)
output.merge!(container_execution_time: execution_time, status: output[:exit_code].zero? ? :ok : :failed)
rescue Runner::Error::ExecutionTimeout => e
Rails.logger.debug { "Running command `#{command}` timed out: #{e.message}" }
output.merge!(status: :timeout, container_execution_time: e.execution_duration)
@@ -122,8 +127,7 @@ class Runner < ApplicationRecord
raise e if raise_exception && defined?(e) && e.present?
# If the process was killed with SIGKILL, it is most likely that the OOM killer was triggered.
output[:status] = :out_of_memory if exit_code == 137
output.merge!(stdout: stdout, stderr: stderr, exit_code: exit_code)
output[:status] = :out_of_memory if output[:exit_code] == 137
end
end

2
app/models/submission.rb
View File

@@ -257,6 +257,7 @@ class Submission < ApplicationRecord
container_execution_time: output[:container_execution_time],
waiting_for_container_time: output[:waiting_for_container_time]
)
TestrunMessage.create_for(testrun, output[:messages])
TestrunExecutionEnvironment.create(testrun: testrun, execution_environment: @used_execution_environment)
filename = file.filepath
@@ -271,6 +272,7 @@ class Submission < ApplicationRecord
output.merge!(assessment)
output.merge!(filename: filename, message: feedback_message(file, output), weight: file.weight)
output.except!(:messages)
end
def feedback_message(file, output)

5
app/models/testrun.rb
View File

@@ -12,8 +12,13 @@ class Testrun < ApplicationRecord
container_depleted: 2,
timeout: 3,
out_of_memory: 4,
terminated_by_client: 5,
}, _default: :ok, _prefix: true
validates :exit_code, numericality: {only_integer: true, min: 0, max: 255}, allow_nil: true
validates :status, presence: true
def log
testrun_messages.output.select(:log).map(&:log).join.presence
end
end

70
app/models/testrun_message.rb
View File

@@ -11,13 +11,14 @@ class TestrunMessage < ApplicationRecord
turtlebatch: 4,
render: 5,
exit: 6,
timeout: 7,
out_of_memory: 8,
status: 9,
hint: 10,
client_kill: 11,
exception: 12,
result: 13,
status: 7,
hint: 8,
client_kill: 9,
exception: 10,
result: 11,
canvasevent: 12,
timeout: 13, # TODO: Shouldn't be in the data, this is a status and can be removed after the migration finished
out_of_memory: 14, # TODO: Shouldn't be in the data, this is a status and can be removed after the migration finished
}, _default: :write, _prefix: true
enum stream: {
@@ -28,9 +29,62 @@ class TestrunMessage < ApplicationRecord
validates :cmd, presence: true
validates :timestamp, presence: true
validates :stream, length: {minimum: 0, allow_nil: false}, if: -> { cmd_write? }
validates :log, length: {minimum: 0, allow_nil: false}, if: -> { cmd_write? }
validate :either_data_or_log
default_scope { order(timestamp: :asc) }
scope :output, -> { where(cmd: 1, stream: %i[stdout stderr]) }
def self.create_for(testrun, messages)
# We don't want to store anything if the testrun passed
return if testrun.passed?
messages.map! do |message|
# We create a new hash and move all known keys
result = {}
result[:testrun] = testrun
result[:log] = (message.delete(:log) || message.delete(:data)) if message[:cmd] == :write || message.key?(:log)
result[:timestamp] = message.delete :timestamp
result[:stream] = message.delete :stream if message.key?(:stream)
result[:cmd] = message.delete :cmd
# The remaining keys will be stored in the `data` column
result[:data] = message.presence if message.present?
result
end
# Before storing all messages, we truncate some to save storage
filtered_messages = filter_messages_by_size testrun, messages
# An array with hashes is passed, all are stored
TestrunMessage.create!(filtered_messages)
end
def self.filter_messages_by_size(testrun, messages)
limits = if testrun.submission.cause == 'requestComments'
{data: {limit: 25, size: 0}, log: {limit: 5000, size: 0}}
else
{data: {limit: 10, size: 0}, log: {limit: 500, size: 0}}
end
filtered_messages = messages.map do |message|
if message.key?(:log) && limits[:log][:size] < limits[:log][:limit]
message[:log] = message[:log][0, limits[:log][:limit] - limits[:log][:size]]
limits[:log][:size] += message[:log].size
elsif message[:data] && limits[:data][:size] < limits[:data][:limit]
limits[:data][:size] += 1
elsif !message.key?(:log) && limits[:data][:size] < limits[:data][:limit]
# Accept short TestrunMessages (e.g. just transporting a status information)
# without increasing the `limits[:data][:limit]` before the limit is reached
else
# Clear all remaining messages
message = nil
end
message
end
filtered_messages.select(&:present?)
end
def either_data_or_log
if [data, log].count(&:present?) > 1
errors.add(log, "can't be present if data is also present")

4
app/views/exercises/external_users/statistics.html.slim
View File

@@ -65,9 +65,9 @@ h1
td.align-middle
-this.testruns.includes(:file).order("files.name").each do |run|
- if run.passed
.unit-test-result.positive-result title=[run.file&.filepath, run.output].join("\n").strip
.unit-test-result.positive-result title=[run.file&.filepath, run.log].join.strip
- else
.unit-test-result.unknown-result title=[run.file&.filepath, run.output].join("\n").strip
.unit-test-result.unknown-result title=[run.file&.filepath, run.log].join.strip
td = @working_times_until[index] if index > 0 if policy(@exercise).detailed_statistics?
- elsif this.is_a? UserExerciseIntervention
td = this.created_at.strftime("%F %T")

17
app/views/request_for_comments/show.html.slim
View File

@@ -38,20 +38,7 @@
.collapsed.testrun-output.text
span.fa.fa-chevron-down.collapse-button
- output_runs.each do |testrun|
- output = testrun.try(:output)
- if output
- Sentry.set_extras(output: output)
- begin
- Timeout::timeout(2) do
// (?:\\"|.) is required to correctly identify " within the output.
// The outer (?: |\d+?) is used to correctly identify integers within the JSON
- messages = output.scan(/{(?:(?:"(?:\\"|.)+?":(?:"(?:\\"|.)*?"|-?\d+?|\[.*?\]|null))+?,?)+}/)
- messages.map! {|el| JSON.parse(el)}
- messages.keep_if {|message| message['cmd'] == 'write'}
- messages.map! {|message| message['data']}
- output = messages.join ''
- rescue Timeout::Error
pre= output or t('request_for_comments.no_output')
pre= testrun.log or t('request_for_comments.no_output')
- assess_runs = testruns.select {|run| run.cause == 'assess' }
- unless @current_user.admin?
@@ -64,7 +51,7 @@
div class=("result #{testrun.passed ? 'passed' : 'failed'}")
.collapsed.testrun-output.text
span.fa.fa-chevron-down.collapse-button
pre= testrun.output or t('request_for_comments.no_output')
pre= testrun.log or t('request_for_comments.no_output')
- if @current_user.admin? && user.is_a?(ExternalUser)
= render('admin_menu')