Automatically submit LTI grade on each score run
With this commit, we refactor the overall score handling of CodeOcean. Previously, "Score" and "Submit" were two distinct actions, requiring users to confirm the LTI transmission of their score (after assessing their submission). This yielded many questions and was unnecessary, since LTI parameters are no longer expiring after each use. Therefore, we can now transmit the current grade on each score run with the very same LTI parameters. As a consequence, the LTI consumer gets a more detailed history of the scores, enabling further analytical insights. For users, the previous "Submit" button got replaced with a notification that is shown as soon as the full score got reached. Then, learners can decide to "finalize" their work on the given exercise, which will initiate a redirect to a follow-up action (as defined in the RedirectBehavior). This RedirectBehavior has also been unified and simplified for better readability. As part of this refactoring, we rephrased the notifications and UX workflow of a) the LTI transmission, b) the finalization of an exercise (measured by reaching the full score) and c) the deadline handling (on time, within grace period, too late). Those information are now separately shown, potentially resulting in multiple notifications. As a side effect, they are much better maintainable, and the LTI transmission is more decoupled from this notification handling.
This commit is contained in:
kiragrammel
committed by
Sebastian Serth
Sebastian Serth
parent
1e06ab3fa9
commit
175c8933f3
@ -123,47 +123,68 @@ module Lti
|
||||
raise Error.new("Score #{submission.normalized_score} must be between 0 and #{MAXIMUM_SCORE}!")
|
||||
end
|
||||
|
||||
submission.users.map {|user| send_score_for submission, user }
|
||||
# Prepare score to be sent
|
||||
score = submission.normalized_score
|
||||
deadline = :none
|
||||
if submission.before_deadline?
|
||||
# Keep the full score
|
||||
deadline = :before_deadline
|
||||
elsif submission.within_grace_period?
|
||||
# Reduce score by 20%
|
||||
score *= 0.8
|
||||
deadline = :within_grace_period
|
||||
elsif submission.after_late_deadline?
|
||||
# Reduce score by 100%
|
||||
score *= 0.0
|
||||
deadline = :after_late_deadline
|
||||
end
|
||||
|
||||
# Actually send the score for all users
|
||||
detailed_results = submission.users.map {|user| send_score_for submission, user, score }
|
||||
|
||||
# Prepare return value
|
||||
erroneous_results = detailed_results.filter {|result| result[:status] == 'error' }
|
||||
unsupported_results = detailed_results.filter {|result| result[:status] == 'unsupported' }
|
||||
statistics = {
|
||||
all: detailed_results,
|
||||
success: detailed_results - erroneous_results - unsupported_results,
|
||||
error: erroneous_results,
|
||||
unsupported: unsupported_results,
|
||||
}
|
||||
|
||||
{
|
||||
users: statistics.transform_values {|value| value.pluck(:user) },
|
||||
score: {original: submission.normalized_score, sent: score},
|
||||
deadline:,
|
||||
detailed_results:,
|
||||
}
|
||||
end
|
||||
|
||||
private :send_scores
|
||||
|
||||
def send_score_for(submission, user)
|
||||
if user.external_user? && user.consumer
|
||||
lti_parameter = user.lti_parameters.find_by(exercise: submission.exercise, study_group: submission.study_group)
|
||||
provider = build_tool_provider(consumer: user.consumer, parameters: lti_parameter&.lti_parameters)
|
||||
end
|
||||
def send_score_for(submission, user, score)
|
||||
return {status: 'error', user:} unless user.external_user? && user.consumer
|
||||
|
||||
if provider.nil?
|
||||
{status: 'error', user: user.displayname}
|
||||
elsif provider.outcome_service?
|
||||
Sentry.set_extras({
|
||||
provider: provider.inspect,
|
||||
score: submission.normalized_score,
|
||||
lti_parameter: lti_parameter.inspect,
|
||||
session: session.to_hash,
|
||||
exercise_id: submission.exercise_id,
|
||||
})
|
||||
normalized_lti_score = submission.normalized_score
|
||||
if submission.before_deadline?
|
||||
# Keep the full score
|
||||
elsif submission.within_grace_period?
|
||||
# Reduce score by 20%
|
||||
normalized_lti_score *= 0.8
|
||||
elsif submission.after_late_deadline?
|
||||
# Reduce score by 100%
|
||||
normalized_lti_score *= 0.0
|
||||
end
|
||||
lti_parameter = user.lti_parameters.find_by(exercise: submission.exercise, study_group: submission.study_group)
|
||||
provider = build_tool_provider(consumer: user.consumer, parameters: lti_parameter&.lti_parameters)
|
||||
return {status: 'error', user:} if provider.nil?
|
||||
return {status: 'unsupported', user:} unless provider.outcome_service?
|
||||
|
||||
begin
|
||||
response = provider.post_replace_result!(normalized_lti_score)
|
||||
{code: response.response_code, message: response.post_response.body, status: response.code_major, score_sent: normalized_lti_score, user: user.displayname}
|
||||
rescue IMS::LTI::XMLParseError, Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET, SocketError, EOFError
|
||||
# A parsing error might happen if the LTI provider is down and doesn't return a valid XML response
|
||||
{status: 'error', user: user.displayname}
|
||||
end
|
||||
else
|
||||
{status: 'unsupported', user: user.displayname}
|
||||
Sentry.set_extras({
|
||||
provider: provider.inspect,
|
||||
normalized_score: submission.normalized_score,
|
||||
score:,
|
||||
lti_parameter: lti_parameter.inspect,
|
||||
session: defined?(session) ? session.to_hash : nil,
|
||||
exercise_id: submission.exercise_id,
|
||||
})
|
||||
|
||||
begin
|
||||
response = provider.post_replace_result!(score)
|
||||
{code: response.response_code, message: response.post_response.body, status: response.code_major, user:}
|
||||
rescue IMS::LTI::XMLParseError, Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET, SocketError, EOFError
|
||||
# A parsing error might happen if the LTI provider is down and doesn't return a valid XML response
|
||||
{status: 'error', user:}
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -6,60 +6,19 @@ module RedirectBehavior
|
||||
def redirect_after_submit
|
||||
Rails.logger.debug { "Redirecting user with score:s #{@submission.normalized_score}" }
|
||||
|
||||
if @submission.normalized_score.to_d == BigDecimal('1.0')
|
||||
if redirect_to_community_solution?
|
||||
redirect_to_community_solution
|
||||
return
|
||||
end
|
||||
# Redirect to the corresponding community solution if enabled and the user is eligible.
|
||||
return redirect_to_community_solution if redirect_to_community_solution?
|
||||
|
||||
# if user is external and has an own rfc, redirect to it and message him to clean up and accept the answer. (we need to check that the user is external,
|
||||
# otherwise an internal user could be shown a false rfc here, since current_user.id is polymorphic, but only makes sense for external users when used with rfcs.)
|
||||
# redirect 10 percent pseudorandomly to the feedback page
|
||||
if current_user.respond_to? :external_id
|
||||
if @submission.redirect_to_feedback? && !@embed_options[:disable_redirect_to_feedback]
|
||||
redirect_to_user_feedback
|
||||
return
|
||||
end
|
||||
# Redirect 10 percent pseudo-randomly to the feedback page.
|
||||
return redirect_to_user_feedback if !@embed_options[:disable_redirect_to_feedback] && @submission.redirect_to_feedback?
|
||||
|
||||
rfc = @submission.own_unsolved_rfc(current_user)
|
||||
if rfc
|
||||
# set a message that informs the user that his own RFC should be closed.
|
||||
flash[:notice] = I18n.t('exercises.submit.full_score_redirect_to_own_rfc')
|
||||
flash.keep(:notice)
|
||||
# If the user has an own rfc, redirect to it and message them to resolve and reflect on it.
|
||||
return redirect_to_unsolved_rfc(own: true) if redirect_to_own_unsolved_rfc?
|
||||
|
||||
respond_to do |format|
|
||||
format.html { redirect_to(rfc) }
|
||||
format.json { render(json: {redirect: url_for(rfc)}) }
|
||||
end
|
||||
return
|
||||
end
|
||||
# Otherwise, redirect to an unsolved rfc and ask for assistance.
|
||||
return redirect_to_unsolved_rfc if redirect_to_unsolved_rfc?
|
||||
|
||||
# else: show open rfc for same exercise if available
|
||||
rfc = @submission.unsolved_rfc(current_user)
|
||||
unless rfc.nil? || @embed_options[:disable_redirect_to_rfcs] || @embed_options[:disable_rfc]
|
||||
# set a message that informs the user that his score was perfect and help in RFC is greatly appreciated.
|
||||
flash[:notice] = I18n.t('exercises.submit.full_score_redirect_to_rfc')
|
||||
flash.keep(:notice)
|
||||
|
||||
# increase counter 'times_featured' in rfc
|
||||
rfc.increment(:times_featured)
|
||||
|
||||
respond_to do |format|
|
||||
format.html { redirect_to(rfc) }
|
||||
format.json { render(json: {redirect: url_for(rfc)}) }
|
||||
end
|
||||
return
|
||||
end
|
||||
end
|
||||
else
|
||||
# redirect to feedback page if score is less than 100 percent
|
||||
if @exercise.needs_more_feedback?(@submission) && !@embed_options[:disable_redirect_to_feedback]
|
||||
redirect_to_user_feedback
|
||||
else
|
||||
redirect_to_lti_return_path
|
||||
end
|
||||
return
|
||||
end
|
||||
# Fallback: Show the score and allow learners to return to the LTI consumer.
|
||||
redirect_to_lti_return_path
|
||||
end
|
||||
|
||||
@ -74,9 +33,9 @@ module RedirectBehavior
|
||||
end
|
||||
|
||||
def redirect_to_community_solution?
|
||||
return false unless Java21Study.allow_redirect_to_community_solution?(current_user, @exercise)
|
||||
return false unless Java21Study.allow_redirect_to_community_solution?(current_user, @submission.exercise)
|
||||
|
||||
@community_solution = CommunitySolution.find_by(exercise: @exercise)
|
||||
@community_solution = CommunitySolution.find_by(exercise: @submission.exercise)
|
||||
return false if @community_solution.blank?
|
||||
|
||||
last_contribution = CommunitySolutionContribution.where(community_solution: @community_solution).order(created_at: :asc).last
|
||||
@ -100,11 +59,11 @@ module RedirectBehavior
|
||||
end
|
||||
|
||||
def redirect_to_user_feedback
|
||||
uef = UserExerciseFeedback.find_by(exercise: @exercise, user: current_user)
|
||||
uef = UserExerciseFeedback.find_by(exercise: @submission.exercise, user: current_user)
|
||||
url = if uef
|
||||
edit_user_exercise_feedback_path(uef)
|
||||
else
|
||||
new_user_exercise_feedback_path(user_exercise_feedback: {exercise_id: @exercise.id})
|
||||
new_user_exercise_feedback_path(user_exercise_feedback: {exercise_id: @submission.exercise.id})
|
||||
end
|
||||
|
||||
respond_to do |format|
|
||||
@ -113,6 +72,32 @@ module RedirectBehavior
|
||||
end
|
||||
end
|
||||
|
||||
def redirect_to_unsolved_rfc(own: false)
|
||||
# Set a message that informs the user that their own RFC should be closed or help in another RFC is greatly appreciated.
|
||||
flash[:notice] = I18n.t("exercises.editor.exercise_finished_redirect_to_#{own ? 'own_' : ''}rfc")
|
||||
flash.keep(:notice)
|
||||
|
||||
# Increase counter 'times_featured' in rfc
|
||||
@rfc.increment(:times_featured) unless own
|
||||
|
||||
respond_to do |format|
|
||||
format.html { redirect_to(@rfc) }
|
||||
format.json { render(json: {redirect: url_for(@rfc)}) }
|
||||
end
|
||||
end
|
||||
|
||||
def redirect_to_own_unsolved_rfc?
|
||||
@rfc = @submission.own_unsolved_rfc(current_user)
|
||||
@rfc.present?
|
||||
end
|
||||
|
||||
def redirect_to_unsolved_rfc?
|
||||
return false if @embed_options[:disable_redirect_to_rfcs] || @embed_options[:disable_rfc]
|
||||
|
||||
@rfc = @submission.unsolved_rfc(current_user)
|
||||
@rfc.present?
|
||||
end
|
||||
|
||||
def redirect_to_lti_return_path
|
||||
Sentry.set_extras(
|
||||
consumers_id: current_user.consumer_id,
|
||||
|
||||
@ -10,7 +10,7 @@ class ExercisesController < ApplicationController
|
||||
before_action :handle_file_uploads, only: %i[create update]
|
||||
before_action :set_execution_environments, only: %i[index create edit new update]
|
||||
before_action :set_exercise_and_authorize,
|
||||
only: MEMBER_ACTIONS + %i[clone implement working_times intervention statistics submit reload feedback
|
||||
only: MEMBER_ACTIONS + %i[clone implement working_times intervention statistics reload feedback
|
||||
study_group_dashboard export_external_check export_external_confirm
|
||||
external_user_statistics]
|
||||
before_action :collect_set_and_unset_exercise_tags, only: MEMBER_ACTIONS
|
||||
@ -548,57 +548,6 @@ class ExercisesController < ApplicationController
|
||||
render 'exercises/external_users/statistics'
|
||||
end
|
||||
|
||||
def submit
|
||||
@submission = Submission.create(submission_params)
|
||||
@submission.calculate_score(current_user)
|
||||
|
||||
if @submission.users.map {|user| lti_outcome_service?(@submission.exercise, user, @submission.study_group_id) }.any?
|
||||
transmit_lti_score
|
||||
else
|
||||
redirect_after_submit
|
||||
end
|
||||
rescue Runner::Error => e
|
||||
Rails.logger.debug { "Runner error while submitting submission #{@submission.id}: #{e.message}" }
|
||||
respond_to do |format|
|
||||
format.html { redirect_to(implement_exercise_path(@submission.exercise)) }
|
||||
format.json { render(json: {danger: I18n.t('exercises.editor.depleted'), status: :container_depleted}) }
|
||||
end
|
||||
end
|
||||
|
||||
def transmit_lti_score
|
||||
responses = send_scores(@submission)
|
||||
messages = {}
|
||||
failed_users = []
|
||||
|
||||
responses.each do |response|
|
||||
if Lti::ERROR_STATUS.include? response[:status]
|
||||
failed_users << response[:user]
|
||||
elsif response[:score_sent] != @submission.normalized_score # the score was sent successfully, but received too late
|
||||
messages[:warning] = I18n.t('exercises.submit.too_late')
|
||||
end
|
||||
end
|
||||
|
||||
if failed_users.size == responses.size # all submissions failed
|
||||
messages[:danger] = I18n.t('exercises.submit.failure')
|
||||
elsif failed_users.size.positive? # at least one submission failed
|
||||
messages[:warning] = [[messages[:warning]], I18n.t('exercises.submit.warning_not_for_all_users_submitted', user: failed_users.join(', '))].join('<br><br>')
|
||||
messages[:warning] = "#{messages[:warning]}\n\n#{I18n.t('exercises.submit.warning_not_for_all_users_submitted', user: failed_users.join(', '))}".strip
|
||||
else
|
||||
messages.each do |type, message_text|
|
||||
flash.now[type] = message_text
|
||||
flash.keep(type)
|
||||
end
|
||||
return redirect_after_submit
|
||||
end
|
||||
|
||||
respond_to do |format|
|
||||
format.html { redirect_to(implement_exercise_path(@submission.exercise), **messages) }
|
||||
format.json { render(json: messages) } # We must not change the HTTP status code here, since otherwise the custom message is not displayed.
|
||||
end
|
||||
end
|
||||
|
||||
private :transmit_lti_score
|
||||
|
||||
def destroy
|
||||
destroy_and_respond(object: @exercise)
|
||||
end
|
||||
|
||||
@ -52,11 +52,11 @@ class RemoteEvaluationController < ApplicationController
|
||||
def process_lti_response(lti_response)
|
||||
if (lti_response[:status] == 'success') && (lti_response[:score_sent] != @submission.normalized_score)
|
||||
# Score has been reduced due to the passed deadline
|
||||
{message: I18n.t('exercises.submit.too_late'), status: 207, score: lti_response[:score_sent] * 100}
|
||||
{message: I18n.t('exercises.editor.submit_too_late', score_sent: lti_response[:score_sent] * 100), status: 207, score: lti_response[:score_sent] * 100}
|
||||
elsif lti_response[:status] == 'success'
|
||||
{message: I18n.t('sessions.destroy_through_lti.success_with_outcome', consumer: @submission.user.consumer.name), status: 202}
|
||||
else
|
||||
{message: I18n.t('exercises.submit.failure'), status: 424}
|
||||
{message: I18n.t('exercises.editor.submit_failure_all'), status: 424}
|
||||
end
|
||||
# TODO: Delete LTI parameters?
|
||||
end
|
||||
|
||||
@ -2,12 +2,13 @@
|
||||
|
||||
class SubmissionsController < ApplicationController
|
||||
include CommonBehavior
|
||||
include Lti
|
||||
include FileConversion
|
||||
include Lti
|
||||
include RedirectBehavior
|
||||
include SubmissionParameters
|
||||
include Tubesock::Hijack
|
||||
|
||||
before_action :set_submission, only: %i[download download_file run score show statistics test]
|
||||
before_action :set_submission, only: %i[download download_file run score show statistics test finalize]
|
||||
before_action :set_testrun, only: %i[run score test]
|
||||
before_action :set_files, only: %i[download show]
|
||||
before_action :set_files_and_specific_file, only: %i[download_file run test]
|
||||
@ -72,6 +73,11 @@ class SubmissionsController < ApplicationController
|
||||
end
|
||||
end
|
||||
|
||||
def finalize
|
||||
@submission.update!(cause: 'submit')
|
||||
redirect_after_submit
|
||||
end
|
||||
|
||||
def show; end
|
||||
|
||||
def render_file
|
||||
@ -166,7 +172,7 @@ class SubmissionsController < ApplicationController
|
||||
durations = @submission.run(@file) do |socket, starting_time|
|
||||
runner_socket = socket
|
||||
@testrun[:starting_time] = starting_time
|
||||
client_socket.send_data JSON.dump({cmd: :status, status: :container_running})
|
||||
client_socket.send_data({cmd: :status, status: :container_running}.to_json)
|
||||
|
||||
runner_socket.on :stdout do |data|
|
||||
message = retrieve_message_from_output data, :stdout
|
||||
@ -256,9 +262,11 @@ class SubmissionsController < ApplicationController
|
||||
return true if disable_scoring
|
||||
|
||||
# The score is stored separately, we can forward it to the client immediately
|
||||
client_socket&.send_data(JSON.dump(@submission.calculate_score(current_user)))
|
||||
client_socket&.send_data(@submission.calculate_score(current_user).to_json)
|
||||
# To enable hints when scoring a submission, uncomment the next line:
|
||||
# send_hints(client_socket, StructuredError.where(submission: @submission))
|
||||
|
||||
transmit_lti_score(client_socket)
|
||||
rescue Runner::Error::RunnerInUse => e
|
||||
extract_durations(e)
|
||||
send_and_store client_socket, {cmd: :status, status: :runner_in_use}
|
||||
@ -300,7 +308,7 @@ class SubmissionsController < ApplicationController
|
||||
return true if @embed_options[:disable_run]
|
||||
|
||||
# The score is stored separately, we can forward it to the client immediately
|
||||
client_socket&.send_data(JSON.dump(@submission.test(@file, current_user)))
|
||||
client_socket&.send_data(@submission.test(@file, current_user).to_json)
|
||||
rescue Runner::Error::RunnerInUse => e
|
||||
extract_durations(e)
|
||||
send_and_store client_socket, {cmd: :status, status: :runner_in_use}
|
||||
@ -338,7 +346,7 @@ class SubmissionsController < ApplicationController
|
||||
return unless client_socket
|
||||
|
||||
# We don't want to store this (arbitrary) exit command and redirect it ourselves
|
||||
client_socket.send_data JSON.dump({cmd: :exit})
|
||||
client_socket.send_data({cmd: :exit}.to_json)
|
||||
client_socket.send_data nil, :close
|
||||
# We must not close the socket manually (with `client_socket.close`), as this would close it twice.
|
||||
# When the socket is closed twice, nginx registers a `Connection reset by peer` error.
|
||||
@ -401,7 +409,7 @@ class SubmissionsController < ApplicationController
|
||||
end
|
||||
@testrun[:messages].push message
|
||||
@testrun[:status] = message[:status] if message[:status]
|
||||
client_socket.send_data JSON.dump(message)
|
||||
client_socket.send_data(message.to_json)
|
||||
end
|
||||
|
||||
def max_output_buffer_size
|
||||
@ -473,6 +481,53 @@ class SubmissionsController < ApplicationController
|
||||
}
|
||||
end
|
||||
|
||||
def check_scoring_too_late(submit_info)
|
||||
# The submission was either performed before any deadline or no deadline was configured at all for the current exercise.
|
||||
return if %i[within_grace_period after_late_deadline].exclude? submit_info[:deadline]
|
||||
# The `lis_outcome_service` was not provided by the LMS, hence we were not able to send any score.
|
||||
return if submit_info[:users][:unsupported].include?(current_user)
|
||||
|
||||
{status: :scoring_too_late, score_sent: submit_info[:score][:sent]}
|
||||
end
|
||||
|
||||
def check_full_score
|
||||
# The submission was not scored with the full score, hence the exercise is not finished yet.
|
||||
return unless @submission.full_score?
|
||||
|
||||
{status: :exercise_finished, url: finalize_submission_path(@submission)}
|
||||
end
|
||||
|
||||
def transmit_lti_score(client_socket)
|
||||
submit_info = send_scores(@submission)
|
||||
scored_users = submit_info[:users]
|
||||
|
||||
notifications = []
|
||||
if scored_users[:all] == scored_users[:error] || scored_users[:error].include?(current_user)
|
||||
# The score was not sent for any user or sending the score for the current user failed.
|
||||
# In the latter case, we want to encourage the current user to reopen the exercise through the LMS.
|
||||
# Hence, we always display the most severe error message.
|
||||
notifications << {status: :scoring_failure}
|
||||
elsif scored_users[:all] != scored_users[:success] && scored_users[:success].include?(current_user)
|
||||
# The score was sent successfully for current user.
|
||||
# However, at the same time, the transmission failed for some other users.
|
||||
# This could either be due to a temporary network error, which is unlikely, or a more "permanent" error.
|
||||
# Permanent errors would be that the deadline has passed on the LMS (which would then not provide a `lis_outcome_service`),
|
||||
# working together with an internal user, or with someone who has never opened the exercise before.
|
||||
notifications << {status: :not_for_all_users_submitted, failed_users: scored_users[:error].map(&:displayname).join(', ')}
|
||||
end
|
||||
|
||||
if notifications.empty? || notifications.first[:status] != :scoring_failure
|
||||
# Either, the score was sent successfully for the current user,
|
||||
# or it was not attempted for any user (i.e., no `lis_outcome_service`).
|
||||
notifications << check_scoring_too_late(submit_info)
|
||||
notifications << check_full_score
|
||||
end
|
||||
|
||||
notifications.compact.each do |notification|
|
||||
client_socket&.send_data(notification&.merge(cmd: :status)&.to_json)
|
||||
end
|
||||
end
|
||||
|
||||
def retrieve_message_from_output(data, stream)
|
||||
parsed = JSON.parse(data)
|
||||
if parsed.instance_of?(Hash) && parsed.key?('cmd')
|
||||
|
||||
Reference in New Issue
Block a user