175c8933f3
With this commit, we refactor the overall score handling of CodeOcean. Previously, "Score" and "Submit" were two distinct actions, requiring users to confirm the LTI transmission of their score (after assessing their submission). This yielded many questions and was unnecessary, since LTI parameters are no longer expiring after each use. Therefore, we can now transmit the current grade on each score run with the very same LTI parameters. As a consequence, the LTI consumer gets a more detailed history of the scores, enabling further analytical insights. For users, the previous "Submit" button got replaced with a notification that is shown as soon as the full score got reached. Then, learners can decide to "finalize" their work on the given exercise, which will initiate a redirect to a follow-up action (as defined in the RedirectBehavior). This RedirectBehavior has also been unified and simplified for better readability. As part of this refactoring, we rephrased the notifications and UX workflow of a) the LTI transmission, b) the finalization of an exercise (measured by reaching the full score) and c) the deadline handling (on time, within grace period, too late). Those information are now separately shown, potentially resulting in multiple notifications. As a side effect, they are much better maintainable, and the LTI transmission is more decoupled from this notification handling.
318 lines
12 KiB
Ruby
318 lines
12 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'rails_helper'
|
|
|
|
RSpec.describe 'Score', :js do
|
|
let(:exercise) { create(:hello_world) }
|
|
let(:contributor) { create(:external_user) }
|
|
let(:submission) { create(:submission, exercise:, contributor:, score:) }
|
|
|
|
before do
|
|
allow_any_instance_of(ApplicationController).to receive(:current_user).and_return(contributor)
|
|
allow(Submission).to receive(:find).and_return(submission)
|
|
visit(implement_exercise_path(exercise))
|
|
end
|
|
|
|
shared_examples 'exercise finished notification' do
|
|
it "shows an 'exercise finished' notification" do
|
|
# Text needs to be split because it includes the embedded URL in the HTML which is not shown in the notification.
|
|
# We compare the shown notification text and the URL separately.
|
|
expect(page).to have_content(I18n.t('exercises.editor.exercise_finished').split('.').first)
|
|
expect(page).to have_link(nil, href: finalize_submission_path(submission))
|
|
end
|
|
end
|
|
|
|
shared_examples 'no exercise finished notification' do
|
|
it "does not show an 'exercise finished' notification" do
|
|
# Text needs to be split because it includes the embedded URL in the HTML which is not shown in the notification.
|
|
# We compare the shown notification text and the URL separately.
|
|
expect(page).not_to have_content(I18n.t('exercises.editor.exercise_finished').split('.').first)
|
|
expect(page).not_to have_link(nil, href: finalize_submission_path(submission))
|
|
end
|
|
end
|
|
|
|
shared_examples 'notification' do |message_key|
|
|
it "shows a '#{message_key.split('.').last}' notification" do
|
|
options = {}
|
|
options[:score_sent] = (score_sent * 100).to_i if defined? score_sent
|
|
options[:user] = users_error.map(&:displayname).join(', ') if defined? users_error
|
|
|
|
expect(page).to have_content(I18n.t(message_key, **options))
|
|
end
|
|
end
|
|
|
|
shared_examples 'no notification' do |message_key|
|
|
it "does not show a '#{message_key.split('.').last}' notification" do
|
|
expect(page).not_to have_content(I18n.t(message_key))
|
|
end
|
|
end
|
|
|
|
context 'when scoring is successful' do
|
|
let(:lti_outcome_service?) { true }
|
|
|
|
let(:scoring_response) do
|
|
{
|
|
users: {all: users_success + users_error + users_unsupported, success: users_success, error: users_error, unsupported: users_unsupported},
|
|
score: {original: score, sent: score_sent},
|
|
deadline:,
|
|
detailed_results: [],
|
|
}
|
|
end
|
|
|
|
let(:calculate_response) do
|
|
[{
|
|
status: :ok,
|
|
stdout: '',
|
|
stderr: '',
|
|
waiting_for_container_time: 0,
|
|
container_execution_time: 0,
|
|
file_role: :teacher_defined_test,
|
|
count: 1,
|
|
failed: 0,
|
|
error_messages: [],
|
|
passed: 1,
|
|
score:,
|
|
filename: 'exercise_spec.rb',
|
|
message: 'Well done.',
|
|
weight: 1.0,
|
|
hidden_feedback: false,
|
|
exit_code: 0,
|
|
}]
|
|
end
|
|
|
|
before do
|
|
allow_any_instance_of(LtiHelper).to receive(:lti_outcome_service?).and_return(lti_outcome_service?)
|
|
allow(submission).to receive(:calculate_score).and_return(calculate_response)
|
|
allow_any_instance_of(SubmissionsController).to receive(:send_scores).and_return(scoring_response)
|
|
click_button(I18n.t('exercises.editor.score'))
|
|
end
|
|
|
|
shared_context 'when full score reached' do
|
|
let(:score) { 1 }
|
|
end
|
|
|
|
shared_context 'when full score is not reached' do
|
|
let(:score) { 0 }
|
|
end
|
|
|
|
shared_context 'when scored without deadline' do
|
|
let(:deadline) { :none }
|
|
let(:score_sent) { score }
|
|
end
|
|
|
|
shared_context 'when scored before deadline' do
|
|
let(:deadline) { :before_deadline }
|
|
let(:score_sent) { score }
|
|
end
|
|
|
|
shared_context 'when scored within grace period' do
|
|
let(:deadline) { :within_grace_period }
|
|
let(:score_sent) { score * 0.8 }
|
|
end
|
|
|
|
shared_context 'when scored after late deadline' do
|
|
let(:deadline) { :after_late_deadline }
|
|
let(:score_sent) { score * 0 }
|
|
end
|
|
|
|
context 'when the LTI outcome service is supported' do
|
|
describe 'LTI failure' do
|
|
let(:users_success) { [] }
|
|
let(:users_error) { [contributor] }
|
|
let(:users_unsupported) { [] }
|
|
|
|
context 'when full score is reached' do
|
|
include_context 'when full score reached'
|
|
|
|
%w[without_deadline before_deadline within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when full score is not reached' do
|
|
include_context 'when full score is not reached'
|
|
|
|
%w[without_deadline before_deadline within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'LTI success' do
|
|
let(:users_success) { [contributor] }
|
|
let(:users_error) { [] }
|
|
let(:users_unsupported) { [] }
|
|
|
|
context 'when full score is reached' do
|
|
include_context 'when full score reached'
|
|
|
|
%w[without_deadline before_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
|
|
%w[within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when full score is not reached' do
|
|
include_context 'when full score is not reached'
|
|
|
|
%w[within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'LTI success for current contributor and failure for other' do
|
|
let(:users_success) { [contributor] }
|
|
let(:users_error) { [create(:external_user)] }
|
|
let(:users_unsupported) { [] }
|
|
|
|
context 'when full score is reached' do
|
|
include_context 'when full score reached'
|
|
|
|
%w[without_deadline before_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
|
|
%w[within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when full score is not reached' do
|
|
include_context 'when full score is not reached'
|
|
|
|
%w[within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when the LTI outcomes are not supported' do
|
|
let(:lti_outcome_service?) { false }
|
|
let(:users_success) { [] }
|
|
let(:users_error) { [] }
|
|
let(:users_unsupported) { [contributor] }
|
|
|
|
context 'when full score is reached' do
|
|
include_context 'when full score reached'
|
|
|
|
%w[without_deadline before_deadline within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when full score is not reached' do
|
|
include_context 'when full score is not reached'
|
|
|
|
%w[without_deadline before_deadline within_grace_period after_late_deadline].each do |scenario|
|
|
context "when scored #{scenario.tr('_', ' ')}" do
|
|
include_context "when scored #{scenario.tr('_', ' ')}"
|
|
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when scoring is not successful' do
|
|
let(:score) { 0 }
|
|
|
|
context 'when the desired runner is already in use' do
|
|
before do
|
|
allow(submission).to receive(:calculate_score).and_raise(Runner::Error::RunnerInUse)
|
|
click_button(I18n.t('exercises.editor.score'))
|
|
end
|
|
|
|
it_behaves_like 'notification', 'exercises.editor.runner_in_use'
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
|
|
context 'when no runner is available' do
|
|
before do
|
|
allow(submission).to receive(:calculate_score).and_raise(Runner::Error::NotAvailable)
|
|
click_button(I18n.t('exercises.editor.score'))
|
|
end
|
|
|
|
it_behaves_like 'notification', 'exercises.editor.depleted'
|
|
it_behaves_like 'no exercise finished notification'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_all'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_failure_other_users'
|
|
it_behaves_like 'no notification', 'exercises.editor.submit_too_late'
|
|
end
|
|
end
|
|
end
|