diff --git a/.gitignore b/.gitignore index af16e87b..c1030982 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ /config/*.staging-epic.yml /config/deploy/staging-epic.rb /coverage -/log +/log/*.* /public/assets /public/uploads /rubocop.html diff --git a/Capfile b/Capfile index 620d59ef..30744292 100644 --- a/Capfile +++ b/Capfile @@ -6,6 +6,7 @@ require 'capistrano/puma/nginx' require 'capistrano/rails' require 'capistrano/rvm' require 'capistrano/upload-config' +require 'whenever/capistrano' install_plugin Capistrano::SCM::Git install_plugin Capistrano::Puma diff --git a/Gemfile b/Gemfile index 8482ca69..6a6486a6 100644 --- a/Gemfile +++ b/Gemfile @@ -43,6 +43,7 @@ gem 'nokogiri' gem 'd3-rails' gem 'rest-client' gem 'rubyzip' +gem 'whenever', require: false group :development, :staging do gem 'better_errors', platform: :ruby diff --git a/Gemfile.lock b/Gemfile.lock index 43737a82..7edb56f0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,11 +32,6 @@ GEM activesupport (= 4.2.10) arel (~> 6.0) activerecord-deprecated_finders (1.0.4) - activerecord-jdbc-adapter (50.0) - activerecord (>= 2.2) - activerecord-jdbcpostgresql-adapter (50.0) - activerecord-jdbc-adapter (~> 50.0) - jdbc-postgres (>= 9.4, < 43) activesupport (4.2.10) i18n (~> 0.7) minitest (~> 5.1) @@ -51,7 +46,6 @@ GEM autotest-rails (4.2.1) ZenTest (~> 4.5) bcrypt (3.1.11) - bcrypt (3.1.11-java) better_errors (2.4.0) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -97,6 +91,7 @@ GEM mime-types (>= 1.16) childprocess (0.8.0) ffi (~> 1.0, >= 1.0.11) + chronic (0.10.2) codeclimate-test-reporter (1.0.7) simplecov coderay (1.1.2) @@ -108,25 +103,23 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.0.5) - concurrent-ruby (1.0.5-java) concurrent-ruby-ext (1.0.5) concurrent-ruby (= 1.0.5) crass (1.0.3) - d3-rails (4.10.2) + d3-rails (4.13.0) railties (>= 3.1) database_cleaner (1.6.2) debug_inspector (0.0.3) diff-lcs (1.3) docile (1.1.5) - docker-api (1.34.0) + docker-api (1.34.1) excon (>= 0.47.0) multi_json domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) - erubi (1.7.0) + erubi (1.7.1) erubis (2.7.0) eventmachine (1.0.9.1) - eventmachine (1.0.9.1-java) excon (0.60.0) execjs (2.7.0) factory_bot (4.8.2) @@ -139,8 +132,7 @@ GEM faye-websocket (0.10.7) eventmachine (>= 0.12.0) websocket-driver (>= 0.5.1) - ffi (1.9.21) - ffi (1.9.21-java) + ffi (1.9.23) forgery (0.7.0) globalid (0.4.1) activesupport (>= 4.2.0) @@ -155,7 +147,6 @@ GEM jbuilder (2.7.0) activesupport (>= 4.2.0) multi_json (>= 1.2) - jdbc-postgres (42.1.4) jquery-rails (4.3.1) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) @@ -164,7 +155,6 @@ GEM railties (>= 3.1.0) turbolinks json (2.1.0) - json (2.1.0-java) jwt (1.5.6) kramdown (1.16.2) loofah (2.2.0) @@ -189,7 +179,6 @@ GEM newrelic_rpm (4.8.0.341) nokogiri (1.8.2) mini_portile2 (~> 2.3.0) - nokogiri (1.8.2-java) nyan-cat-formatter (0.12.0) rspec (>= 2.99, >= 2.14.2, < 4) oauth (0.4.7) @@ -202,7 +191,7 @@ GEM pagedown-rails (1.1.4) railties (> 3.1) parallel (1.12.1) - parser (2.5.0.2) + parser (2.5.0.3) ast (~> 2.4.0) pg (0.21.0) polyamorous (1.3.3) @@ -211,19 +200,14 @@ GEM pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) - pry (0.11.3-java) - coderay (~> 1.1.0) - method_source (~> 0.9.0) - spoon (~> 0.0) pry-byebug (3.6.0) byebug (~> 10.0) pry (~> 0.10) public_suffix (3.0.2) - puma (3.11.2) - puma (3.11.2-java) + puma (3.11.3) pundit (1.1.0) activesupport (>= 3.0.0) - rack (1.6.8) + rack (1.6.9) rack-mini-profiler (0.10.7) rack (>= 1.2.0) rack-test (0.6.3) @@ -263,7 +247,7 @@ GEM activesupport (>= 3.0) i18n polyamorous (~> 1.3.2) - rb-fsevent (0.10.2) + rb-fsevent (0.10.3) rb-inotify (0.9.10) ffi (>= 0.5.0, < 2) rdoc (6.0.1) @@ -294,15 +278,15 @@ GEM rspec-mocks (~> 3.7.0) rspec-support (~> 3.7.0) rspec-support (3.7.1) - rubocop (0.52.1) + rubocop (0.53.0) parallel (~> 1.10) - parser (>= 2.4.0.2, < 3.0) + parser (>= 2.5) powerpack (~> 0.1) rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - rubocop-rspec (1.22.2) - rubocop (>= 0.52.1) + rubocop-rspec (1.24.0) + rubocop (>= 0.53.0) ruby-progressbar (1.9.0) rubytree (1.0.0) json (~> 2.1) @@ -321,7 +305,7 @@ GEM tilt (>= 1.1, < 3) sdoc (1.0.0) rdoc (>= 5.0) - selenium-webdriver (3.9.0) + selenium-webdriver (3.10.0) childprocess (~> 0.5) rubyzip (~> 1.2) simplecov (0.15.1) @@ -336,8 +320,6 @@ GEM bcrypt (~> 3.1) oauth (~> 0.4, >= 0.4.4) oauth2 (~> 1.0, >= 0.8.0) - spoon (0.0.6) - ffi spring (2.0.2) activesupport (>= 4.2) sprockets (3.7.1) @@ -354,7 +336,6 @@ GEM temple (0.8.0) thor (0.20.0) thread_safe (0.3.6) - thread_safe (0.3.6-java) tilt (2.0.8) tubesock (0.2.7) rack (>= 1.5.0) @@ -367,7 +348,6 @@ GEM execjs (>= 0.3.0, < 3) unf (0.1.4) unf_ext - unf (0.1.4-java) unf_ext (0.0.7.5) unicode-display_width (1.3.0) web-console (3.3.0) @@ -377,15 +357,14 @@ GEM websocket (1.2.5) websocket-driver (0.7.0) websocket-extensions (>= 0.1.0) - websocket-driver (0.7.0-java) - websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) + whenever (0.10.0) + chronic (>= 0.6.3) will_paginate (3.1.6) xpath (3.0.0) nokogiri (~> 1.8) PLATFORMS - java ruby DEPENDENCIES @@ -454,6 +433,7 @@ DEPENDENCIES turbolinks (< 5.0.0) uglifier web-console + whenever will_paginate BUNDLED WITH diff --git a/Vagrantfile b/Vagrantfile index 5ec9182e..cc6acf2c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,5 +8,5 @@ Vagrant.configure(2) do |config| end config.vm.network "private_network", ip: "192.168.59.104" # config.vm.synced_folder "../data", "/vagrant_data" - config.vm.provision "shell", path: "provision.sh" + config.vm.provision "shell", path: "provision.sh", privileged: false end diff --git a/app/controllers/exercise_collections_controller.rb b/app/controllers/exercise_collections_controller.rb index 6d239d9c..3a6112ef 100644 --- a/app/controllers/exercise_collections_controller.rb +++ b/app/controllers/exercise_collections_controller.rb @@ -10,6 +10,7 @@ class ExerciseCollectionsController < ApplicationController end def show + @exercises = @exercise_collection.exercises.paginate(:page => params[:page]) end def new @@ -55,6 +56,6 @@ class ExerciseCollectionsController < ApplicationController end def exercise_collection_params - params[:exercise_collection].permit(:name, :exercise_ids => []) + params[:exercise_collection].permit(:name, :use_anomaly_detection, :user_id, :user_type, :exercise_ids => []).merge(user_type: InternalUser.name) end end diff --git a/app/controllers/user_exercise_feedbacks_controller.rb b/app/controllers/user_exercise_feedbacks_controller.rb index 8abffd66..4c350a81 100644 --- a/app/controllers/user_exercise_feedbacks_controller.rb +++ b/app/controllers/user_exercise_feedbacks_controller.rb @@ -69,7 +69,8 @@ class UserExerciseFeedbacksController < ApplicationController @texts = comment_presets.to_a @times = time_presets.to_a @uef = UserExerciseFeedback.new - @exercise = Exercise.find(params[:user_exercise_feedback][:exercise_id]) + exercise_id = if params[:user_exercise_feedback].nil? then params[:exercise_id] else params[:user_exercise_feedback][:exercise_id] end + @exercise = Exercise.find(exercise_id) authorize! end diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb index 8022ee84..497c2a31 100644 --- a/app/mailers/user_mailer.rb +++ b/app/mailers/user_mailer.rb @@ -37,4 +37,18 @@ class UserMailer < ActionMailer::Base @rfc_link = request_for_comment_url(request_for_comments) mail(subject: t('mailers.user_mailer.send_thank_you_note.subject', author: @author), to: receiver.email) end + + def exercise_anomaly_detected(exercise_collection, anomalies) + @receiver_displayname = exercise_collection.user.displayname + @collection = exercise_collection + @anomalies = anomalies + mail(subject: t('mailers.user_mailer.exercise_anomaly_detected.subject'), to: exercise_collection.user.email) + end + + def exercise_anomaly_needs_feedback(user, exercise, link) + @receiver_displayname = user.displayname + @exercise_title = exercise.title + @link = link + mail(subject: t('mailers.user_mailer.exercise_anomaly_needs_feedback.subject'), to: user.email) + end end diff --git a/app/models/anomaly_notification.rb b/app/models/anomaly_notification.rb new file mode 100644 index 00000000..8c9b90cd --- /dev/null +++ b/app/models/anomaly_notification.rb @@ -0,0 +1,5 @@ +class AnomalyNotification < ActiveRecord::Base + belongs_to :user, polymorphic: true + belongs_to :exercise + belongs_to :exercise_collection +end diff --git a/app/models/exercise.rb b/app/models/exercise.rb index f03d6641..efdb00c7 100644 --- a/app/models/exercise.rb +++ b/app/models/exercise.rb @@ -36,6 +36,7 @@ class Exercise < ActiveRecord::Base validates :token, presence: true, uniqueness: true @working_time_statistics = nil + attr_reader :working_time_statistics MAX_EXERCISE_FEEDBACKS = 20 @@ -65,21 +66,27 @@ class Exercise < ActiveRecord::Base end def user_working_time_query - """ + " SELECT user_id, - sum(working_time_new) AS working_time + user_type, + SUM(working_time_new) AS working_time, + MAX(score) AS score FROM (SELECT user_id, + user_type, + score, CASE WHEN working_time >= '0:05:00' THEN '0' ELSE working_time END AS working_time_new FROM (SELECT user_id, + user_type, + score, id, (created_at - lag(created_at) over (PARTITION BY user_id, exercise_id ORDER BY created_at)) AS working_time FROM submissions WHERE exercise_id=#{id}) AS foo) AS bar - GROUP BY user_id - """ + GROUP BY user_id, user_type + " end def get_quantiles(quantiles) @@ -202,7 +209,7 @@ class Exercise < ActiveRecord::Base def retrieve_working_time_statistics @working_time_statistics = {} self.class.connection.execute(user_working_time_query).each do |tuple| - @working_time_statistics[tuple["user_id"].to_i] = tuple + @working_time_statistics[tuple['user_id'].to_i] = tuple end end @@ -368,4 +375,15 @@ class Exercise < ActiveRecord::Base user_exercise_feedbacks.size <= MAX_EXERCISE_FEEDBACKS end + def last_submission_per_user + Submission.joins("JOIN ( + SELECT + user_id, + user_type, + first_value(id) OVER (PARTITION BY user_id ORDER BY created_at DESC) AS fv + FROM submissions + WHERE exercise_id = #{id} + ) AS t ON t.fv = submissions.id").distinct + end + end diff --git a/app/models/exercise_collection.rb b/app/models/exercise_collection.rb index 10946962..f9f09269 100644 --- a/app/models/exercise_collection.rb +++ b/app/models/exercise_collection.rb @@ -1,6 +1,7 @@ class ExerciseCollection < ActiveRecord::Base has_and_belongs_to_many :exercises + belongs_to :user, polymorphic: true def to_s "#{I18n.t('activerecord.models.exercise_collection.one')}: #{name} (#{id})" diff --git a/app/views/exercise_collections/_form.html.slim b/app/views/exercise_collections/_form.html.slim index 3c336a62..c204db47 100644 --- a/app/views/exercise_collections/_form.html.slim +++ b/app/views/exercise_collections/_form.html.slim @@ -1,11 +1,18 @@ - exercises = Exercise.order(:title) +- users = InternalUser.order(:name) -= form_for(@exercise_collection, data: {exercises: exercises}, multipart: true) do |f| += form_for(@exercise_collection, data: {exercises: exercises, users: users}, multipart: true) do |f| = render('shared/form_errors', object: @exercise_collection) .form-group - = f.label(:name) + = f.label(t('activerecord.attributes.exercise_collections.name')) = f.text_field(:name, class: 'form-control', required: true) .form-group - = f.label(:exercises) + = f.label(t('activerecord.attributes.exercise_collections.use_anomaly_detection')) + = f.check_box(:use_anomaly_detection, {class: 'form-control'}) + .form-group + = f.label(t('activerecord.attributes.exercise_collections.user')) + = f.collection_select(:user_id, users, :id, :name, {}, {class: 'form-control'}) + .form-group + = f.label(t('activerecord.attributes.exercise_collections.exercises')) = f.collection_select(:exercise_ids, exercises, :id, :title, {}, {class: 'form-control', multiple: true}) .actions = render('shared/submit_button', f: f, object: @exercise_collection) diff --git a/app/views/exercise_collections/show.html.slim b/app/views/exercise_collections/show.html.slim index 65c28283..ab85a3ba 100644 --- a/app/views/exercise_collections/show.html.slim +++ b/app/views/exercise_collections/show.html.slim @@ -3,9 +3,25 @@ h1 = render('shared/edit_button', object: @exercise_collection) = row(label: 'exercise_collections.name', value: @exercise_collection.name) += row(label: 'exercise_collections.user', value: link_to(@exercise_collection.user.name, @exercise_collection.user)) unless @exercise_collection.user.nil? += row(label: 'exercise_collections.use_anomaly_detection', value: @exercise_collection.use_anomaly_detection) = row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at) h4 = t('activerecord.attributes.exercise_collections.exercises') -ul.list-unstyled - - @exercise_collection.exercises.sort_by{|c| c.title}.each do |exercise| - li = link_to(exercise, exercise) +.table-responsive + table.table + thead + tr + th = t('activerecord.attributes.exercise.title') + th = t('activerecord.attributes.exercise.execution_environment') + th = t('activerecord.attributes.exercise.user') + th = t('shared.actions') + tbody + - @exercises.sort_by{|c| c.title}.each do |exercise| + tr + td = link_to(exercise.title, exercise) + td = link_to_if(exercise.execution_environment && policy(exercise.execution_environment).show?, exercise.execution_environment, exercise.execution_environment) + td = exercise.user.name + td = link_to(t('shared.statistics'), statistics_exercise_path(exercise)) + += render('shared/pagination', collection: @exercises) diff --git a/app/views/user_mailer/exercise_anomaly_detected.html.slim b/app/views/user_mailer/exercise_anomaly_detected.html.slim new file mode 100644 index 00000000..56233ce0 --- /dev/null +++ b/app/views/user_mailer/exercise_anomaly_detected.html.slim @@ -0,0 +1,38 @@ +== t('mailers.user_mailer.exercise_anomaly_detected.body1', + receiver_displayname: @receiver_displayname, + collection_name: @collection.name) + +table(border=1) + thead + tr + td = t('activerecord.attributes.exercise.title', locale: :de) + td = t('exercises.statistics.average_worktime', locale: :de) + td = t('shared.actions', locale: :de) + tbody + - @anomalies.keys.each do | id | + - exercise = Exercise.find(id) + tr + td = link_to(exercise.title, exercise_path(exercise)) + td = @anomalies[id] + td = link_to(t('shared.statistics', locale: :de), statistics_exercise_path(exercise)) + + +== t('mailers.user_mailer.exercise_anomaly_detected.body2', + receiver_displayname: @receiver_displayname, + collection_name: @collection.name) + +table(border=1) + thead + tr + td = t('activerecord.attributes.exercise.title', locale: :en) + td = t('exercises.statistics.average_worktime', locale: :en) + td = t('shared.actions', locale: :en) + tbody + - @anomalies.keys.each do | id | + - exercise = Exercise.find(id) + tr + td = link_to(exercise.title, exercise_path(exercise)) + td = @anomalies[id] + td = link_to(t('shared.statistics', locale: :en), statistics_exercise_path(exercise)) + +== t('mailers.user_mailer.exercise_anomaly_detected.body3') diff --git a/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim b/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim new file mode 100644 index 00000000..336f3c29 --- /dev/null +++ b/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim @@ -0,0 +1 @@ +== t('mailers.user_mailer.exercise_anomaly_needs_feedback.body', receiver_displayname: @receiver_displayname, exercise: @exercise_title, link: link_to(@link, @link)) diff --git a/config/deploy.rb b/config/deploy.rb index 60edce52..1fac842e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -9,6 +9,8 @@ set :log_level, :info set :puma_threads, [0, 16] set :repo_url, 'git@github.com:openHPI/codeocean.git' +set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" } + namespace :deploy do before 'check:linked_files', 'config:push' @@ -21,3 +23,11 @@ namespace :deploy do end end end + +namespace :whenever do + task :update_crontab do + run 'bundle exec whenever --update-crontab' + end +end + +after 'deploy', 'whenever:update_crontab' diff --git a/config/locales/de.yml b/config/locales/de.yml index cb042f7d..281d349d 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -123,6 +123,8 @@ de: exercise_collections: id: "ID" name: "Name" + user: "Verantwortlicher" + use_anomaly_detection: "Abweichungen in der Arbeitszeit erkennen" updated_at: "Letzte Änderung" exercises: "Aufgaben" user_exercise_feedback: @@ -520,6 +522,64 @@ de:
This mail was automatically sent by CodeOcean.
subject: "%{author_displayname} hat einen neuen Kommentar in einer Diskussion veröffentlicht, die Sie abonniert haben." + exercise_anomaly_detected: + subject: "Unregelmäßigkeiten in Aufgaben Ihrer Aufgabensammlung" + body1: | + English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht. +
+ Die Aufgaben sind: +
+ body2: | +
+ Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy. +
+ The exercises are: +
+ body3: | +
+ If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ exercise_anomaly_needs_feedback: + body: | + English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben: +
+ %{exercise} - %{link} +
+ Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise: +
+ %{exercise} - %{link} +
+ If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ subject: "Eine Aufgabe auf CodeOcean benötigt Ihr Feedback" request_for_comments: click_here: Zum Kommentieren auf die Seitenleiste klicken! comments: Kommentare diff --git a/config/locales/en.yml b/config/locales/en.yml index 4e3564bd..c5e05183 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -123,6 +123,8 @@ en: exercise_collections: id: "ID" name: "Name" + user: "Associated User" + use_anomaly_detection: "Enable Worktime Anomaly Detection" updated_at: "Last Update" exercises: "Exercises" user_exercise_feedback: @@ -520,6 +522,64 @@ en:
This mail was automatically sent by CodeOcean.
subject: "%{author_displayname} has posted a new comment to a discussion you subscribed to on CodeOcean." + exercise_anomaly_detected: + subject: "Anomalies in exercises of your exercise collection" + body1: | + English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht. +
+ Die Aufgaben sind: +
+ body2: | +
+ Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy. +
+ The exercises are: +
+ body3: | +
+ If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ exercise_anomaly_needs_feedback: + body: | + English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben: +
+ %{exercise} - %{link} +
+ Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise: +
+ %{exercise} - %{link} +
+ If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ subject: "An exercise on CodeOcean needs your feedback" request_for_comments: click_here: Click on this sidebar to comment! comments: Comments diff --git a/config/schedule.rb b/config/schedule.rb new file mode 100644 index 00000000..f7ad2f74 --- /dev/null +++ b/config/schedule.rb @@ -0,0 +1,27 @@ +# Use this file to easily define all of your cron jobs. +# +# It's helpful, but not entirely necessary to understand cron before proceeding. +# http://en.wikipedia.org/wiki/Cron + +# Example: +# +# set :output, "/path/to/my/cron_log.log" +# +# every 2.hours do +# command "/usr/bin/some_great_command" +# runner "MyModel.some_method" +# rake "some:great:rake:task" +# end +# +# every 4.days do +# runner "AnotherModel.prune_old_records" +# end + +# Learn more: http://github.com/javan/whenever + +set :output, Whenever.path + '/log/whenever/whenever_$(date +%Y%m%d%H%M%S).log' +set :environment, ENV['RAILS_ENV'] if ENV['RAILS_ENV'] + +every 1.day, at: '3:00 am' do + rake 'detect_exercise_anomalies:with_at_least[50,50]' +end diff --git a/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb b/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb new file mode 100644 index 00000000..2aad7e28 --- /dev/null +++ b/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb @@ -0,0 +1,5 @@ +class AddAnomalyDetectionFlagToExerciseCollection < ActiveRecord::Migration + def change + add_column :exercise_collections, :use_anomaly_detection, :boolean, :default => false + end +end diff --git a/db/migrate/20171122124222_add_index_to_exercises.rb b/db/migrate/20171122124222_add_index_to_exercises.rb new file mode 100644 index 00000000..cf1f4674 --- /dev/null +++ b/db/migrate/20171122124222_add_index_to_exercises.rb @@ -0,0 +1,5 @@ +class AddIndexToExercises < ActiveRecord::Migration + def change + add_index :exercises, :id + end +end diff --git a/db/migrate/20171210172208_add_user_to_exercise_collection.rb b/db/migrate/20171210172208_add_user_to_exercise_collection.rb new file mode 100644 index 00000000..6dee1adf --- /dev/null +++ b/db/migrate/20171210172208_add_user_to_exercise_collection.rb @@ -0,0 +1,5 @@ +class AddUserToExerciseCollection < ActiveRecord::Migration + def change + add_reference :exercise_collections, :user, polymorphic: true, index: true + end +end diff --git a/db/migrate/20180226131340_create_anomaly_notifications.rb b/db/migrate/20180226131340_create_anomaly_notifications.rb new file mode 100644 index 00000000..5de3dbe4 --- /dev/null +++ b/db/migrate/20180226131340_create_anomaly_notifications.rb @@ -0,0 +1,11 @@ +class CreateAnomalyNotifications < ActiveRecord::Migration + def change + create_table :anomaly_notifications do |t| + t.belongs_to :user, polymorphic: true, index: true + t.belongs_to :exercise, index: true + t.belongs_to :exercise_collection, index: true + t.string :reason + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 0280a8c4..55339422 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,11 +11,25 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180222145909) do +ActiveRecord::Schema.define(version: 20180226131340) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "anomaly_notifications", force: :cascade do |t| + t.integer "user_id" + t.string "user_type" + t.integer "exercise_id" + t.integer "exercise_collection_id" + t.string "reason" + t.datetime "created_at" + t.datetime "updated_at" + end + + add_index "anomaly_notifications", ["exercise_collection_id"], name: "index_anomaly_notifications_on_exercise_collection_id", using: :btree + add_index "anomaly_notifications", ["exercise_id"], name: "index_anomaly_notifications_on_exercise_id", using: :btree + add_index "anomaly_notifications", ["user_type", "user_id"], name: "index_anomaly_notifications_on_user_type_and_user_id", using: :btree + create_table "code_harbor_links", force: :cascade do |t| t.string "oauth2token", limit: 255 t.datetime "created_at" @@ -104,8 +118,13 @@ ActiveRecord::Schema.define(version: 20180222145909) do t.string "name" t.datetime "created_at" t.datetime "updated_at" + t.boolean "use_anomaly_detection", default: false + t.integer "user_id" + t.string "user_type" end + add_index "exercise_collections", ["user_type", "user_id"], name: "index_exercise_collections_on_user_type_and_user_id", using: :btree + create_table "exercise_collections_exercises", id: false, force: :cascade do |t| t.integer "exercise_collection_id" t.integer "exercise_id" @@ -137,6 +156,8 @@ ActiveRecord::Schema.define(version: 20180222145909) do t.integer "expected_difficulty", default: 1 end + add_index "exercises", ["id"], name: "index_exercises_on_id", using: :btree + create_table "exercises_proxy_exercises", id: false, force: :cascade do |t| t.integer "proxy_exercise_id" t.integer "exercise_id" diff --git a/lib/tasks/detect_exercise_anomalies.rake b/lib/tasks/detect_exercise_anomalies.rake new file mode 100644 index 00000000..e22550b2 --- /dev/null +++ b/lib/tasks/detect_exercise_anomalies.rake @@ -0,0 +1,165 @@ +include Rails.application.routes.url_helpers + +namespace :detect_exercise_anomalies do + # uncomment for debug logging: + # logger = Logger.new(STDOUT) + # logger.level = Logger::DEBUG + # Rails.logger = logger + + # These factors determine if an exercise is an anomaly, given the average working time (avg): + # (avg * MIN_TIME_FACTOR) <= working_time <= (avg * MAX_TIME_FACTOR) + MIN_TIME_FACTOR = 0.1 + MAX_TIME_FACTOR = 2 + + # Determines how many users are picked from the best/average/worst performers of each anomaly for feedback + NUMBER_OF_USERS_PER_CLASS = 10 + + # Determines margin below which user working times will be considered data errors (e.g. copy/paste solutions) + MIN_USER_WORKING_TIME = 0.0 + + # Cache exercise working times, because queries are expensive and values do not change between collections + WORKING_TIME_CACHE = {} + AVERAGE_WORKING_TIME_CACHE = {} + + task :with_at_least, [:number_of_exercises, :number_of_solutions] => :environment do |task, args| + number_of_exercises = args[:number_of_exercises] + number_of_solutions = args[:number_of_solutions] + + puts "Searching for exercise collections with at least #{number_of_exercises} exercises and #{number_of_solutions} users." + # Get all exercise collections that have at least the specified amount of exercises and at least the specified + # number of submissions AND are flagged for anomaly detection + collections = get_collections(number_of_exercises, number_of_solutions) + puts "Found #{collections.length}." + + collections.each do |collection| + puts "\t- #{collection}" + anomalies = find_anomalies(collection) + + if anomalies.length > 0 and not collection.user.nil? + notify_collection_author(collection, anomalies) + notify_users(collection, anomalies) + reset_anomaly_detection_flag(collection) + end + end + puts 'Done.' + end + + def get_collections(number_of_exercises, number_of_solutions) + ExerciseCollection + .where(:use_anomaly_detection => true) + .joins("join exercise_collections_exercises ece on exercise_collections.id = ece.exercise_collection_id + join + (select e.id + from exercises e + join submissions s on s.exercise_id = e.id + group by e.id + having count(s.user_id) > #{ExerciseCollection.sanitize(number_of_solutions)} + ) as exercises_with_submissions on exercises_with_submissions.id = ece.exercise_id") + .group('exercise_collections.id') + .having('count(exercises_with_submissions.id) > ?', number_of_exercises) + end + + def find_anomalies(collection) + working_times = {} + collection.exercises.each do |exercise| + puts "\t\t> #{exercise.title}" + working_times[exercise.id] = get_average_working_time(exercise) + end + average = working_times.values.reduce(:+) / working_times.size + working_times.select do |exercise_id, working_time| + working_time > average * MAX_TIME_FACTOR or working_time < average * MIN_TIME_FACTOR + end + end + + def time_to_f(timestamp) + unless timestamp.nil? + timestamp = timestamp.split(':') + return timestamp[0].to_i * 60 * 60 + timestamp[1].to_i * 60 + timestamp[2].to_f + end + nil + end + + def get_average_working_time(exercise) + unless AVERAGE_WORKING_TIME_CACHE.key?(exercise.id) + seconds = time_to_f exercise.average_working_time + AVERAGE_WORKING_TIME_CACHE[exercise.id] = seconds + end + AVERAGE_WORKING_TIME_CACHE[exercise.id] + end + + def get_user_working_times(exercise) + unless WORKING_TIME_CACHE.key?(exercise.id) + exercise.retrieve_working_time_statistics + WORKING_TIME_CACHE[exercise.id] = exercise.working_time_statistics + end + WORKING_TIME_CACHE[exercise.id] + end + + def notify_collection_author(collection, anomalies) + puts "\t\tSending E-Mail to author (#{collection.user.displayname} <#{collection.user.email}>)..." + UserMailer.exercise_anomaly_detected(collection, anomalies).deliver_now + end + + def notify_users(collection, anomalies) + by_id_and_type = proc { |u| {user_id: u[:user_id], user_type: u[:user_type]} } + + puts "\t\tSending E-Mails to best and worst performing users of each anomaly..." + anomalies.each do |exercise_id, average_working_time| + puts "\t\tAnomaly in exercise #{exercise_id} (avg: #{average_working_time} seconds):" + exercise = Exercise.find(exercise_id) + users_to_notify = [] + + users = {} + [:performers_by_time, :performers_by_score].each do |method| + # merge users found by multiple methods returning a hash {best: [], worst: []} + users = users.merge(send(method, exercise, NUMBER_OF_USERS_PER_CLASS)) {|key, this, other| this + other} + end + + # write reasons for feedback emails to db + users.keys.each do |key| + segment = users[key].uniq &by_id_and_type + users_to_notify += segment + segment.each do |user| + reason = "{\"segment\": \"#{key.to_s}\", \"feature\": \"#{user[:reason]}\", value: \"#{user[:value]}\"}" + AnomalyNotification.create(user_id: user[:user_id], user_type: user[:user_type], + exercise: exercise, exercise_collection: collection, reason: reason) + end + end + + users_to_notify.uniq! &by_id_and_type + users_to_notify.each do |u| + user = u[:user_type] == InternalUser.name ? InternalUser.find(u[:user_id]) : ExternalUser.find(u[:user_id]) + host = CodeOcean::Application.config.action_mailer.default_url_options[:host] + feedback_link = url_for(action: :new, controller: :user_exercise_feedbacks, exercise_id: exercise.id, host: host) + UserMailer.exercise_anomaly_needs_feedback(user, exercise, feedback_link).deliver + end + puts "\t\tAsked #{users_to_notify.size} users for feedback." + end + end + + def performers_by_score(exercise, n) + submissions = exercise.last_submission_per_user.where('score is not null').order(score: :desc) + map_block = proc {|item| {user_id: item.user_id, user_type: item.user_type, value: item.score, reason: 'score'}} + best_performers = submissions.first(n).to_a.map &map_block + worst_performers = submissions.last(n).to_a.map &map_block + return {:best => best_performers, :worst => worst_performers} + end + + def performers_by_time(exercise, n) + working_times = get_user_working_times(exercise).values.map do |item| + {user_id: item['user_id'], user_type: item['user_type'], score: item['score'].to_f, + value: time_to_f(item['working_time']), reason: 'time'} + end + avg_score = exercise.average_score + working_times.reject! {|item| item[:value].nil? or item[:value] <= MIN_USER_WORKING_TIME or item[:score] < avg_score} + working_times.sort_by! {|item| item[:value]} + return {:best => working_times.first(n), :worst => working_times.last(n)} + end + + def reset_anomaly_detection_flag(collection) + puts "\t\tResetting flag..." + collection.use_anomaly_detection = false + collection.save! + end + +end diff --git a/log/whenever/.gitignore b/log/whenever/.gitignore new file mode 100644 index 00000000..397b4a76 --- /dev/null +++ b/log/whenever/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/provision.sh b/provision.sh index 60c9c04b..a6128d1d 100644 --- a/provision.sh +++ b/provision.sh @@ -2,71 +2,87 @@ # rvm/rails installation from https://gorails.com/setup/ubuntu/14.04 # passenger installation from https://www.phusionpassenger.com/library/install/nginx/install/oss/trusty/ +######## VERSION INFORMATION ######## + +postgres_version=10 +ruby_version=2.3.6 +rails_version=4.2.10 + +########## INSTALL SCRIPT ########### + +# PostgreSQL +sudo add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + # passenger -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7 -apt-get install -y apt-transport-https ca-certificates -sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list' +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7 +sudo apt-get -qq -y install apt-transport-https ca-certificates +sudo sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list' # rails -add-apt-repository ppa:chris-lea/node.js +sudo add-apt-repository -y ppa:chris-lea/node.js -apt-get update +sudo apt-get -qq update # code_ocean -apt-get install -y postgresql-client postgresql-10 postgresql-server-dev-10 vagrant +sudo apt-get -qq -y install postgresql-client postgresql-$postgres_version postgresql-server-dev-$postgres_version vagrant # Docker if [ ! -f /etc/default/docker ] then - curl -sSL https://get.docker.com/ | sh + curl -sSL https://get.docker.com/ | sudo sh fi if ! grep code_ocean /etc/default/docker then - cat >>/etc/default/docker </etc/postgresql/10/main/pg_hba.conf < /etc/nginx/sites-available/code_ocean < 879.325828, 51 => 924.870057, 31 => 1031.21233, 69 => 2159.182116} + UserMailer.exercise_anomaly_detected(collection, anomalies) + end +end