Merge pull request #144 from openHPI/exercise-anomaly-detection

Exercise anomaly detection

.gitignore

@@ -10,7 +10,7 @@
 /config/*.staging-epic.yml
 /config/deploy/staging-epic.rb
 /coverage
-/log
+/log/*.*
 /public/assets
 /public/uploads
 /rubocop.html

Capfile

@@ -6,6 +6,7 @@ require 'capistrano/puma/nginx'
 require 'capistrano/rails'
 require 'capistrano/rvm'
 require 'capistrano/upload-config'
+require 'whenever/capistrano'
 
 install_plugin Capistrano::SCM::Git
 install_plugin Capistrano::Puma

Gemfile

@@ -43,6 +43,7 @@ gem 'nokogiri'
 gem 'd3-rails'
 gem 'rest-client'
 gem 'rubyzip'
+gem 'whenever', require: false
 
 group :development, :staging do
   gem 'better_errors', platform: :ruby

Gemfile.lock

@@ -32,11 +32,6 @@ GEM
       activesupport (= 4.2.10)
       arel (~> 6.0)
     activerecord-deprecated_finders (1.0.4)
-    activerecord-jdbc-adapter (50.0)
-      activerecord (>= 2.2)
-    activerecord-jdbcpostgresql-adapter (50.0)
-      activerecord-jdbc-adapter (~> 50.0)
-      jdbc-postgres (>= 9.4, < 43)
     activesupport (4.2.10)
       i18n (~> 0.7)
       minitest (~> 5.1)
@@ -51,7 +46,6 @@ GEM
     autotest-rails (4.2.1)
       ZenTest (~> 4.5)
     bcrypt (3.1.11)
-    bcrypt (3.1.11-java)
     better_errors (2.4.0)
       coderay (>= 1.0.0)
       erubi (>= 1.0.0)
@@ -97,6 +91,7 @@ GEM
       mime-types (>= 1.16)
     childprocess (0.8.0)
       ffi (~> 1.0, >= 1.0.11)
+    chronic (0.10.2)
     codeclimate-test-reporter (1.0.7)
       simplecov
     coderay (1.1.2)
@@ -108,25 +103,23 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.0.5)
-    concurrent-ruby (1.0.5-java)
     concurrent-ruby-ext (1.0.5)
       concurrent-ruby (= 1.0.5)
     crass (1.0.3)
-    d3-rails (4.10.2)
+    d3-rails (4.13.0)
       railties (>= 3.1)
     database_cleaner (1.6.2)
     debug_inspector (0.0.3)
     diff-lcs (1.3)
     docile (1.1.5)
-    docker-api (1.34.0)
+    docker-api (1.34.1)
       excon (>= 0.47.0)
       multi_json
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
-    erubi (1.7.0)
+    erubi (1.7.1)
     erubis (2.7.0)
     eventmachine (1.0.9.1)
-    eventmachine (1.0.9.1-java)
     excon (0.60.0)
     execjs (2.7.0)
     factory_bot (4.8.2)
@@ -139,8 +132,7 @@ GEM
     faye-websocket (0.10.7)
       eventmachine (>= 0.12.0)
       websocket-driver (>= 0.5.1)
-    ffi (1.9.21)
+    ffi (1.9.23)
-    ffi (1.9.21-java)
     forgery (0.7.0)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
@@ -155,7 +147,6 @@ GEM
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
-    jdbc-postgres (42.1.4)
     jquery-rails (4.3.1)
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
@@ -164,7 +155,6 @@ GEM
       railties (>= 3.1.0)
       turbolinks
     json (2.1.0)
-    json (2.1.0-java)
     jwt (1.5.6)
     kramdown (1.16.2)
     loofah (2.2.0)
@@ -189,7 +179,6 @@ GEM
     newrelic_rpm (4.8.0.341)
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
-    nokogiri (1.8.2-java)
     nyan-cat-formatter (0.12.0)
       rspec (>= 2.99, >= 2.14.2, < 4)
     oauth (0.4.7)
@@ -202,7 +191,7 @@ GEM
     pagedown-rails (1.1.4)
       railties (> 3.1)
     parallel (1.12.1)
-    parser (2.5.0.2)
+    parser (2.5.0.3)
       ast (~> 2.4.0)
     pg (0.21.0)
     polyamorous (1.3.3)
@@ -211,19 +200,14 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    pry (0.11.3-java)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-      spoon (~> 0.0)
     pry-byebug (3.6.0)
       byebug (~> 10.0)
       pry (~> 0.10)
     public_suffix (3.0.2)
-    puma (3.11.2)
+    puma (3.11.3)
-    puma (3.11.2-java)
     pundit (1.1.0)
       activesupport (>= 3.0.0)
-    rack (1.6.8)
+    rack (1.6.9)
     rack-mini-profiler (0.10.7)
       rack (>= 1.2.0)
     rack-test (0.6.3)
@@ -263,7 +247,7 @@ GEM
       activesupport (>= 3.0)
       i18n
       polyamorous (~> 1.3.2)
-    rb-fsevent (0.10.2)
+    rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
     rdoc (6.0.1)
@@ -294,15 +278,15 @@ GEM
       rspec-mocks (~> 3.7.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rubocop (0.52.1)
+    rubocop (0.53.0)
       parallel (~> 1.10)
-      parser (>= 2.4.0.2, < 3.0)
+      parser (>= 2.5)
       powerpack (~> 0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    rubocop-rspec (1.22.2)
+    rubocop-rspec (1.24.0)
-      rubocop (>= 0.52.1)
+      rubocop (>= 0.53.0)
     ruby-progressbar (1.9.0)
     rubytree (1.0.0)
       json (~> 2.1)
@@ -321,7 +305,7 @@ GEM
       tilt (>= 1.1, < 3)
     sdoc (1.0.0)
       rdoc (>= 5.0)
-    selenium-webdriver (3.9.0)
+    selenium-webdriver (3.10.0)
       childprocess (~> 0.5)
       rubyzip (~> 1.2)
     simplecov (0.15.1)
@@ -336,8 +320,6 @@ GEM
       bcrypt (~> 3.1)
       oauth (~> 0.4, >= 0.4.4)
       oauth2 (~> 1.0, >= 0.8.0)
-    spoon (0.0.6)
-      ffi
     spring (2.0.2)
       activesupport (>= 4.2)
     sprockets (3.7.1)
@@ -354,7 +336,6 @@ GEM
     temple (0.8.0)
     thor (0.20.0)
     thread_safe (0.3.6)
-    thread_safe (0.3.6-java)
     tilt (2.0.8)
     tubesock (0.2.7)
       rack (>= 1.5.0)
@@ -367,7 +348,6 @@ GEM
       execjs (>= 0.3.0, < 3)
     unf (0.1.4)
       unf_ext
-    unf (0.1.4-java)
     unf_ext (0.0.7.5)
     unicode-display_width (1.3.0)
     web-console (3.3.0)
@@ -377,15 +357,14 @@ GEM
     websocket (1.2.5)
     websocket-driver (0.7.0)
       websocket-extensions (>= 0.1.0)
-    websocket-driver (0.7.0-java)
-      websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.3)
+    whenever (0.10.0)
+      chronic (>= 0.6.3)
     will_paginate (3.1.6)
     xpath (3.0.0)
       nokogiri (~> 1.8)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES
@@ -454,6 +433,7 @@ DEPENDENCIES
   turbolinks (< 5.0.0)
   uglifier
   web-console
+  whenever
   will_paginate
 
 BUNDLED WITH

app/controllers/exercise_collections_controller.rb

@@ -9,6 +9,7 @@ class ExerciseCollectionsController < ApplicationController
   end
 
   def show
+    @exercises = @exercise_collection.exercises.paginate(:page => params[:page])
   end
 
   def new
@@ -46,6 +47,6 @@ class ExerciseCollectionsController < ApplicationController
   end
 
   def exercise_collection_params
-    params[:exercise_collection].permit(:name, :exercise_ids => [])
+    params[:exercise_collection].permit(:name, :use_anomaly_detection, :user_id, :user_type, :exercise_ids => []).merge(user_type: InternalUser.name)
   end
 end

app/controllers/user_exercise_feedbacks_controller.rb

@@ -69,7 +69,8 @@ class UserExerciseFeedbacksController < ApplicationController
     @texts = comment_presets.to_a
     @times = time_presets.to_a
     @uef = UserExerciseFeedback.new
-    @exercise = Exercise.find(params[:user_exercise_feedback][:exercise_id])
+    exercise_id = if params[:user_exercise_feedback].nil? then params[:exercise_id] else params[:user_exercise_feedback][:exercise_id] end
+    @exercise = Exercise.find(exercise_id)
     authorize!
   end
 

app/mailers/user_mailer.rb

@@ -37,4 +37,18 @@ class UserMailer < ActionMailer::Base
     @rfc_link = request_for_comment_url(request_for_comments)
     mail(subject: t('mailers.user_mailer.send_thank_you_note.subject', author: @author), to: receiver.email)
   end
+  
+  def exercise_anomaly_detected(exercise_collection, anomalies)
+    @receiver_displayname = exercise_collection.user.displayname
+    @collection = exercise_collection
+    @anomalies = anomalies
+    mail(subject: t('mailers.user_mailer.exercise_anomaly_detected.subject'), to: exercise_collection.user.email)
+  end
+
+  def exercise_anomaly_needs_feedback(user, exercise, link)
+    @receiver_displayname = user.displayname
+    @exercise_title = exercise.title
+    @link = link
+    mail(subject: t('mailers.user_mailer.exercise_anomaly_needs_feedback.subject'), to: user.email)
+  end
 end

app/models/anomaly_notification.rb

@@ -0,0 +1,5 @@
+class AnomalyNotification < ActiveRecord::Base
+  belongs_to :user, polymorphic: true
+  belongs_to :exercise
+  belongs_to :exercise_collection
+end

app/models/exercise.rb

@@ -36,6 +36,7 @@ class Exercise < ActiveRecord::Base
   validates :token, presence: true, uniqueness: true
 
   @working_time_statistics = nil
+  attr_reader :working_time_statistics
 
   MAX_EXERCISE_FEEDBACKS = 20
 
@@ -65,21 +66,27 @@ class Exercise < ActiveRecord::Base
   end
 
   def user_working_time_query
-    """
+    "
       SELECT user_id,
-             sum(working_time_new) AS working_time
+             user_type,
+             SUM(working_time_new) AS working_time,
+             MAX(score) AS score
       FROM
         (SELECT user_id,
+                user_type,
+                score,
                 CASE WHEN working_time >= '0:05:00' THEN '0' ELSE working_time END AS working_time_new
          FROM
             (SELECT user_id,
+                    user_type,
+                    score,
                     id,
                     (created_at - lag(created_at) over (PARTITION BY user_id, exercise_id
                                                         ORDER BY created_at)) AS working_time
             FROM submissions
             WHERE exercise_id=#{id}) AS foo) AS bar
-      GROUP BY user_id
+      GROUP BY user_id, user_type
-    """
+    "
   end
 
   def get_quantiles(quantiles)
@@ -202,7 +209,7 @@ class Exercise < ActiveRecord::Base
   def retrieve_working_time_statistics
     @working_time_statistics = {}
     self.class.connection.execute(user_working_time_query).each do |tuple|
-      @working_time_statistics[tuple["user_id"].to_i] = tuple
+      @working_time_statistics[tuple['user_id'].to_i] = tuple
     end
   end
 
@@ -368,4 +375,15 @@ class Exercise < ActiveRecord::Base
     user_exercise_feedbacks.size <= MAX_EXERCISE_FEEDBACKS
   end
 
+  def last_submission_per_user
+    Submission.joins("JOIN (
+          SELECT
+              user_id,
+              user_type,
+              first_value(id) OVER (PARTITION BY user_id ORDER BY created_at DESC) AS fv
+          FROM submissions
+          WHERE exercise_id = #{id}
+        ) AS t ON t.fv = submissions.id").distinct
+  end
+
 end

app/models/exercise_collection.rb

@@ -1,6 +1,7 @@
 class ExerciseCollection < ActiveRecord::Base
 
   has_and_belongs_to_many :exercises
+  belongs_to :user, polymorphic: true
 
   def to_s
     "#{I18n.t('activerecord.models.exercise_collection.one')}: #{name} (#{id})"

app/views/exercise_collections/_form.html.slim

@@ -1,11 +1,18 @@
 - exercises = Exercise.order(:title)
+- users = InternalUser.order(:name)
 
-= form_for(@exercise_collection, data: {exercises: exercises}, multipart: true) do |f|
+= form_for(@exercise_collection, data: {exercises: exercises, users: users}, multipart: true) do |f|
   = render('shared/form_errors', object: @exercise_collection)
   .form-group
-    = f.label(:name)
+    = f.label(t('activerecord.attributes.exercise_collections.name'))
     = f.text_field(:name, class: 'form-control', required: true)
   .form-group
-    = f.label(:exercises)
+    = f.label(t('activerecord.attributes.exercise_collections.use_anomaly_detection'))
+    = f.check_box(:use_anomaly_detection, {class: 'form-control'})
+  .form-group
+    = f.label(t('activerecord.attributes.exercise_collections.user'))
+    = f.collection_select(:user_id, users, :id, :name, {}, {class: 'form-control'})
+  .form-group
+    = f.label(t('activerecord.attributes.exercise_collections.exercises'))
     = f.collection_select(:exercise_ids, exercises, :id, :title, {}, {class: 'form-control', multiple: true})
   .actions = render('shared/submit_button', f: f, object: @exercise_collection)

app/views/exercise_collections/show.html.slim

@@ -3,9 +3,25 @@ h1
   = render('shared/edit_button', object: @exercise_collection)
 
 = row(label: 'exercise_collections.name', value: @exercise_collection.name)
+= row(label: 'exercise_collections.user', value: link_to(@exercise_collection.user.name, @exercise_collection.user)) unless @exercise_collection.user.nil?
+= row(label: 'exercise_collections.use_anomaly_detection', value: @exercise_collection.use_anomaly_detection)
 = row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at)
 
 h4 = t('activerecord.attributes.exercise_collections.exercises')
-ul.list-unstyled
+.table-responsive
-  - @exercise_collection.exercises.sort_by{|c| c.title}.each do |exercise|
+  table.table
-    li = link_to(exercise, exercise)
+    thead
+      tr
+        th = t('activerecord.attributes.exercise.title')
+        th = t('activerecord.attributes.exercise.execution_environment')
+        th = t('activerecord.attributes.exercise.user')
+        th = t('shared.actions')
+    tbody
+      - @exercises.sort_by{|c| c.title}.each do |exercise|
+        tr
+          td = link_to(exercise.title, exercise)
+          td = link_to_if(exercise.execution_environment && policy(exercise.execution_environment).show?, exercise.execution_environment, exercise.execution_environment)
+          td = exercise.user.name
+          td = link_to(t('shared.statistics'), statistics_exercise_path(exercise))
+        
+= render('shared/pagination', collection: @exercises)

app/views/user_mailer/exercise_anomaly_detected.html.slim

@@ -0,0 +1,38 @@
+== t('mailers.user_mailer.exercise_anomaly_detected.body1',
+        receiver_displayname: @receiver_displayname,
+        collection_name: @collection.name)
+
+table(border=1)
+  thead
+    tr
+      td = t('activerecord.attributes.exercise.title', locale: :de)
+      td = t('exercises.statistics.average_worktime', locale: :de)
+      td = t('shared.actions', locale: :de)
+  tbody
+    - @anomalies.keys.each do | id |
+      - exercise = Exercise.find(id)
+      tr
+        td = link_to(exercise.title, exercise_path(exercise))
+        td = @anomalies[id]
+        td = link_to(t('shared.statistics', locale: :de), statistics_exercise_path(exercise))
+
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body2',
+        receiver_displayname: @receiver_displayname,
+        collection_name: @collection.name)
+
+table(border=1)
+  thead
+    tr
+      td = t('activerecord.attributes.exercise.title', locale: :en)
+      td = t('exercises.statistics.average_worktime', locale: :en)
+      td = t('shared.actions', locale: :en)
+  tbody
+    - @anomalies.keys.each do | id |
+      - exercise = Exercise.find(id)
+      tr
+        td = link_to(exercise.title, exercise_path(exercise))
+        td = @anomalies[id]
+        td = link_to(t('shared.statistics', locale: :en), statistics_exercise_path(exercise))
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body3')

app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim

@@ -0,0 +1 @@
+== t('mailers.user_mailer.exercise_anomaly_needs_feedback.body', receiver_displayname: @receiver_displayname, exercise: @exercise_title, link: link_to(@link, @link))

config/deploy.rb

@@ -9,6 +9,8 @@ set :log_level, :info
 set :puma_threads, [0, 16]
 set :repo_url, 'git@github.com:openHPI/codeocean.git'
 
+set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" }
+
 namespace :deploy do
   before 'check:linked_files', 'config:push'
 
@@ -21,3 +23,11 @@ namespace :deploy do
     end
   end
 end
+
+namespace :whenever do
+  task :update_crontab do
+    run 'bundle exec whenever --update-crontab'
+  end
+end
+
+after 'deploy', 'whenever:update_crontab'

config/locales/de.yml

@@ -123,6 +123,8 @@ de:
       exercise_collections:
         id: "ID"
         name: "Name"
+        user: "Verantwortlicher"
+        use_anomaly_detection: "Abweichungen in der Arbeitszeit erkennen"
         updated_at: "Letzte Änderung"
         exercises: "Aufgaben"
       user_exercise_feedback:
@@ -518,6 +520,64 @@ de:
           <br>
           This mail was automatically sent by CodeOcean. <br>
         subject: "%{author_displayname} hat einen neuen Kommentar in einer Diskussion veröffentlicht, die Sie abonniert haben."
+      exercise_anomaly_detected:
+        subject: "Unregelmäßigkeiten in Aufgaben Ihrer Aufgabensammlung"
+        body1: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+          <br>
+          Die Aufgaben sind:
+          <br>
+        body2: |
+          <br>
+          Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+          <br>
+          The exercises are:
+          <br>
+        body3: |
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+      exercise_anomaly_needs_feedback:
+        body: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+        subject: "Eine Aufgabe auf CodeOcean benötigt Ihr Feedback"
   request_for_comments:
     click_here: Zum Kommentieren auf die Seitenleiste klicken!
     comments: Kommentare

config/locales/en.yml

@@ -123,6 +123,8 @@ en:
       exercise_collections:
         id: "ID"
         name: "Name"
+        user: "Associated User"
+        use_anomaly_detection: "Enable Worktime Anomaly Detection"
         updated_at: "Last Update"
         exercises: "Exercises"
       user_exercise_feedback:
@@ -518,6 +520,64 @@ en:
           <br>
           This mail was automatically sent by CodeOcean. <br>
         subject: "%{author_displayname} has posted a new comment to a discussion you subscribed to on CodeOcean."
+      exercise_anomaly_detected:
+        subject: "Anomalies in exercises of your exercise collection"
+        body1: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+          <br>
+          Die Aufgaben sind:
+          <br>
+        body2: |
+          <br>
+          Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+          <br>
+          The exercises are:
+          <br>
+        body3: |
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+      exercise_anomaly_needs_feedback:
+        body: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+        subject: "An exercise on CodeOcean needs your feedback"
   request_for_comments:
     click_here: Click on this sidebar to comment!
     comments: Comments

config/schedule.rb

@@ -0,0 +1,27 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+set :output, Whenever.path + '/log/whenever/whenever_$(date +%Y%m%d%H%M%S).log'
+set :environment, ENV['RAILS_ENV'] if ENV['RAILS_ENV']
+
+every 1.day, at: '3:00 am' do
+  rake 'detect_exercise_anomalies:with_at_least[50,50]'
+end

db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb

@@ -0,0 +1,5 @@
+class AddAnomalyDetectionFlagToExerciseCollection < ActiveRecord::Migration
+  def change
+    add_column :exercise_collections, :use_anomaly_detection, :boolean, :default => false
+  end
+end

db/migrate/20171122124222_add_index_to_exercises.rb

@@ -0,0 +1,5 @@
+class AddIndexToExercises < ActiveRecord::Migration
+  def change
+    add_index :exercises, :id
+  end
+end

db/migrate/20171210172208_add_user_to_exercise_collection.rb

@@ -0,0 +1,5 @@
+class AddUserToExerciseCollection < ActiveRecord::Migration
+  def change
+    add_reference :exercise_collections, :user,  polymorphic: true, index: true
+  end
+end

db/migrate/20180226131340_create_anomaly_notifications.rb

@@ -0,0 +1,11 @@
+class CreateAnomalyNotifications < ActiveRecord::Migration
+  def change
+    create_table :anomaly_notifications do |t|
+      t.belongs_to :user, polymorphic: true, index: true
+      t.belongs_to :exercise, index: true
+      t.belongs_to :exercise_collection, index: true
+      t.string :reason
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -11,11 +11,25 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180222145909) do
+ActiveRecord::Schema.define(version: 20180226131340) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "anomaly_notifications", force: :cascade do |t|
+    t.integer  "user_id"
+    t.string   "user_type"
+    t.integer  "exercise_id"
+    t.integer  "exercise_collection_id"
+    t.string   "reason"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "anomaly_notifications", ["exercise_collection_id"], name: "index_anomaly_notifications_on_exercise_collection_id", using: :btree
+  add_index "anomaly_notifications", ["exercise_id"], name: "index_anomaly_notifications_on_exercise_id", using: :btree
+  add_index "anomaly_notifications", ["user_type", "user_id"], name: "index_anomaly_notifications_on_user_type_and_user_id", using: :btree
+
   create_table "code_harbor_links", force: :cascade do |t|
     t.string   "oauth2token", limit: 255
     t.datetime "created_at"
@@ -104,8 +118,13 @@ ActiveRecord::Schema.define(version: 20180222145909) do
     t.string   "name"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.boolean  "use_anomaly_detection", default: false
+    t.integer  "user_id"
+    t.string   "user_type"
   end
 
+  add_index "exercise_collections", ["user_type", "user_id"], name: "index_exercise_collections_on_user_type_and_user_id", using: :btree
+
   create_table "exercise_collections_exercises", id: false, force: :cascade do |t|
     t.integer "exercise_collection_id"
     t.integer "exercise_id"
@@ -137,6 +156,8 @@ ActiveRecord::Schema.define(version: 20180222145909) do
     t.integer  "expected_difficulty",                  default: 1
   end
 
+  add_index "exercises", ["id"], name: "index_exercises_on_id", using: :btree
+
   create_table "exercises_proxy_exercises", id: false, force: :cascade do |t|
     t.integer  "proxy_exercise_id"
     t.integer  "exercise_id"

lib/tasks/detect_exercise_anomalies.rake

@@ -0,0 +1,165 @@
+include Rails.application.routes.url_helpers
+
+namespace :detect_exercise_anomalies do
+  # uncomment for debug logging:
+  # logger           = Logger.new(STDOUT)
+  # logger.level     = Logger::DEBUG
+  # Rails.logger     = logger
+
+  # These factors determine if an exercise is an anomaly, given the average working time (avg):
+  # (avg * MIN_TIME_FACTOR) <= working_time <= (avg * MAX_TIME_FACTOR)
+  MIN_TIME_FACTOR = 0.1
+  MAX_TIME_FACTOR = 2
+
+  # Determines how many users are picked from the best/average/worst performers of each anomaly for feedback
+  NUMBER_OF_USERS_PER_CLASS = 10
+
+  # Determines margin below which user working times will be considered data errors (e.g. copy/paste solutions)
+  MIN_USER_WORKING_TIME = 0.0
+
+  # Cache exercise working times, because queries are expensive and values do not change between collections
+  WORKING_TIME_CACHE = {}
+  AVERAGE_WORKING_TIME_CACHE = {}
+
+  task :with_at_least, [:number_of_exercises, :number_of_solutions] => :environment do |task, args|
+    number_of_exercises = args[:number_of_exercises]
+    number_of_solutions = args[:number_of_solutions]
+
+    puts "Searching for exercise collections with at least #{number_of_exercises} exercises and #{number_of_solutions} users."
+    # Get all exercise collections that have at least the specified amount of exercises and at least the specified
+    # number of submissions AND are flagged for anomaly detection
+    collections = get_collections(number_of_exercises, number_of_solutions)
+    puts "Found #{collections.length}."
+
+    collections.each do |collection|
+      puts "\t- #{collection}"
+      anomalies = find_anomalies(collection)
+
+      if anomalies.length > 0 and not collection.user.nil?
+        notify_collection_author(collection, anomalies)
+        notify_users(collection, anomalies)
+        reset_anomaly_detection_flag(collection)
+      end
+    end
+    puts 'Done.'
+  end
+
+  def get_collections(number_of_exercises, number_of_solutions)
+    ExerciseCollection
+      .where(:use_anomaly_detection => true)
+      .joins("join exercise_collections_exercises ece on exercise_collections.id = ece.exercise_collection_id
+                            join
+                              (select e.id
+                               from exercises e
+                                 join submissions s on s.exercise_id = e.id
+                               group by e.id
+                               having count(s.user_id) > #{ExerciseCollection.sanitize(number_of_solutions)}
+                              ) as exercises_with_submissions on exercises_with_submissions.id = ece.exercise_id")
+      .group('exercise_collections.id')
+      .having('count(exercises_with_submissions.id) > ?', number_of_exercises)
+  end
+
+  def find_anomalies(collection)
+    working_times = {}
+    collection.exercises.each do |exercise|
+      puts "\t\t> #{exercise.title}"
+      working_times[exercise.id] = get_average_working_time(exercise)
+    end
+    average = working_times.values.reduce(:+) / working_times.size
+    working_times.select do |exercise_id, working_time|
+      working_time > average * MAX_TIME_FACTOR or working_time < average * MIN_TIME_FACTOR
+    end
+  end
+
+  def time_to_f(timestamp)
+    unless timestamp.nil?
+      timestamp = timestamp.split(':')
+      return timestamp[0].to_i * 60 * 60 + timestamp[1].to_i * 60 + timestamp[2].to_f
+    end
+    nil
+  end
+
+  def get_average_working_time(exercise)
+    unless AVERAGE_WORKING_TIME_CACHE.key?(exercise.id)
+      seconds = time_to_f exercise.average_working_time
+      AVERAGE_WORKING_TIME_CACHE[exercise.id] = seconds
+    end
+    AVERAGE_WORKING_TIME_CACHE[exercise.id]
+  end
+
+  def get_user_working_times(exercise)
+    unless WORKING_TIME_CACHE.key?(exercise.id)
+      exercise.retrieve_working_time_statistics
+      WORKING_TIME_CACHE[exercise.id] = exercise.working_time_statistics
+    end
+    WORKING_TIME_CACHE[exercise.id]
+  end
+
+  def notify_collection_author(collection, anomalies)
+    puts "\t\tSending E-Mail to author (#{collection.user.displayname} <#{collection.user.email}>)..."
+    UserMailer.exercise_anomaly_detected(collection, anomalies).deliver_now
+  end
+
+  def notify_users(collection, anomalies)
+    by_id_and_type = proc { |u| {user_id: u[:user_id], user_type: u[:user_type]} }
+
+    puts "\t\tSending E-Mails to best and worst performing users of each anomaly..."
+    anomalies.each do |exercise_id, average_working_time|
+      puts "\t\tAnomaly in exercise #{exercise_id} (avg: #{average_working_time} seconds):"
+      exercise = Exercise.find(exercise_id)
+      users_to_notify = []
+
+      users = {}
+      [:performers_by_time, :performers_by_score].each do |method|
+        # merge users found by multiple methods returning a hash {best: [], worst: []}
+        users = users.merge(send(method, exercise, NUMBER_OF_USERS_PER_CLASS)) {|key, this, other| this + other}
+      end
+
+      # write reasons for feedback emails to db
+      users.keys.each do |key|
+        segment = users[key].uniq &by_id_and_type
+        users_to_notify += segment
+        segment.each do |user|
+          reason = "{\"segment\": \"#{key.to_s}\", \"feature\": \"#{user[:reason]}\", value: \"#{user[:value]}\"}"
+          AnomalyNotification.create(user_id: user[:user_id], user_type: user[:user_type],
+                                     exercise: exercise, exercise_collection: collection, reason: reason)
+        end
+      end
+
+      users_to_notify.uniq! &by_id_and_type
+      users_to_notify.each do |u|
+        user = u[:user_type] == InternalUser.name ? InternalUser.find(u[:user_id]) : ExternalUser.find(u[:user_id])
+        host = CodeOcean::Application.config.action_mailer.default_url_options[:host]
+        feedback_link = url_for(action: :new, controller: :user_exercise_feedbacks, exercise_id: exercise.id, host: host)
+        UserMailer.exercise_anomaly_needs_feedback(user, exercise, feedback_link).deliver
+      end
+      puts "\t\tAsked #{users_to_notify.size} users for feedback."
+    end
+  end
+
+  def performers_by_score(exercise, n)
+    submissions = exercise.last_submission_per_user.where('score is not null').order(score: :desc)
+    map_block = proc {|item| {user_id: item.user_id, user_type: item.user_type, value: item.score, reason: 'score'}}
+    best_performers = submissions.first(n).to_a.map &map_block
+    worst_performers = submissions.last(n).to_a.map &map_block
+    return {:best => best_performers, :worst => worst_performers}
+  end
+
+  def performers_by_time(exercise, n)
+    working_times = get_user_working_times(exercise).values.map do |item|
+      {user_id: item['user_id'], user_type: item['user_type'], score: item['score'].to_f,
+       value: time_to_f(item['working_time']), reason: 'time'}
+    end
+    avg_score = exercise.average_score
+    working_times.reject! {|item| item[:value].nil? or item[:value] <= MIN_USER_WORKING_TIME or item[:score] < avg_score}
+    working_times.sort_by! {|item| item[:value]}
+    return {:best => working_times.first(n), :worst => working_times.last(n)}
+  end
+
+  def reset_anomaly_detection_flag(collection)
+    puts "\t\tResetting flag..."
+    collection.use_anomaly_detection = false
+    collection.save!
+  end
+
+end

log/whenever/.gitignore

@@ -0,0 +1 @@
+*.log

test/mailers/previews/user_mailer_preview.rb

@@ -0,0 +1,7 @@
+class UserMailerPreview < ActionMailer::Preview
+  def exercise_anomaly_detected()
+    collection = ExerciseCollection.new(name: 'Hello World', user: FactoryBot.create(:admin))
+    anomalies = {49 => 879.325828, 51 => 924.870057, 31 => 1031.21233, 69 => 2159.182116}
+    UserMailer.exercise_anomaly_detected(collection, anomalies)
+  end
+end