diff --git a/.gitignore b/.gitignore
index af16e87b..c1030982 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,7 +10,7 @@
/config/*.staging-epic.yml
/config/deploy/staging-epic.rb
/coverage
-/log
+/log/*.*
/public/assets
/public/uploads
/rubocop.html
diff --git a/Capfile b/Capfile
index 620d59ef..30744292 100644
--- a/Capfile
+++ b/Capfile
@@ -6,6 +6,7 @@ require 'capistrano/puma/nginx'
require 'capistrano/rails'
require 'capistrano/rvm'
require 'capistrano/upload-config'
+require 'whenever/capistrano'
install_plugin Capistrano::SCM::Git
install_plugin Capistrano::Puma
diff --git a/Gemfile b/Gemfile
index 8482ca69..6a6486a6 100644
--- a/Gemfile
+++ b/Gemfile
@@ -43,6 +43,7 @@ gem 'nokogiri'
gem 'd3-rails'
gem 'rest-client'
gem 'rubyzip'
+gem 'whenever', require: false
group :development, :staging do
gem 'better_errors', platform: :ruby
diff --git a/Gemfile.lock b/Gemfile.lock
index 43737a82..7edb56f0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -32,11 +32,6 @@ GEM
activesupport (= 4.2.10)
arel (~> 6.0)
activerecord-deprecated_finders (1.0.4)
- activerecord-jdbc-adapter (50.0)
- activerecord (>= 2.2)
- activerecord-jdbcpostgresql-adapter (50.0)
- activerecord-jdbc-adapter (~> 50.0)
- jdbc-postgres (>= 9.4, < 43)
activesupport (4.2.10)
i18n (~> 0.7)
minitest (~> 5.1)
@@ -51,7 +46,6 @@ GEM
autotest-rails (4.2.1)
ZenTest (~> 4.5)
bcrypt (3.1.11)
- bcrypt (3.1.11-java)
better_errors (2.4.0)
coderay (>= 1.0.0)
erubi (>= 1.0.0)
@@ -97,6 +91,7 @@ GEM
mime-types (>= 1.16)
childprocess (0.8.0)
ffi (~> 1.0, >= 1.0.11)
+ chronic (0.10.2)
codeclimate-test-reporter (1.0.7)
simplecov
coderay (1.1.2)
@@ -108,25 +103,23 @@ GEM
execjs
coffee-script-source (1.12.2)
concurrent-ruby (1.0.5)
- concurrent-ruby (1.0.5-java)
concurrent-ruby-ext (1.0.5)
concurrent-ruby (= 1.0.5)
crass (1.0.3)
- d3-rails (4.10.2)
+ d3-rails (4.13.0)
railties (>= 3.1)
database_cleaner (1.6.2)
debug_inspector (0.0.3)
diff-lcs (1.3)
docile (1.1.5)
- docker-api (1.34.0)
+ docker-api (1.34.1)
excon (>= 0.47.0)
multi_json
domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0)
- erubi (1.7.0)
+ erubi (1.7.1)
erubis (2.7.0)
eventmachine (1.0.9.1)
- eventmachine (1.0.9.1-java)
excon (0.60.0)
execjs (2.7.0)
factory_bot (4.8.2)
@@ -139,8 +132,7 @@ GEM
faye-websocket (0.10.7)
eventmachine (>= 0.12.0)
websocket-driver (>= 0.5.1)
- ffi (1.9.21)
- ffi (1.9.21-java)
+ ffi (1.9.23)
forgery (0.7.0)
globalid (0.4.1)
activesupport (>= 4.2.0)
@@ -155,7 +147,6 @@ GEM
jbuilder (2.7.0)
activesupport (>= 4.2.0)
multi_json (>= 1.2)
- jdbc-postgres (42.1.4)
jquery-rails (4.3.1)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
@@ -164,7 +155,6 @@ GEM
railties (>= 3.1.0)
turbolinks
json (2.1.0)
- json (2.1.0-java)
jwt (1.5.6)
kramdown (1.16.2)
loofah (2.2.0)
@@ -189,7 +179,6 @@ GEM
newrelic_rpm (4.8.0.341)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
- nokogiri (1.8.2-java)
nyan-cat-formatter (0.12.0)
rspec (>= 2.99, >= 2.14.2, < 4)
oauth (0.4.7)
@@ -202,7 +191,7 @@ GEM
pagedown-rails (1.1.4)
railties (> 3.1)
parallel (1.12.1)
- parser (2.5.0.2)
+ parser (2.5.0.3)
ast (~> 2.4.0)
pg (0.21.0)
polyamorous (1.3.3)
@@ -211,19 +200,14 @@ GEM
pry (0.11.3)
coderay (~> 1.1.0)
method_source (~> 0.9.0)
- pry (0.11.3-java)
- coderay (~> 1.1.0)
- method_source (~> 0.9.0)
- spoon (~> 0.0)
pry-byebug (3.6.0)
byebug (~> 10.0)
pry (~> 0.10)
public_suffix (3.0.2)
- puma (3.11.2)
- puma (3.11.2-java)
+ puma (3.11.3)
pundit (1.1.0)
activesupport (>= 3.0.0)
- rack (1.6.8)
+ rack (1.6.9)
rack-mini-profiler (0.10.7)
rack (>= 1.2.0)
rack-test (0.6.3)
@@ -263,7 +247,7 @@ GEM
activesupport (>= 3.0)
i18n
polyamorous (~> 1.3.2)
- rb-fsevent (0.10.2)
+ rb-fsevent (0.10.3)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
rdoc (6.0.1)
@@ -294,15 +278,15 @@ GEM
rspec-mocks (~> 3.7.0)
rspec-support (~> 3.7.0)
rspec-support (3.7.1)
- rubocop (0.52.1)
+ rubocop (0.53.0)
parallel (~> 1.10)
- parser (>= 2.4.0.2, < 3.0)
+ parser (>= 2.5)
powerpack (~> 0.1)
rainbow (>= 2.2.2, < 4.0)
ruby-progressbar (~> 1.7)
unicode-display_width (~> 1.0, >= 1.0.1)
- rubocop-rspec (1.22.2)
- rubocop (>= 0.52.1)
+ rubocop-rspec (1.24.0)
+ rubocop (>= 0.53.0)
ruby-progressbar (1.9.0)
rubytree (1.0.0)
json (~> 2.1)
@@ -321,7 +305,7 @@ GEM
tilt (>= 1.1, < 3)
sdoc (1.0.0)
rdoc (>= 5.0)
- selenium-webdriver (3.9.0)
+ selenium-webdriver (3.10.0)
childprocess (~> 0.5)
rubyzip (~> 1.2)
simplecov (0.15.1)
@@ -336,8 +320,6 @@ GEM
bcrypt (~> 3.1)
oauth (~> 0.4, >= 0.4.4)
oauth2 (~> 1.0, >= 0.8.0)
- spoon (0.0.6)
- ffi
spring (2.0.2)
activesupport (>= 4.2)
sprockets (3.7.1)
@@ -354,7 +336,6 @@ GEM
temple (0.8.0)
thor (0.20.0)
thread_safe (0.3.6)
- thread_safe (0.3.6-java)
tilt (2.0.8)
tubesock (0.2.7)
rack (>= 1.5.0)
@@ -367,7 +348,6 @@ GEM
execjs (>= 0.3.0, < 3)
unf (0.1.4)
unf_ext
- unf (0.1.4-java)
unf_ext (0.0.7.5)
unicode-display_width (1.3.0)
web-console (3.3.0)
@@ -377,15 +357,14 @@ GEM
websocket (1.2.5)
websocket-driver (0.7.0)
websocket-extensions (>= 0.1.0)
- websocket-driver (0.7.0-java)
- websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.3)
+ whenever (0.10.0)
+ chronic (>= 0.6.3)
will_paginate (3.1.6)
xpath (3.0.0)
nokogiri (~> 1.8)
PLATFORMS
- java
ruby
DEPENDENCIES
@@ -454,6 +433,7 @@ DEPENDENCIES
turbolinks (< 5.0.0)
uglifier
web-console
+ whenever
will_paginate
BUNDLED WITH
diff --git a/Vagrantfile b/Vagrantfile
index 5ec9182e..cc6acf2c 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -8,5 +8,5 @@ Vagrant.configure(2) do |config|
end
config.vm.network "private_network", ip: "192.168.59.104"
# config.vm.synced_folder "../data", "/vagrant_data"
- config.vm.provision "shell", path: "provision.sh"
+ config.vm.provision "shell", path: "provision.sh", privileged: false
end
diff --git a/app/controllers/exercise_collections_controller.rb b/app/controllers/exercise_collections_controller.rb
index 6d239d9c..3a6112ef 100644
--- a/app/controllers/exercise_collections_controller.rb
+++ b/app/controllers/exercise_collections_controller.rb
@@ -10,6 +10,7 @@ class ExerciseCollectionsController < ApplicationController
end
def show
+ @exercises = @exercise_collection.exercises.paginate(:page => params[:page])
end
def new
@@ -55,6 +56,6 @@ class ExerciseCollectionsController < ApplicationController
end
def exercise_collection_params
- params[:exercise_collection].permit(:name, :exercise_ids => [])
+ params[:exercise_collection].permit(:name, :use_anomaly_detection, :user_id, :user_type, :exercise_ids => []).merge(user_type: InternalUser.name)
end
end
diff --git a/app/controllers/user_exercise_feedbacks_controller.rb b/app/controllers/user_exercise_feedbacks_controller.rb
index 8abffd66..4c350a81 100644
--- a/app/controllers/user_exercise_feedbacks_controller.rb
+++ b/app/controllers/user_exercise_feedbacks_controller.rb
@@ -69,7 +69,8 @@ class UserExerciseFeedbacksController < ApplicationController
@texts = comment_presets.to_a
@times = time_presets.to_a
@uef = UserExerciseFeedback.new
- @exercise = Exercise.find(params[:user_exercise_feedback][:exercise_id])
+ exercise_id = if params[:user_exercise_feedback].nil? then params[:exercise_id] else params[:user_exercise_feedback][:exercise_id] end
+ @exercise = Exercise.find(exercise_id)
authorize!
end
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index 8022ee84..497c2a31 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -37,4 +37,18 @@ class UserMailer < ActionMailer::Base
@rfc_link = request_for_comment_url(request_for_comments)
mail(subject: t('mailers.user_mailer.send_thank_you_note.subject', author: @author), to: receiver.email)
end
+
+ def exercise_anomaly_detected(exercise_collection, anomalies)
+ @receiver_displayname = exercise_collection.user.displayname
+ @collection = exercise_collection
+ @anomalies = anomalies
+ mail(subject: t('mailers.user_mailer.exercise_anomaly_detected.subject'), to: exercise_collection.user.email)
+ end
+
+ def exercise_anomaly_needs_feedback(user, exercise, link)
+ @receiver_displayname = user.displayname
+ @exercise_title = exercise.title
+ @link = link
+ mail(subject: t('mailers.user_mailer.exercise_anomaly_needs_feedback.subject'), to: user.email)
+ end
end
diff --git a/app/models/anomaly_notification.rb b/app/models/anomaly_notification.rb
new file mode 100644
index 00000000..8c9b90cd
--- /dev/null
+++ b/app/models/anomaly_notification.rb
@@ -0,0 +1,5 @@
+class AnomalyNotification < ActiveRecord::Base
+ belongs_to :user, polymorphic: true
+ belongs_to :exercise
+ belongs_to :exercise_collection
+end
diff --git a/app/models/exercise.rb b/app/models/exercise.rb
index f03d6641..efdb00c7 100644
--- a/app/models/exercise.rb
+++ b/app/models/exercise.rb
@@ -36,6 +36,7 @@ class Exercise < ActiveRecord::Base
validates :token, presence: true, uniqueness: true
@working_time_statistics = nil
+ attr_reader :working_time_statistics
MAX_EXERCISE_FEEDBACKS = 20
@@ -65,21 +66,27 @@ class Exercise < ActiveRecord::Base
end
def user_working_time_query
- """
+ "
SELECT user_id,
- sum(working_time_new) AS working_time
+ user_type,
+ SUM(working_time_new) AS working_time,
+ MAX(score) AS score
FROM
(SELECT user_id,
+ user_type,
+ score,
CASE WHEN working_time >= '0:05:00' THEN '0' ELSE working_time END AS working_time_new
FROM
(SELECT user_id,
+ user_type,
+ score,
id,
(created_at - lag(created_at) over (PARTITION BY user_id, exercise_id
ORDER BY created_at)) AS working_time
FROM submissions
WHERE exercise_id=#{id}) AS foo) AS bar
- GROUP BY user_id
- """
+ GROUP BY user_id, user_type
+ "
end
def get_quantiles(quantiles)
@@ -202,7 +209,7 @@ class Exercise < ActiveRecord::Base
def retrieve_working_time_statistics
@working_time_statistics = {}
self.class.connection.execute(user_working_time_query).each do |tuple|
- @working_time_statistics[tuple["user_id"].to_i] = tuple
+ @working_time_statistics[tuple['user_id'].to_i] = tuple
end
end
@@ -368,4 +375,15 @@ class Exercise < ActiveRecord::Base
user_exercise_feedbacks.size <= MAX_EXERCISE_FEEDBACKS
end
+ def last_submission_per_user
+ Submission.joins("JOIN (
+ SELECT
+ user_id,
+ user_type,
+ first_value(id) OVER (PARTITION BY user_id ORDER BY created_at DESC) AS fv
+ FROM submissions
+ WHERE exercise_id = #{id}
+ ) AS t ON t.fv = submissions.id").distinct
+ end
+
end
diff --git a/app/models/exercise_collection.rb b/app/models/exercise_collection.rb
index 10946962..f9f09269 100644
--- a/app/models/exercise_collection.rb
+++ b/app/models/exercise_collection.rb
@@ -1,6 +1,7 @@
class ExerciseCollection < ActiveRecord::Base
has_and_belongs_to_many :exercises
+ belongs_to :user, polymorphic: true
def to_s
"#{I18n.t('activerecord.models.exercise_collection.one')}: #{name} (#{id})"
diff --git a/app/views/exercise_collections/_form.html.slim b/app/views/exercise_collections/_form.html.slim
index 3c336a62..c204db47 100644
--- a/app/views/exercise_collections/_form.html.slim
+++ b/app/views/exercise_collections/_form.html.slim
@@ -1,11 +1,18 @@
- exercises = Exercise.order(:title)
+- users = InternalUser.order(:name)
-= form_for(@exercise_collection, data: {exercises: exercises}, multipart: true) do |f|
+= form_for(@exercise_collection, data: {exercises: exercises, users: users}, multipart: true) do |f|
= render('shared/form_errors', object: @exercise_collection)
.form-group
- = f.label(:name)
+ = f.label(t('activerecord.attributes.exercise_collections.name'))
= f.text_field(:name, class: 'form-control', required: true)
.form-group
- = f.label(:exercises)
+ = f.label(t('activerecord.attributes.exercise_collections.use_anomaly_detection'))
+ = f.check_box(:use_anomaly_detection, {class: 'form-control'})
+ .form-group
+ = f.label(t('activerecord.attributes.exercise_collections.user'))
+ = f.collection_select(:user_id, users, :id, :name, {}, {class: 'form-control'})
+ .form-group
+ = f.label(t('activerecord.attributes.exercise_collections.exercises'))
= f.collection_select(:exercise_ids, exercises, :id, :title, {}, {class: 'form-control', multiple: true})
.actions = render('shared/submit_button', f: f, object: @exercise_collection)
diff --git a/app/views/exercise_collections/show.html.slim b/app/views/exercise_collections/show.html.slim
index 65c28283..ab85a3ba 100644
--- a/app/views/exercise_collections/show.html.slim
+++ b/app/views/exercise_collections/show.html.slim
@@ -3,9 +3,25 @@ h1
= render('shared/edit_button', object: @exercise_collection)
= row(label: 'exercise_collections.name', value: @exercise_collection.name)
+= row(label: 'exercise_collections.user', value: link_to(@exercise_collection.user.name, @exercise_collection.user)) unless @exercise_collection.user.nil?
+= row(label: 'exercise_collections.use_anomaly_detection', value: @exercise_collection.use_anomaly_detection)
= row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at)
h4 = t('activerecord.attributes.exercise_collections.exercises')
-ul.list-unstyled
- - @exercise_collection.exercises.sort_by{|c| c.title}.each do |exercise|
- li = link_to(exercise, exercise)
+.table-responsive
+ table.table
+ thead
+ tr
+ th = t('activerecord.attributes.exercise.title')
+ th = t('activerecord.attributes.exercise.execution_environment')
+ th = t('activerecord.attributes.exercise.user')
+ th = t('shared.actions')
+ tbody
+ - @exercises.sort_by{|c| c.title}.each do |exercise|
+ tr
+ td = link_to(exercise.title, exercise)
+ td = link_to_if(exercise.execution_environment && policy(exercise.execution_environment).show?, exercise.execution_environment, exercise.execution_environment)
+ td = exercise.user.name
+ td = link_to(t('shared.statistics'), statistics_exercise_path(exercise))
+
+= render('shared/pagination', collection: @exercises)
diff --git a/app/views/user_mailer/exercise_anomaly_detected.html.slim b/app/views/user_mailer/exercise_anomaly_detected.html.slim
new file mode 100644
index 00000000..56233ce0
--- /dev/null
+++ b/app/views/user_mailer/exercise_anomaly_detected.html.slim
@@ -0,0 +1,38 @@
+== t('mailers.user_mailer.exercise_anomaly_detected.body1',
+ receiver_displayname: @receiver_displayname,
+ collection_name: @collection.name)
+
+table(border=1)
+ thead
+ tr
+ td = t('activerecord.attributes.exercise.title', locale: :de)
+ td = t('exercises.statistics.average_worktime', locale: :de)
+ td = t('shared.actions', locale: :de)
+ tbody
+ - @anomalies.keys.each do | id |
+ - exercise = Exercise.find(id)
+ tr
+ td = link_to(exercise.title, exercise_path(exercise))
+ td = @anomalies[id]
+ td = link_to(t('shared.statistics', locale: :de), statistics_exercise_path(exercise))
+
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body2',
+ receiver_displayname: @receiver_displayname,
+ collection_name: @collection.name)
+
+table(border=1)
+ thead
+ tr
+ td = t('activerecord.attributes.exercise.title', locale: :en)
+ td = t('exercises.statistics.average_worktime', locale: :en)
+ td = t('shared.actions', locale: :en)
+ tbody
+ - @anomalies.keys.each do | id |
+ - exercise = Exercise.find(id)
+ tr
+ td = link_to(exercise.title, exercise_path(exercise))
+ td = @anomalies[id]
+ td = link_to(t('shared.statistics', locale: :en), statistics_exercise_path(exercise))
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body3')
diff --git a/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim b/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim
new file mode 100644
index 00000000..336f3c29
--- /dev/null
+++ b/app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim
@@ -0,0 +1 @@
+== t('mailers.user_mailer.exercise_anomaly_needs_feedback.body', receiver_displayname: @receiver_displayname, exercise: @exercise_title, link: link_to(@link, @link))
diff --git a/config/deploy.rb b/config/deploy.rb
index 60edce52..1fac842e 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -9,6 +9,8 @@ set :log_level, :info
set :puma_threads, [0, 16]
set :repo_url, 'git@github.com:openHPI/codeocean.git'
+set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" }
+
namespace :deploy do
before 'check:linked_files', 'config:push'
@@ -21,3 +23,11 @@ namespace :deploy do
end
end
end
+
+namespace :whenever do
+ task :update_crontab do
+ run 'bundle exec whenever --update-crontab'
+ end
+end
+
+after 'deploy', 'whenever:update_crontab'
diff --git a/config/locales/de.yml b/config/locales/de.yml
index cb042f7d..281d349d 100644
--- a/config/locales/de.yml
+++ b/config/locales/de.yml
@@ -123,6 +123,8 @@ de:
exercise_collections:
id: "ID"
name: "Name"
+ user: "Verantwortlicher"
+ use_anomaly_detection: "Abweichungen in der Arbeitszeit erkennen"
updated_at: "Letzte Änderung"
exercises: "Aufgaben"
user_exercise_feedback:
@@ -520,6 +522,64 @@ de:
This mail was automatically sent by CodeOcean.
subject: "%{author_displayname} hat einen neuen Kommentar in einer Diskussion veröffentlicht, die Sie abonniert haben."
+ exercise_anomaly_detected:
+ subject: "Unregelmäßigkeiten in Aufgaben Ihrer Aufgabensammlung"
+ body1: |
+ English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+
+ Die Aufgaben sind:
+
+ body2: |
+
+ Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+
+ The exercises are:
+
+ body3: |
+
+ If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ exercise_anomaly_needs_feedback:
+ body: |
+ English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+
+ %{exercise} - %{link}
+
+ Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+
+ %{exercise} - %{link}
+
+ If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ subject: "Eine Aufgabe auf CodeOcean benötigt Ihr Feedback"
request_for_comments:
click_here: Zum Kommentieren auf die Seitenleiste klicken!
comments: Kommentare
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4e3564bd..c5e05183 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -123,6 +123,8 @@ en:
exercise_collections:
id: "ID"
name: "Name"
+ user: "Associated User"
+ use_anomaly_detection: "Enable Worktime Anomaly Detection"
updated_at: "Last Update"
exercises: "Exercises"
user_exercise_feedback:
@@ -520,6 +522,64 @@ en:
This mail was automatically sent by CodeOcean.
subject: "%{author_displayname} has posted a new comment to a discussion you subscribed to on CodeOcean."
+ exercise_anomaly_detected:
+ subject: "Anomalies in exercises of your exercise collection"
+ body1: |
+ English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+
+ Die Aufgaben sind:
+
+ body2: |
+
+ Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+
+ The exercises are:
+
+ body3: |
+
+ If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ exercise_anomaly_needs_feedback:
+ body: |
+ English version below
+ _________________________
+
+ Hallo %{receiver_displayname},
+
+ um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+
+ %{exercise} - %{link}
+
+ Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.
+
+ Diese Mail wurde automatisch von CodeOcean verschickt.
+
+ _________________________
+
+ Dear %{receiver_displayname},
+
+ we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+
+ %{exercise} - %{link}
+
+ If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again.
+
+ This mail was automatically sent by CodeOcean.
+ subject: "An exercise on CodeOcean needs your feedback"
request_for_comments:
click_here: Click on this sidebar to comment!
comments: Comments
diff --git a/config/schedule.rb b/config/schedule.rb
new file mode 100644
index 00000000..f7ad2f74
--- /dev/null
+++ b/config/schedule.rb
@@ -0,0 +1,27 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+# command "/usr/bin/some_great_command"
+# runner "MyModel.some_method"
+# rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+# runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+set :output, Whenever.path + '/log/whenever/whenever_$(date +%Y%m%d%H%M%S).log'
+set :environment, ENV['RAILS_ENV'] if ENV['RAILS_ENV']
+
+every 1.day, at: '3:00 am' do
+ rake 'detect_exercise_anomalies:with_at_least[50,50]'
+end
diff --git a/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb b/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb
new file mode 100644
index 00000000..2aad7e28
--- /dev/null
+++ b/db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb
@@ -0,0 +1,5 @@
+class AddAnomalyDetectionFlagToExerciseCollection < ActiveRecord::Migration
+ def change
+ add_column :exercise_collections, :use_anomaly_detection, :boolean, :default => false
+ end
+end
diff --git a/db/migrate/20171122124222_add_index_to_exercises.rb b/db/migrate/20171122124222_add_index_to_exercises.rb
new file mode 100644
index 00000000..cf1f4674
--- /dev/null
+++ b/db/migrate/20171122124222_add_index_to_exercises.rb
@@ -0,0 +1,5 @@
+class AddIndexToExercises < ActiveRecord::Migration
+ def change
+ add_index :exercises, :id
+ end
+end
diff --git a/db/migrate/20171210172208_add_user_to_exercise_collection.rb b/db/migrate/20171210172208_add_user_to_exercise_collection.rb
new file mode 100644
index 00000000..6dee1adf
--- /dev/null
+++ b/db/migrate/20171210172208_add_user_to_exercise_collection.rb
@@ -0,0 +1,5 @@
+class AddUserToExerciseCollection < ActiveRecord::Migration
+ def change
+ add_reference :exercise_collections, :user, polymorphic: true, index: true
+ end
+end
diff --git a/db/migrate/20180226131340_create_anomaly_notifications.rb b/db/migrate/20180226131340_create_anomaly_notifications.rb
new file mode 100644
index 00000000..5de3dbe4
--- /dev/null
+++ b/db/migrate/20180226131340_create_anomaly_notifications.rb
@@ -0,0 +1,11 @@
+class CreateAnomalyNotifications < ActiveRecord::Migration
+ def change
+ create_table :anomaly_notifications do |t|
+ t.belongs_to :user, polymorphic: true, index: true
+ t.belongs_to :exercise, index: true
+ t.belongs_to :exercise_collection, index: true
+ t.string :reason
+ t.timestamps
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0280a8c4..55339422 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,11 +11,25 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180222145909) do
+ActiveRecord::Schema.define(version: 20180226131340) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
+ create_table "anomaly_notifications", force: :cascade do |t|
+ t.integer "user_id"
+ t.string "user_type"
+ t.integer "exercise_id"
+ t.integer "exercise_collection_id"
+ t.string "reason"
+ t.datetime "created_at"
+ t.datetime "updated_at"
+ end
+
+ add_index "anomaly_notifications", ["exercise_collection_id"], name: "index_anomaly_notifications_on_exercise_collection_id", using: :btree
+ add_index "anomaly_notifications", ["exercise_id"], name: "index_anomaly_notifications_on_exercise_id", using: :btree
+ add_index "anomaly_notifications", ["user_type", "user_id"], name: "index_anomaly_notifications_on_user_type_and_user_id", using: :btree
+
create_table "code_harbor_links", force: :cascade do |t|
t.string "oauth2token", limit: 255
t.datetime "created_at"
@@ -104,8 +118,13 @@ ActiveRecord::Schema.define(version: 20180222145909) do
t.string "name"
t.datetime "created_at"
t.datetime "updated_at"
+ t.boolean "use_anomaly_detection", default: false
+ t.integer "user_id"
+ t.string "user_type"
end
+ add_index "exercise_collections", ["user_type", "user_id"], name: "index_exercise_collections_on_user_type_and_user_id", using: :btree
+
create_table "exercise_collections_exercises", id: false, force: :cascade do |t|
t.integer "exercise_collection_id"
t.integer "exercise_id"
@@ -137,6 +156,8 @@ ActiveRecord::Schema.define(version: 20180222145909) do
t.integer "expected_difficulty", default: 1
end
+ add_index "exercises", ["id"], name: "index_exercises_on_id", using: :btree
+
create_table "exercises_proxy_exercises", id: false, force: :cascade do |t|
t.integer "proxy_exercise_id"
t.integer "exercise_id"
diff --git a/lib/tasks/detect_exercise_anomalies.rake b/lib/tasks/detect_exercise_anomalies.rake
new file mode 100644
index 00000000..e22550b2
--- /dev/null
+++ b/lib/tasks/detect_exercise_anomalies.rake
@@ -0,0 +1,165 @@
+include Rails.application.routes.url_helpers
+
+namespace :detect_exercise_anomalies do
+ # uncomment for debug logging:
+ # logger = Logger.new(STDOUT)
+ # logger.level = Logger::DEBUG
+ # Rails.logger = logger
+
+ # These factors determine if an exercise is an anomaly, given the average working time (avg):
+ # (avg * MIN_TIME_FACTOR) <= working_time <= (avg * MAX_TIME_FACTOR)
+ MIN_TIME_FACTOR = 0.1
+ MAX_TIME_FACTOR = 2
+
+ # Determines how many users are picked from the best/average/worst performers of each anomaly for feedback
+ NUMBER_OF_USERS_PER_CLASS = 10
+
+ # Determines margin below which user working times will be considered data errors (e.g. copy/paste solutions)
+ MIN_USER_WORKING_TIME = 0.0
+
+ # Cache exercise working times, because queries are expensive and values do not change between collections
+ WORKING_TIME_CACHE = {}
+ AVERAGE_WORKING_TIME_CACHE = {}
+
+ task :with_at_least, [:number_of_exercises, :number_of_solutions] => :environment do |task, args|
+ number_of_exercises = args[:number_of_exercises]
+ number_of_solutions = args[:number_of_solutions]
+
+ puts "Searching for exercise collections with at least #{number_of_exercises} exercises and #{number_of_solutions} users."
+ # Get all exercise collections that have at least the specified amount of exercises and at least the specified
+ # number of submissions AND are flagged for anomaly detection
+ collections = get_collections(number_of_exercises, number_of_solutions)
+ puts "Found #{collections.length}."
+
+ collections.each do |collection|
+ puts "\t- #{collection}"
+ anomalies = find_anomalies(collection)
+
+ if anomalies.length > 0 and not collection.user.nil?
+ notify_collection_author(collection, anomalies)
+ notify_users(collection, anomalies)
+ reset_anomaly_detection_flag(collection)
+ end
+ end
+ puts 'Done.'
+ end
+
+ def get_collections(number_of_exercises, number_of_solutions)
+ ExerciseCollection
+ .where(:use_anomaly_detection => true)
+ .joins("join exercise_collections_exercises ece on exercise_collections.id = ece.exercise_collection_id
+ join
+ (select e.id
+ from exercises e
+ join submissions s on s.exercise_id = e.id
+ group by e.id
+ having count(s.user_id) > #{ExerciseCollection.sanitize(number_of_solutions)}
+ ) as exercises_with_submissions on exercises_with_submissions.id = ece.exercise_id")
+ .group('exercise_collections.id')
+ .having('count(exercises_with_submissions.id) > ?', number_of_exercises)
+ end
+
+ def find_anomalies(collection)
+ working_times = {}
+ collection.exercises.each do |exercise|
+ puts "\t\t> #{exercise.title}"
+ working_times[exercise.id] = get_average_working_time(exercise)
+ end
+ average = working_times.values.reduce(:+) / working_times.size
+ working_times.select do |exercise_id, working_time|
+ working_time > average * MAX_TIME_FACTOR or working_time < average * MIN_TIME_FACTOR
+ end
+ end
+
+ def time_to_f(timestamp)
+ unless timestamp.nil?
+ timestamp = timestamp.split(':')
+ return timestamp[0].to_i * 60 * 60 + timestamp[1].to_i * 60 + timestamp[2].to_f
+ end
+ nil
+ end
+
+ def get_average_working_time(exercise)
+ unless AVERAGE_WORKING_TIME_CACHE.key?(exercise.id)
+ seconds = time_to_f exercise.average_working_time
+ AVERAGE_WORKING_TIME_CACHE[exercise.id] = seconds
+ end
+ AVERAGE_WORKING_TIME_CACHE[exercise.id]
+ end
+
+ def get_user_working_times(exercise)
+ unless WORKING_TIME_CACHE.key?(exercise.id)
+ exercise.retrieve_working_time_statistics
+ WORKING_TIME_CACHE[exercise.id] = exercise.working_time_statistics
+ end
+ WORKING_TIME_CACHE[exercise.id]
+ end
+
+ def notify_collection_author(collection, anomalies)
+ puts "\t\tSending E-Mail to author (#{collection.user.displayname} <#{collection.user.email}>)..."
+ UserMailer.exercise_anomaly_detected(collection, anomalies).deliver_now
+ end
+
+ def notify_users(collection, anomalies)
+ by_id_and_type = proc { |u| {user_id: u[:user_id], user_type: u[:user_type]} }
+
+ puts "\t\tSending E-Mails to best and worst performing users of each anomaly..."
+ anomalies.each do |exercise_id, average_working_time|
+ puts "\t\tAnomaly in exercise #{exercise_id} (avg: #{average_working_time} seconds):"
+ exercise = Exercise.find(exercise_id)
+ users_to_notify = []
+
+ users = {}
+ [:performers_by_time, :performers_by_score].each do |method|
+ # merge users found by multiple methods returning a hash {best: [], worst: []}
+ users = users.merge(send(method, exercise, NUMBER_OF_USERS_PER_CLASS)) {|key, this, other| this + other}
+ end
+
+ # write reasons for feedback emails to db
+ users.keys.each do |key|
+ segment = users[key].uniq &by_id_and_type
+ users_to_notify += segment
+ segment.each do |user|
+ reason = "{\"segment\": \"#{key.to_s}\", \"feature\": \"#{user[:reason]}\", value: \"#{user[:value]}\"}"
+ AnomalyNotification.create(user_id: user[:user_id], user_type: user[:user_type],
+ exercise: exercise, exercise_collection: collection, reason: reason)
+ end
+ end
+
+ users_to_notify.uniq! &by_id_and_type
+ users_to_notify.each do |u|
+ user = u[:user_type] == InternalUser.name ? InternalUser.find(u[:user_id]) : ExternalUser.find(u[:user_id])
+ host = CodeOcean::Application.config.action_mailer.default_url_options[:host]
+ feedback_link = url_for(action: :new, controller: :user_exercise_feedbacks, exercise_id: exercise.id, host: host)
+ UserMailer.exercise_anomaly_needs_feedback(user, exercise, feedback_link).deliver
+ end
+ puts "\t\tAsked #{users_to_notify.size} users for feedback."
+ end
+ end
+
+ def performers_by_score(exercise, n)
+ submissions = exercise.last_submission_per_user.where('score is not null').order(score: :desc)
+ map_block = proc {|item| {user_id: item.user_id, user_type: item.user_type, value: item.score, reason: 'score'}}
+ best_performers = submissions.first(n).to_a.map &map_block
+ worst_performers = submissions.last(n).to_a.map &map_block
+ return {:best => best_performers, :worst => worst_performers}
+ end
+
+ def performers_by_time(exercise, n)
+ working_times = get_user_working_times(exercise).values.map do |item|
+ {user_id: item['user_id'], user_type: item['user_type'], score: item['score'].to_f,
+ value: time_to_f(item['working_time']), reason: 'time'}
+ end
+ avg_score = exercise.average_score
+ working_times.reject! {|item| item[:value].nil? or item[:value] <= MIN_USER_WORKING_TIME or item[:score] < avg_score}
+ working_times.sort_by! {|item| item[:value]}
+ return {:best => working_times.first(n), :worst => working_times.last(n)}
+ end
+
+ def reset_anomaly_detection_flag(collection)
+ puts "\t\tResetting flag..."
+ collection.use_anomaly_detection = false
+ collection.save!
+ end
+
+end
diff --git a/log/whenever/.gitignore b/log/whenever/.gitignore
new file mode 100644
index 00000000..397b4a76
--- /dev/null
+++ b/log/whenever/.gitignore
@@ -0,0 +1 @@
+*.log
diff --git a/provision.sh b/provision.sh
index 60c9c04b..a6128d1d 100644
--- a/provision.sh
+++ b/provision.sh
@@ -2,71 +2,87 @@
# rvm/rails installation from https://gorails.com/setup/ubuntu/14.04
# passenger installation from https://www.phusionpassenger.com/library/install/nginx/install/oss/trusty/
+######## VERSION INFORMATION ########
+
+postgres_version=10
+ruby_version=2.3.6
+rails_version=4.2.10
+
+########## INSTALL SCRIPT ###########
+
+# PostgreSQL
+sudo add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+
# passenger
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7
-apt-get install -y apt-transport-https ca-certificates
-sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list'
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7
+sudo apt-get -qq -y install apt-transport-https ca-certificates
+sudo sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list'
# rails
-add-apt-repository ppa:chris-lea/node.js
+sudo add-apt-repository -y ppa:chris-lea/node.js
-apt-get update
+sudo apt-get -qq update
# code_ocean
-apt-get install -y postgresql-client postgresql-10 postgresql-server-dev-10 vagrant
+sudo apt-get -qq -y install postgresql-client postgresql-$postgres_version postgresql-server-dev-$postgres_version vagrant
# Docker
if [ ! -f /etc/default/docker ]
then
- curl -sSL https://get.docker.com/ | sh
+ curl -sSL https://get.docker.com/ | sudo sh
fi
if ! grep code_ocean /etc/default/docker
then
- cat >>/etc/default/docker </etc/postgresql/10/main/pg_hba.conf < /etc/nginx/sites-available/code_ocean < 879.325828, 51 => 924.870057, 31 => 1031.21233, 69 => 2159.182116}
+ UserMailer.exercise_anomaly_detected(collection, anomalies)
+ end
+end