merged master into disable_rfcs

.gitignore

@@ -10,7 +10,7 @@
 /config/*.staging-epic.yml
 /config/deploy/staging-epic.rb
 /coverage
-/log
+/log/*.*
 /public/assets
 /public/uploads
 /rubocop.html

.travis.yml

@@ -8,6 +8,7 @@ addons:
     repo_token:
       secure: "cZoMNjQKB/D7W4B7JDk9PXooy2WCDypu7R4C/Vi0DziZCU9HRwLbdt9aoH5hgHFa7Fe2rHFgflPAAP7h698ozvP0waFtPqLAj+PbEt27LbBDvW8JcvNkKXA0rj5wyTkzuc/0kD+kPB4oDXMak6gZlB9HCJDsa3kdXScQGTVuPdU="
   postgresql: "9.6"
+  firefox: "latest"
 
 before_install:
   - export DISPLAY=:99.0
@@ -17,6 +18,12 @@ before_install:
   - sleep 5
   - docker pull openhpi/co_execenv_python
   - docker pull openhpi/co_execenv_java
+  - mkdir ~/geckodriver
+  - wget -O ~/geckodriver/download.tar.gz https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz
+  - tar -xvzf ~/geckodriver/download.tar.gz -C ~/geckodriver/
+  - rm ~/geckodriver/download.tar.gz
+  - chmod +x ~/geckodriver/geckodriver
+  - export PATH=~/geckodriver/:$PATH
 
 before_script:
   - cp .rspec.travis .rspec

Capfile

@@ -6,6 +6,7 @@ require 'capistrano/puma/nginx'
 require 'capistrano/rails'
 require 'capistrano/rvm'
 require 'capistrano/upload-config'
+require 'whenever/capistrano'
 
 install_plugin Capistrano::SCM::Git
 install_plugin Capistrano::Puma

Gemfile

@@ -4,7 +4,6 @@ gem 'activerecord-jdbcpostgresql-adapter', platform: :jruby
 gem 'bcrypt'
 gem 'bootstrap-will_paginate'
 gem 'carrierwave'
-gem 'coffee-rails'
 gem 'concurrent-ruby'
 gem 'concurrent-ruby-ext', platform: :ruby
 gem 'activerecord-deprecated_finders', require: 'active_record/deprecated_finders'
@@ -28,7 +27,7 @@ gem 'ransack'
 gem 'rubytree'
 gem 'sass-rails'
 gem 'sdoc', group: :doc
-gem 'slim'
+gem 'slim-rails'
 gem 'bootstrap_pagedown'
 gem 'pagedown-rails'
 gem 'sorcery'
@@ -40,9 +39,10 @@ gem 'tubesock'
 gem 'faye-websocket'
 gem 'eventmachine', '1.0.9.1' # explicitly added, this is used by faye-websocket, version 1.2.5 still has an error in eventmachine.rb:202: [BUG] Segmentation fault, which is not yet fixed and causes the whole ruby process to crash
 gem 'nokogiri'
-gem 'd3-rails'
+gem 'd3-rails', '~>4.0'
 gem 'rest-client'
 gem 'rubyzip'
+gem 'whenever', require: false
 
 group :development, :staging do
   gem 'better_errors', platform: :ruby
@@ -66,12 +66,13 @@ end
 group :test do
   gem 'autotest-rails'
   gem 'capybara'
+  gem 'capybara-selenium'
+  gem 'headless'
   gem 'codeclimate-test-reporter', require: false
   gem 'database_cleaner'
   gem 'nyan-cat-formatter'
   gem 'rake'
   gem 'rspec-autotest'
   gem 'rspec-rails'
-  gem 'selenium-webdriver'
   gem 'simplecov', require: false
 end

Gemfile.lock

@@ -32,11 +32,6 @@ GEM
       activesupport (= 4.2.10)
       arel (~> 6.0)
     activerecord-deprecated_finders (1.0.4)
-    activerecord-jdbc-adapter (50.0)
-      activerecord (>= 2.2)
-    activerecord-jdbcpostgresql-adapter (50.0)
-      activerecord-jdbc-adapter (~> 50.0)
-      jdbc-postgres (>= 9.4, < 43)
     activesupport (4.2.10)
       i18n (~> 0.7)
       minitest (~> 5.1)
@@ -51,7 +46,6 @@ GEM
     autotest-rails (4.2.1)
       ZenTest (~> 4.5)
     bcrypt (3.1.11)
-    bcrypt (3.1.11-java)
     better_errors (2.4.0)
       coderay (>= 1.0.0)
       erubi (>= 1.0.0)
@@ -91,12 +85,16 @@ GEM
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (>= 2.0, < 4.0)
+    capybara-selenium (0.0.6)
+      capybara
+      selenium-webdriver
     carrierwave (1.2.2)
       activemodel (>= 4.0.0)
       activesupport (>= 4.0.0)
       mime-types (>= 1.16)
     childprocess (0.8.0)
       ffi (~> 1.0, >= 1.0.11)
+    chronic (0.10.2)
     codeclimate-test-reporter (1.0.7)
       simplecov
     coderay (1.1.2)
@@ -108,25 +106,23 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.0.5)
-    concurrent-ruby (1.0.5-java)
     concurrent-ruby-ext (1.0.5)
       concurrent-ruby (= 1.0.5)
     crass (1.0.3)
-    d3-rails (4.10.2)
+    d3-rails (4.13.0)
       railties (>= 3.1)
     database_cleaner (1.6.2)
     debug_inspector (0.0.3)
     diff-lcs (1.3)
     docile (1.1.5)
-    docker-api (1.34.0)
+    docker-api (1.34.1)
       excon (>= 0.47.0)
       multi_json
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
-    erubi (1.7.0)
+    erubi (1.7.1)
     erubis (2.7.0)
     eventmachine (1.0.9.1)
-    eventmachine (1.0.9.1-java)
     excon (0.60.0)
     execjs (2.7.0)
     factory_bot (4.8.2)
@@ -139,11 +135,11 @@ GEM
     faye-websocket (0.10.7)
       eventmachine (>= 0.12.0)
       websocket-driver (>= 0.5.1)
-    ffi (1.9.21)
+    ffi (1.9.23)
-    ffi (1.9.21-java)
     forgery (0.7.0)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
+    headless (2.3.1)
     highline (1.7.10)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
@@ -155,7 +151,6 @@ GEM
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
-    jdbc-postgres (42.1.4)
     jquery-rails (4.3.1)
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
@@ -164,7 +159,6 @@ GEM
       railties (>= 3.1.0)
       turbolinks
     json (2.1.0)
-    json (2.1.0-java)
     jwt (1.5.6)
     kramdown (1.16.2)
     loofah (2.2.0)
@@ -189,7 +183,6 @@ GEM
     newrelic_rpm (4.8.0.341)
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
-    nokogiri (1.8.2-java)
     nyan-cat-formatter (0.12.0)
       rspec (>= 2.99, >= 2.14.2, < 4)
     oauth (0.4.7)
@@ -202,7 +195,7 @@ GEM
     pagedown-rails (1.1.4)
       railties (> 3.1)
     parallel (1.12.1)
-    parser (2.5.0.2)
+    parser (2.5.0.3)
       ast (~> 2.4.0)
     pg (0.21.0)
     polyamorous (1.3.3)
@@ -211,19 +204,14 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    pry (0.11.3-java)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-      spoon (~> 0.0)
     pry-byebug (3.6.0)
       byebug (~> 10.0)
       pry (~> 0.10)
     public_suffix (3.0.2)
-    puma (3.11.2)
+    puma (3.11.3)
-    puma (3.11.2-java)
     pundit (1.1.0)
       activesupport (>= 3.0.0)
-    rack (1.6.8)
+    rack (1.6.9)
     rack-mini-profiler (0.10.7)
       rack (>= 1.2.0)
     rack-test (0.6.3)
@@ -263,7 +251,7 @@ GEM
       activesupport (>= 3.0)
       i18n
       polyamorous (~> 1.3.2)
-    rb-fsevent (0.10.2)
+    rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
     rdoc (6.0.1)
@@ -294,15 +282,15 @@ GEM
       rspec-mocks (~> 3.7.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rubocop (0.52.1)
+    rubocop (0.53.0)
       parallel (~> 1.10)
-      parser (>= 2.4.0.2, < 3.0)
+      parser (>= 2.5)
       powerpack (~> 0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    rubocop-rspec (1.22.2)
+    rubocop-rspec (1.24.0)
-      rubocop (>= 0.52.1)
+      rubocop (>= 0.53.0)
     ruby-progressbar (1.9.0)
     rubytree (1.0.0)
       json (~> 2.1)
@@ -321,7 +309,7 @@ GEM
       tilt (>= 1.1, < 3)
     sdoc (1.0.0)
       rdoc (>= 5.0)
-    selenium-webdriver (3.9.0)
+    selenium-webdriver (3.10.0)
       childprocess (~> 0.5)
       rubyzip (~> 1.2)
     simplecov (0.15.1)
@@ -332,12 +320,14 @@ GEM
     slim (3.0.9)
       temple (>= 0.7.6, < 0.9)
       tilt (>= 1.3.3, < 2.1)
+    slim-rails (3.1.3)
+      actionpack (>= 3.1)
+      railties (>= 3.1)
+      slim (~> 3.0)
     sorcery (0.11.0)
       bcrypt (~> 3.1)
       oauth (~> 0.4, >= 0.4.4)
       oauth2 (~> 1.0, >= 0.8.0)
-    spoon (0.0.6)
-      ffi
     spring (2.0.2)
       activesupport (>= 4.2)
     sprockets (3.7.1)
@@ -354,7 +344,6 @@ GEM
     temple (0.8.0)
     thor (0.20.0)
     thread_safe (0.3.6)
-    thread_safe (0.3.6-java)
     tilt (2.0.8)
     tubesock (0.2.7)
       rack (>= 1.5.0)
@@ -367,7 +356,6 @@ GEM
       execjs (>= 0.3.0, < 3)
     unf (0.1.4)
       unf_ext
-    unf (0.1.4-java)
     unf_ext (0.0.7.5)
     unicode-display_width (1.3.0)
     web-console (3.3.0)
@@ -377,15 +365,14 @@ GEM
     websocket (1.2.5)
     websocket-driver (0.7.0)
       websocket-extensions (>= 0.1.0)
-    websocket-driver (0.7.0-java)
-      websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.3)
+    whenever (0.10.0)
+      chronic (>= 0.6.3)
     will_paginate (3.1.6)
     xpath (3.0.0)
       nokogiri (~> 1.8)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES
@@ -404,18 +391,19 @@ DEPENDENCIES
   capistrano-upload-config
   capistrano3-puma
   capybara
+  capybara-selenium
   carrierwave
   codeclimate-test-reporter
-  coffee-rails
   concurrent-ruby
   concurrent-ruby-ext
-  d3-rails
+  d3-rails (~> 4.0)
   database_cleaner
   docker-api
   eventmachine (= 1.0.9.1)
   factory_bot_rails
   faye-websocket
   forgery
+  headless
   highline
   ims-lti (= 1.1.10)
   jbuilder
@@ -444,9 +432,8 @@ DEPENDENCIES
   rubyzip
   sass-rails
   sdoc
-  selenium-webdriver
   simplecov
-  slim
+  slim-rails
   sorcery
   spring
   thread_safe
@@ -454,6 +441,7 @@ DEPENDENCIES
   turbolinks (< 5.0.0)
   uglifier
   web-console
+  whenever
   will_paginate
 
 BUNDLED WITH

Vagrantfile

@@ -8,5 +8,5 @@ Vagrant.configure(2) do |config|
   end
   config.vm.network "private_network", ip: "192.168.59.104"
   # config.vm.synced_folder "../data", "/vagrant_data"
-  config.vm.provision "shell", path: "provision.sh"
+  config.vm.provision "shell", path: "provision.sh", privileged: false
 end

app/assets/javascripts/bootstrap-dropdown-submenu.js

@@ -0,0 +1,27 @@
+$(document).ready(function () {
+
+  var subMenusSelector = 'ul.dropdown-menu [data-toggle=dropdown]';
+
+  function openSubMenu(event) {
+    if (this.pathname === '/') {
+      event.preventDefault();
+    }
+    event.stopPropagation();
+
+    $(subMenusSelector).parent().removeClass('open');
+    $(this).parent().addClass('open');
+
+    var menu = $(this).parent().find("ul");
+    var menupos = menu.offset();
+
+    var newPos;
+    if ((menupos.left + menu.width()) + 30 > $(window).width()) {
+      newPos = -menu.width();
+    } else {
+      newPos = $(this).parent().width();
+    }
+    menu.css({left: newPos});
+  }
+
+  $(subMenusSelector).on('click', openSubMenu).on('mouseenter', openSubMenu);
+});

app/assets/javascripts/dashboard.js

@@ -23,10 +23,8 @@ $(function() {
     groups = new vis.DataSet(buildChartGroups());
     graph = new vis.Graph2d(document.getElementById('graph'), dataset, groups, {
       dataAxis: {
-        customRange: {
+        left: {
-          left: {
+          range: {min: 0}
-            min: 0
-          }
         },
         showMinorLabels: false
       },

app/assets/javascripts/editor/editor.js.erb

@@ -476,6 +476,7 @@ configureEditors: function () {
     this.clearOutput();
     $('#hint').fadeOut();
     $('#flowrHint').fadeOut();
+    this.clearHints();
     this.showOutputBar();
   },
 
@@ -512,6 +513,30 @@ configureEditors: function () {
     }
   },
 
+  clearHints: function() {
+    var container = $('#error-hints');
+    container.find('ul.body > li.hint').remove();
+    container.fadeOut();
+  },
+
+  showHint: function(message) {
+    var template = function(description, hint) {
+      return '\
+           <li class="hint">\
+             <div class="description">\
+               ' + description + '\
+             </div>\
+             <div class="hint">\
+               ' + hint + '\
+             </div>\
+           </li>\
+         '
+    };
+    var container = $('#error-hints');
+    container.find('ul.body').append(template(message.description, message.hint));
+    container.fadeIn();
+  },
+
   showContainerDepletedMessage: function() {
     $.flash.danger({
       icon: ['fa', 'fa-clock-o'],
@@ -527,6 +552,10 @@ configureEditors: function () {
   },
 
   showWebsocketError: function() {
+    if (window.navigator.userAgent.indexOf('Edge') > -1) {
+      // Mute errors in Microsoft Edge
+      return;
+    }
     $.flash.danger({
       text: $('#flash').data('message-failure')
     });
@@ -692,4 +721,4 @@ configureEditors: function () {
     // create autosave when the editor is opened the first time
     this.autosave();
   }
-};
+};

app/assets/javascripts/editor/execution.js.erb

@@ -4,10 +4,11 @@ CodeOceanEditorWebsocket = {
   createSocketUrl: function(url) {
       var sockURL = new URL(window.location);
       sockURL.pathname = url;
-      sockURL.protocol = '<%= DockerClient.config['ws_client_protocol'] %>';
+      // sanitize socket protocol string, strip trailing slash and other malicious chars if they are there
+      sockURL.protocol = '<%= DockerClient.config['ws_client_protocol']&.match(/(\w+):*\/*/)&.to_a&.at(1) %>:';
 
       // strip anchor if it is in the url
-      sockURL.hash = ''
+      sockURL.hash = '';
 
       return sockURL.toString();
   },
@@ -30,6 +31,7 @@ CodeOceanEditorWebsocket = {
   initializeSocketForScoring: function(url) {
     this.initializeSocket(url);
     this.websocket.on('default',this.handleScoringResponse.bind(this));
+    this.websocket.on('hint', this.showHint.bind(this));
     this.websocket.on('exit', this.handleExitCommand.bind(this));
   },
 
@@ -43,6 +45,7 @@ CodeOceanEditorWebsocket = {
     this.websocket.on('exit', this.handleExitCommand.bind(this));
     this.websocket.on('timeout', this.showTimeoutMessage.bind(this));
     this.websocket.on('status', this.showStatus.bind(this));
+    this.websocket.on('hint', this.showHint.bind(this));
   },
 
   handleExitCommand: function() {
@@ -53,4 +56,4 @@ CodeOceanEditorWebsocket = {
     this.cleanUpTurtle();
     this.cleanUpUI();
   }
-};
+};

app/assets/javascripts/editor/websocket.js.erb

@@ -108,5 +108,5 @@ CommandSocket.prototype.flush = function() {
  */
 CommandSocket.prototype.killWebSocket = function() {
   this.websocket.flush();
-  this.websocket.close();
+  this.websocket.close(1000);
-};
+};

app/assets/javascripts/exercise_collections.js.erb

@@ -0,0 +1,102 @@
+$(function() {
+  if ($.isController('exercise_collections')) {
+    var data = $('#data').data('working-times');
+    var averageWorkingTimeValue = parseFloat($('#data').data('average-working-time'));
+
+    var margin = { top: 30, right: 40, bottom: 30, left: 50 },
+      width = 720 - margin.left - margin.right,
+      height = 500 - margin.top - margin.bottom;
+
+    var x = d3.scaleBand().range([0, width]);
+    var y = d3.scaleLinear().range([height, 0]);
+
+    var xAxis = d3.axisBottom(x);
+    var yAxisLeft = d3.axisLeft(y);
+
+    var tooltip = d3.select("#graph").append("div").attr("class", "exercise-id-tooltip");
+
+    var averageWorkingTime = d3.line()
+      .x(function (d) { return x(d.index) + x.bandwidth()/2; })
+      .y(function () { return y(averageWorkingTimeValue); });
+
+    var minWorkingTime = d3.line()
+      .x(function (d) { return x(d.index) + x.bandwidth()/2; })
+      .y(function () { return y(0.1*averageWorkingTimeValue); });
+
+    var maxWorkingTime = d3.line()
+      .x(function (d) { return x(d.index) + x.bandwidth()/2; })
+      .y(function () { return y(2*averageWorkingTimeValue); });
+
+    var svg = d3.select('#graph')
+      .append("svg")
+      .attr("width", width + margin.left + margin.right)
+      .attr("height", height + margin.top + margin.bottom)
+      .append("g")
+      .attr("transform",
+        "translate(" + margin.left + "," + margin.top + ")");
+
+    // Get the data
+    data = Object.keys(data).map(function (key, index) {
+      return {
+        index: index,
+        exercise_id: parseInt(key),
+        working_time: parseFloat(data[key])
+      };
+    });
+
+    // Scale the range of the data
+    x.domain(data.map(function (d) { return d.index; }));
+    y.domain([0, d3.max(data, function (d) { return d.working_time; })]);
+
+    // Add the X Axis
+    svg.append("g")
+      .attr("class", "x axis")
+      .attr("transform", "translate(0," + height + ")")
+      .call(xAxis);
+
+    // Add the Y Axis
+    svg.append("g")
+      .attr("class", "y axis")
+      .style("fill", "steelblue")
+      .call(yAxisLeft);
+
+    // Draw the bars
+    svg.selectAll("bar")
+      .data(data)
+      .enter()
+      .append("rect")
+      .attr("class", "value-bar")
+      .on("mousemove", function (d){
+        tooltip
+          .style("left", d3.event.pageX - 50 + "px")
+          .style("top", d3.event.pageY + 50 + "px")
+          .style("display", "inline-block")
+          .html("<%= I18n.t('activerecord.models.exercise.one') %> ID: " + d.exercise_id + "<br>" +
+            "<%= I18n.t('exercises.statistics.average_worktime') %>: " + d.working_time + "s");
+      })
+      .on("mouseout", function (){ tooltip.style("display", "none");})
+      .on("click", function (d) {
+        window.location.href = "/exercises/" + d.exercise_id + "/statistics";
+      })
+      .attr("x", function (d) { return x(d.index); })
+      .attr("width", x.bandwidth())
+      .attr("y", function (d) { return y(d.working_time); })
+      .attr("height", function (d) { return height - y(d.working_time); });
+
+    // Add the average working time path
+    svg.append("path")
+      .datum(data)
+      .attr("class", "line average-working-time")
+      .attr("d", averageWorkingTime);
+
+    // Add the anomaly paths (min/max average exercise working time)
+    svg.append("path")
+      .datum(data)
+      .attr("class", "line minimum-working-time")
+      .attr("d", minWorkingTime);
+    svg.append("path")
+      .datum(data)
+      .attr("class", "line maximum-working-time")
+      .attr("d", maxWorkingTime);
+  }
+});

app/assets/javascripts/exercise_graphs.js

@@ -1,9 +1,7 @@
 $(function() {
-    // http://localhost:3333/exercises/38/statistics good for testing
+    // /exercises/38/statistics good for testing
-    // originally at--> localhost:3333/exercises/69/statistics
 
     if ($.isController('exercises') && $('.graph-functions-2').isPresent()) {
-        // GET THE DATA
         var submissions = $('#data').data('submissions');
         var submissions_length = submissions.length;
 
@@ -14,10 +12,7 @@ $(function() {
         submissionsAutosaves = [];
         var maximumValue = 0;
 
-        var wtimes = $('#wtimes').data('working_times'); //.hidden#wtimes data-working_times=ActiveSupport::JSON.encode(working_times_until)
+        var wtimes = $('#wtimes').data('working_times');
-        
-        // console.log(submissions);
-        // console.log(wtimes);
 
         for (var i = 0;i<submissions_length;i++){
             var submission = submissions[i];
@@ -46,9 +41,6 @@ $(function() {
                 submissionsSaves.push(submissionArray[1]);
             }
         }
-        // console.log(submissionsScoreAndTimeAssess.length);
-        // console.log(submissionsScoreAndTimeSubmits);
-        // console.log(submissionsScoreAndTimeRuns);
 
         function get_minutes (time_stamp) {
             try {
@@ -94,33 +86,22 @@ $(function() {
                 height = (width * height_ratio) - margin.top - margin.bottom;
 
             // Set the ranges
-            var x = d3.scale.linear().range([0, width]);
+            var x = d3.scaleLinear().range([0, width]);
-            var y = d3.scale.linear().range([height,0]);
+            var y = d3.scaleLinear().range([height,0]);
 
-            //var x = d3.scale.linear()
+            //var x = d3.scaleLinear()
             //    .range([0, width]);
-            //var y = d3.scale.linear()
+            //var y = d3.scaleLinear()
             //    .range([0,height]); // - (height/20
 
-            var xAxis = d3.svg.axis()
+            var xAxis = d3.axisBottom(x).ticks(20);
-                .scale(x)
+            var yAxis = d3.axisLeft()
-                .orient("bottom")
+                .scale(d3.scaleLinear().domain([0,maximumValue]).range([height,0]))
-                .ticks(20);
-
-
-            var yAxis = d3.svg.axis()
-                .scale(d3.scale.linear().domain([0,maximumValue]).range([height,0]))//y
-                // .scale(y)
-                .orient("left")
                 .ticks(maximumValue)
-                .innerTickSize(-width)
+                .tickSizeInner(-width)
-                .outerTickSize(0);
+                .tickSizeOuter(0);
 
-            //var line = d3.svg.line()
+            var line = d3.line()
-            //    .x(function(d) { return x(d.date); })
-            //    .y(function(d) { return y(d.close); });
-
-            var line = d3.svg.line()
                 .x(function (d) {
                     // console.log(d[1]);
                     return x(d[1]);
@@ -288,23 +269,12 @@ $(function() {
                         .text(color_hash[String(i)][0]);
 
                 });
-
-
-
-            // function type(d) {
-            //     d.frequency = +d.frequency;
-            //     return d;
-            // }
-
-            //.on("mousemove", mMove)//new again
-            //.append("title");
-
         }
 
         try{
             graph_assesses();
         } catch(err){
-            alert("could not draw the graph");
+            console.error("Could not draw the graph", err);
         }
 
   }

app/assets/javascripts/statistics_activity_history.js

@@ -0,0 +1,87 @@
+$(document).ready(function () {
+
+  function manageActivityHistory(prefix) {
+    var containerId = prefix + '-activity-history';
+
+    if ($('.graph#' + containerId).isPresent()) {
+
+      var chartData;
+      var dataset;
+      var graph;
+      var groups;
+
+      var buildChartGroups = function () {
+        return _.map(chartData, function (element) {
+          return {
+            content: element.name,
+            id: element.key,
+            visible: true,
+            options: {
+              interpolation: false,
+              yAxisOrientation: element.axis ? element.axis : 'left'
+            }
+          };
+        });
+      };
+
+      var initializeChart = function () {
+        dataset = new vis.DataSet();
+        groups = new vis.DataSet(buildChartGroups());
+        graph = new vis.Graph2d(document.getElementById(containerId), dataset, groups, {
+          dataAxis: {
+            left: {
+              range: {min: 0}
+            },
+            right: {
+              range: {min: 0}
+            },
+            showMinorLabels: true,
+            alignZeros: true
+          },
+          drawPoints: {
+            style: 'circle'
+          },
+          legend: true,
+          start: $('#from-date')[0].value || 0,
+          end: $('#to-date')[0].value || 0
+        });
+      };
+
+      var refreshData = function (callback) {
+        var params = new URLSearchParams(window.location.search.slice(1));
+        var jqxhr = $.ajax(prefix + '-activity-history.json', {
+          dataType: 'json',
+          data: {from: params.get('from'), to: params.get('to'), interval: params.get('interval')},
+          method: 'GET'
+        });
+        jqxhr.done(function (response) {
+          (callback || _.noop)(response);
+          updateChartData(response);
+        });
+      };
+
+      var updateChartData = function (response) {
+        _.each(response, function (group) {
+          _.each(group.data, function (data) {
+            dataset.add({
+              group: group.key,
+              x: data.key,
+              y: data.value
+            });
+          });
+        });
+      };
+
+      refreshData(function (data) {
+        chartData = data;
+        $('#' + containerId).parent().find('.spinner').hide();
+        initializeChart();
+      });
+    }
+  }
+
+  if ($.isController('statistics')) {
+    manageActivityHistory('rfc');
+    manageActivityHistory('user');
+  }
+});

app/assets/javascripts/statistics_graphs.js

@@ -0,0 +1,107 @@
+$(document).ready(function () {
+  if ($.isController('statistics') && $('.graph#user-activity').isPresent()) {
+
+    function manageGraph(containerId, url, refreshAfter) {
+      var CHART_START = window.vis ? vis.moment().add(-1, 'minute') : undefined;
+      var DEFAULT_REFRESH_INTERVAL = refreshAfter * 1000 || 10000;
+
+      var refreshInterval;
+
+      var initialData;
+      var dataset;
+      var graph;
+      var groups;
+
+      var buildChartGroups = function() {
+        return _.map(initialData, function(element) {
+          return {
+            content: element.name + (element.unit ? ' [' + element.unit + ']' : ''),
+            id: element.key,
+            visible: false,
+            options: {
+              yAxisOrientation: element.axis ? element.axis : 'left'
+            }
+          };
+        });
+      };
+
+      var initializeChart = function() {
+        dataset = new vis.DataSet();
+        groups = new vis.DataSet(buildChartGroups());
+        graph = new vis.Graph2d(document.getElementById(containerId), dataset, groups, {
+          dataAxis: {
+            left: {
+              range: {min: 0}
+            },
+            right: {
+              range: {min: 0}
+            },
+            showMinorLabels: true,
+            alignZeros: true
+          },
+          drawPoints: {
+            style: 'circle'
+          },
+          end: vis.moment(),
+          legend: true,
+          start: CHART_START
+        });
+      };
+
+      var refreshChart = function() {
+        var now = vis.moment();
+        var window = graph.getWindow();
+        var interval = window.end - window.start;
+        graph.setWindow(now - interval, now);
+      };
+
+      var refreshData = function(callback) {
+        if (! ($.isController('statistics') && $('#' + containerId).isPresent())) {
+          clearInterval(refreshInterval);
+        } else {
+          var jqxhr = $.ajax(url, {
+            dataType: 'json',
+            method: 'GET'
+          });
+          jqxhr.done(function(response) {
+            (callback || _.noop)(response);
+            setGroupVisibility(response);
+            updateChartData(response);
+            requestAnimationFrame(refreshChart);
+          });
+        }
+      };
+
+      var setGroupVisibility = function(response) {
+        _.each(response, function(data) {
+          groups.update({
+            id: data.key,
+            visible: true
+          });
+        });
+      };
+
+      var updateChartData = function(response) {
+        _.each(response, function(data) {
+          dataset.add({
+            group: data.key,
+            x: vis.moment(),
+            y: data.data
+          });
+        });
+      };
+
+      refreshData(function (data) {
+        initialData = data;
+        $('#' + containerId).parent().find('.spinner').hide();
+        initializeChart();
+
+        var refresh_interval = location.search.match(/interval=(\d+)/) ? parseInt(RegExp.$1) : DEFAULT_REFRESH_INTERVAL;
+        refreshInterval = setInterval(refreshData, refresh_interval);
+      });
+    }
+
+    manageGraph('user-activity', 'graphs/user-activity', 10);
+    manageGraph('rfc-activity', 'graphs/rfc-activity', 30);
+  }
+});

app/assets/javascripts/working_time_graphs.js

@@ -38,18 +38,6 @@ $(function() {
           }
       }
 
-      // minutes_count[(maximum_minutes + 1)] = 0;
-      //$('.graph-functions').html("<p></p>")
-
-      // var minutes_count = new Array(10);
-      // var minutes_array_len = minutes_array.length;
-      // for (var i=0; i< minutes_count; i++){
-      //
-      //     for (var j = 0; j < minutes_array_len; j++){
-      //         if ()
-      //     }
-      // }
-
       function getWidth() {
           if (self.innerHeight) {
               return self.innerWidth;
@@ -81,22 +69,17 @@ $(function() {
 
           //var formatDate = d3.time.format("%M");
 
-          var x = d3.scale.linear()
+          var x = d3.scaleLinear()
               .range([0, width]);
-          var y = d3.scale.linear()
+          var y = d3.scaleLinear()
               .range([height, 0]); // - (height/20
-          var xAxis = d3.svg.axis()
+          var xAxis = d3.axisBottom(x).ticks(20);
-              .scale(x)
+          var yAxis = d3.axisLeft(y)
-              .orient("bottom")
-              .ticks(20);
-          var yAxis = d3.svg.axis()
-              .scale(y)
-              .orient("left")
               .ticks(20)
-              .innerTickSize(-width)
+              .tickSizeInner(-width)
-              .outerTickSize(0);
+              .tickSizeOuter(0);
 
-          var line = d3.svg.line()
+          var line = d3.line()
               .x(function (d, i) {
                   return x(i);
               })
@@ -225,7 +208,7 @@ $(function() {
           var x = d3.scale.ordinal()
               .rangeRoundBands([0, width], .1);
 
-          var y = d3.scale.linear()
+          var y = d3.scaleLinear()
               .range([0,height-(margin.top + margin.bottom)]);
 
 
@@ -236,7 +219,7 @@ $(function() {
 
 
           var yAxis = d3.svg.axis()
-              .scale(d3.scale.linear().domain([0,max_of_array]).range([height,0]))//y
+              .scale(d3.scaleLinear().domain([0,max_of_array]).range([height,0]))//y
               .orient("left")
               .ticks(10)
               .innerTickSize(-width);
@@ -299,7 +282,7 @@ $(function() {
               .text("Working Time (Minutes)")
               .style('font-size', 14);
 
-          y = d3.scale.linear()
+          y = d3.scaleLinear()
               .domain([(0),max_of_array])
               .range([0,height]);
 

app/assets/stylesheets/bootstrap-dropdown-submenu.css.scss

@@ -0,0 +1,38 @@
+.dropdown-submenu {
+  position: relative;
+}
+
+.dropdown-submenu > .dropdown-menu {
+  top: 0;
+  left: 100%;
+}
+
+.dropdown-submenu > a:after {
+  display: block;
+  content: " ";
+  float: right;
+  width: 0;
+  height: 0;
+  border-color: transparent;
+  border-style: solid;
+  border-width: 5px 0 5px 5px;
+  border-left-color: #cccccc;
+  margin-top: 5px;
+  margin-right: -10px;
+}
+
+.dropdown-submenu:hover > a:after {
+  border-left-color: #ffffff;
+}
+
+.dropdown-submenu.pull-left {
+  float: none;
+}
+
+.dropdown-submenu.pull-left > .dropdown-menu {
+  left: -100%;
+  margin-left: 10px;
+  -webkit-border-radius: 6px 0 6px 6px;
+  -moz-border-radius: 6px 0 6px 6px;
+  border-radius: 6px 0 6px 6px;
+}

app/assets/stylesheets/editor.css.scss

@@ -193,3 +193,24 @@ button i.fa-spin {
 .enforce-bottom-margin {
   margin-bottom: 5px !important;
 }
+
+#error-hints {
+  display: none;
+  background-color: #FAFAFA;
+
+  .heading {
+    font-weight: bold;
+    font-size: larger;
+  }
+
+  ul.body {
+
+    li.hint {
+      .description {
+        font-style: italic;
+      }
+
+      .hint {}
+    }
+  }
+}

app/assets/stylesheets/exercise_collections.scss

@@ -0,0 +1,69 @@
+$time-color: #008cba;
+$min-color: #8efa00;
+$avg-color: #ffca00;
+$max-color: #ff2600;
+
+path.line.minimum-working-time {
+  stroke: $min-color;
+}
+
+path.line.average-working-time {
+  stroke: $avg-color;
+}
+
+path.line.maximum-working-time {
+  stroke: $max-color;
+}
+
+rect.value-bar {
+  fill: $time-color;
+  cursor: pointer;
+}
+
+#legend {
+  display: flex;
+  margin-top: 20px;
+
+  .legend-entry {
+    flex-grow: 1;
+    display: flex;
+
+    .box {
+      width: 20px;
+      height: 20px;
+      border: solid 1px #000;
+    }
+
+    .box.time {
+      background-color: $time-color;
+    }
+
+    .box.min {
+      background-color: $min-color;
+    }
+
+    .box.avg {
+      background-color: $avg-color;
+    }
+
+    .box.max {
+      background-color: $max-color;
+    }
+
+    .box-label {
+      margin-left: 5px;
+      margin-right: 15px;
+    }
+  }
+}
+
+.exercise-id-tooltip {
+  position: absolute;
+  display: none;
+  min-width: 80px;
+  height: auto;
+  background: none repeat scroll 0 0 #ffffff;
+  border: 1px solid #008cba;
+  padding: 14px;
+  text-align: center;
+}

app/assets/stylesheets/request-for-comments.css.scss

@@ -53,64 +53,72 @@
     }
   }
 
-}
+  .testrun-assess-results {
 
-.testrun-assess-results {
+    .testrun-container {
+      display: flex;
+      margin-bottom: 10px;
 
-  display: flex;
+      .testrun-output {
+        overflow-x: auto;
+        flex-grow: 1;
+      }
+    }
+
+    .result {
+      margin-right: 10px;
+      margin-top: 20px;
+      width: 10px;
+      height: 10px;
+    }
+
+    .passed {
+      border-radius: 50%;
+      background-color: #8efa00;
+      -webkit-box-shadow: 0 0 11px 1px rgba(44,222,0,1);
+      -moz-box-shadow: 0 0 11px 1px rgba(44,222,0,1);
+      box-shadow: 0 0 11px 1px rgba(44,222,0,1);
+    }
+
+    .unknown {
+      border-radius: 50%;
+      background-color: #ffca00;
+      -webkit-box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+      -moz-box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+      box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+    }
+
+    .failed {
+      border-radius: 50%;
+      background-color: #ff2600;
+      -webkit-box-shadow: 0 0 11px 1px rgba(222,0,0,1);
+      -moz-box-shadow: 0 0 11px 1px rgba(222,0,0,1);
+      box-shadow: 0 0 11px 1px rgba(222,0,0,1);
+    }
 
-  .result {
-    margin-right: 10px;
-    width: 10px;
-    height: 10px;
   }
 
-  .passed {
+  #mark-as-solved-button {
-    border-radius: 50%;
+    margin-top: 20px;
-    background-color: #8efa00;
-    -webkit-box-shadow: 0 0 11px 1px rgba(44,222,0,1);
-    -moz-box-shadow: 0 0 11px 1px rgba(44,222,0,1);
-    box-shadow: 0 0 11px 1px rgba(44,222,0,1);
   }
 
-  .unknown {
+  #thank-you-container {
-    border-radius: 50%;
+    display: none;
-    background-color: #ffca00;
+    margin-top: 20px;
-    -webkit-box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+    padding: 5px;
-    -moz-box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+    border: solid lightgrey 1px;
-    box-shadow: 0 0 11px 1px rgb(255, 202, 0);
+    background-color: rgba(20, 180, 20, 0.2);
+    border-radius: 4px;
+
+    button {
+      margin-right: 10px;
+    }
   }
 
-  .failed {
+  #thank-you-note {
-    border-radius: 50%;
+    width: 100%;
-    background-color: #ff2600;
+    height: 200px;
-    -webkit-box-shadow: 0 0 11px 1px rgba(222,0,0,1);
-    -moz-box-shadow: 0 0 11px 1px rgba(222,0,0,1);
-    box-shadow: 0 0 11px 1px rgba(222,0,0,1);
   }
-
-}
-
-#mark-as-solved-button {
-  margin-top: 20px;
-}
-
-#thank-you-container {
-  display: none;
-  margin-top: 20px;
-  padding: 5px;
-  border: solid lightgrey 1px;
-  background-color: rgba(20, 180, 20, 0.2);
-  border-radius: 4px;
-
-  button {
-    margin-right: 10px;
-  }
-}
-
-#thank-you-note {
-  width: 100%;
-  height: 200px;
 }
 
 #commentitor {

app/assets/stylesheets/statistics.css.scss

@@ -58,3 +58,96 @@ div.negative-result {
   box-shadow: 0px 0px 11px 1px rgba(222,0,0,1);
 }
 
+tr.highlight {
+  border-top: 2px solid rgba(222,0,0,1);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// StatisticsController:
+
+#statistics-container {
+  margin-bottom: 40px;
+}
+
+.statistics-wrapper {
+  display: grid;
+  grid-template-columns: repeat(4, 1fr);
+  grid-auto-rows: 150px;
+  grid-gap: 10px;
+
+  > a {
+    color: #fff;
+    text-decoration: none;
+
+    > div {
+      border: 2px solid #0055ba;
+      border-radius: 5px;
+      background-color: #008cba;
+      padding: 1em;
+      display: flex;
+      flex-flow: column-reverse;
+      text-align: center;
+
+      > .data {
+        flex-grow: 1;
+        font-size: 40px;
+        vertical-align: middle;
+        line-height: 50px;
+
+        > .unit {
+          font-size: 20px;
+        }
+      }
+
+      > .title {
+        height: 42px;
+      }
+    }
+  }
+}
+
+.group {
+  .title {
+    display: flex;
+    align-items: baseline;
+
+    h1 {
+      flex-grow: 1;
+    }
+
+    h2 {
+      font-size: medium;
+    }
+  }
+}
+
+.spinner {
+  width: 40px;
+  height: 40px;
+  background-color: #333;
+
+  margin: 100px auto;
+  -webkit-animation: sk-rotateplane 1.2s infinite ease-in-out;
+  animation: sk-rotateplane 1.2s infinite ease-in-out;
+}
+
+@-webkit-keyframes sk-rotateplane {
+  0% { -webkit-transform: perspective(120px) }
+  50% { -webkit-transform: perspective(120px) rotateY(180deg) }
+  100% { -webkit-transform: perspective(120px) rotateY(180deg)  rotateX(180deg) }
+}
+
+@keyframes sk-rotateplane {
+  0% {
+    transform: perspective(120px) rotateX(0deg) rotateY(0deg);
+    -webkit-transform: perspective(120px) rotateX(0deg) rotateY(0deg)
+  }
+  50% {
+    transform: perspective(120px) rotateX(-180.1deg) rotateY(0deg);
+    -webkit-transform: perspective(120px) rotateX(-180.1deg) rotateY(0deg)
+  }
+  100% {
+    transform: perspective(120px) rotateX(-180deg) rotateY(-179.9deg);
+    -webkit-transform: perspective(120px) rotateX(-180deg) rotateY(-179.9deg);
+  }
+}

app/controllers/concerns/submission_scoring.rb

@@ -13,7 +13,7 @@ module SubmissionScoring
           submission.exercise.execution_environment.error_templates.each do |template|
             pattern = Regexp.new(template.signature).freeze
             if pattern.match(testrun_output)
-              StructuredError.create_from_template(template, testrun_output)
+              StructuredError.create_from_template(template, testrun_output, submission)
             end
           end
         end

app/controllers/exercise_collections_controller.rb

@@ -1,7 +1,7 @@
 class ExerciseCollectionsController < ApplicationController
   include CommonBehavior
 
-  before_action :set_exercise_collection, only: [:show, :edit, :update, :destroy]
+  before_action :set_exercise_collection, only: [:show, :edit, :update, :destroy, :statistics]
 
   def index
     @exercise_collections = ExerciseCollection.all.paginate(:page => params[:page])
@@ -9,6 +9,7 @@ class ExerciseCollectionsController < ApplicationController
   end
 
   def show
+    @exercises = @exercise_collection.exercises.paginate(:page => params[:page])
   end
 
   def new
@@ -34,6 +35,9 @@ class ExerciseCollectionsController < ApplicationController
     update_and_respond(object: @exercise_collection, params: exercise_collection_params)
   end
 
+  def statistics
+  end
+
   private
 
   def set_exercise_collection
@@ -46,6 +50,6 @@ class ExerciseCollectionsController < ApplicationController
   end
 
   def exercise_collection_params
-    params[:exercise_collection].permit(:name, :exercise_ids => [])
+    params[:exercise_collection].permit(:name, :use_anomaly_detection, :user_id, :user_type, :exercise_ids => []).merge(user_type: InternalUser.name)
   end
 end

app/controllers/exercises_controller.rb

@@ -346,6 +346,16 @@ class ExercisesController < ApplicationController
 
   def statistics
     if(@external_user)
+      @submissions = Submission.where("user_id = ?  AND exercise_id = ?", @external_user.id, @exercise.id).order("created_at")
+      @submissions_and_interventions = (@submissions + UserExerciseIntervention.where("user_id = ?  AND exercise_id = ?", @external_user.id, @exercise.id)).sort_by { |a| a.created_at }
+      deltas = @submissions.map.with_index do |item, index|
+        delta = item.created_at - @submissions[index - 1].created_at if index > 0
+        if delta == nil or delta > 10 * 60 then 0 else delta end
+      end
+      @working_times_until = []
+      @submissions_and_interventions.each_with_index do |submission, index|
+        @working_times_until.push((Time.at(deltas[1..index].inject(:+)).utc.strftime("%H:%M:%S") if index > 0))
+      end
       render 'exercises/external_users/statistics'
     else
       user_statistics = {}

app/controllers/statistics_controller.rb

@@ -0,0 +1,59 @@
+class StatisticsController < ApplicationController
+  include StatisticsHelper
+
+  before_action :authorize!, only: [:show, :graphs, :user_activity, :user_activity_history, :rfc_activity,
+                                    :rfc_activity_history]
+
+  def policy_class
+    StatisticsPolicy
+  end
+
+  def show
+    respond_to do |format|
+      format.html
+      format.json { render(json: statistics_data) }
+    end
+  end
+
+  def graphs
+  end
+
+  def user_activity
+    respond_to do |format|
+      format.json { render(json: user_activity_live_data) }
+    end
+  end
+
+  def user_activity_history
+    respond_to do |format|
+      format.html { render('activity_history', locals: {resource: :user}) }
+      format.json { render_ranged_data :ranged_user_data}
+    end
+  end
+
+  def rfc_activity
+    respond_to do |format|
+      format.json { render(json: rfc_activity_data) }
+    end
+  end
+
+  def rfc_activity_history
+    respond_to do |format|
+      format.html { render('activity_history', locals: {resource: :rfc}) }
+      format.json { render_ranged_data :ranged_rfc_data }
+    end
+  end
+
+  def render_ranged_data(data_source)
+    interval = params[:interval].to_s.empty? ? 'year' : params[:interval]
+    from = DateTime.strptime(params[:from], '%Y-%m-%d') rescue DateTime.new(0)
+    to = DateTime.strptime(params[:to], '%Y-%m-%d') rescue DateTime.now
+    render(json: self.send(data_source, interval, from, to))
+  end
+
+  def authorize!
+    authorize self
+  end
+  private :authorize!
+
+end

app/controllers/submissions_controller.rb

@@ -197,7 +197,8 @@ class SubmissionsController < ApplicationController
 
   def kill_socket(tubesock)
     # search for errors and save them as StructuredError (for scoring runs see submission_scoring.rb)
-    extract_errors
+    errors = extract_errors
+    send_hints(tubesock, errors)
 
     # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb)
     save_run_output
@@ -284,14 +285,16 @@ class SubmissionsController < ApplicationController
   end
 
   def extract_errors
+    results = []
     unless @raw_output.blank?
       @submission.exercise.execution_environment.error_templates.each do |template|
         pattern = Regexp.new(template.signature).freeze
         if pattern.match(@raw_output)
-          StructuredError.create_from_template(template, @raw_output, @submission)
+          results << StructuredError.create_from_template(template, @raw_output, @submission)
         end
       end
     end
+    results
   end
 
   def score
@@ -303,11 +306,22 @@ class SubmissionsController < ApplicationController
       # to ensure responsiveness, we therefore open a thread here.
       Thread.new {
         tubesock.send_data JSON.dump(score_submission(@submission))
+
+        # To enable hints when scoring a submission, uncomment the next line:
+        #send_hints(tubesock, StructuredError.where(submission: @submission))
+
         tubesock.send_data JSON.dump({'cmd' => 'exit'})
       }
     end
   end
 
+  def send_hints(tubesock, errors)
+    errors = errors.to_a.uniq { |e| e.hint}
+    errors.each do | error |
+      tubesock.send_data JSON.dump({cmd: 'hint', hint: error.hint, description: error.error_template.description})
+    end
+  end
+
   def set_docker_client
     @docker_client = DockerClient.new(execution_environment: @submission.execution_environment)
   end

app/controllers/user_exercise_feedbacks_controller.rb

@@ -69,7 +69,8 @@ class UserExerciseFeedbacksController < ApplicationController
     @texts = comment_presets.to_a
     @times = time_presets.to_a
     @uef = UserExerciseFeedback.new
-    @exercise = Exercise.find(params[:user_exercise_feedback][:exercise_id])
+    exercise_id = if params[:user_exercise_feedback].nil? then params[:exercise_id] else params[:user_exercise_feedback][:exercise_id] end
+    @exercise = Exercise.find(exercise_id)
     authorize!
   end
 

app/helpers/statistics_helper.rb

@@ -0,0 +1,211 @@
+module StatisticsHelper
+
+  def statistics_data
+    [
+        {
+            key: 'users',
+            name: t('statistics.sections.users'),
+            entries: user_statistics
+        },
+        {
+            key: 'exercises',
+            name: t('statistics.sections.exercises'),
+            entries: exercise_statistics
+        },
+        {
+            key: 'request_for_comments',
+            name: t('statistics.sections.request_for_comments'),
+            entries: rfc_statistics
+        }
+    ]
+  end
+
+  def user_statistics
+    [
+        {
+            key: 'internal_users',
+            name: t('activerecord.models.internal_user.other'),
+            data: InternalUser.count,
+            url: internal_users_path
+        },
+        {
+            key: 'external_users',
+            name: t('activerecord.models.external_user.other'),
+            data: ExternalUser.count,
+            url: external_users_path
+        },
+        {
+            key: 'currently_active',
+            name: t('statistics.entries.users.currently_active'),
+            data: ExternalUser.joins(:submissions)
+                      .where(['submissions.created_at >= ?', DateTime.now - 5.minutes])
+                      .distinct('external_users.id').count,
+            url: 'statistics/graphs'
+        }
+    ]
+  end
+
+  def exercise_statistics
+    [
+        {
+            key: 'exercises',
+            name: t('activerecord.models.exercise.other'),
+            data: Exercise.count,
+            url: exercises_path
+        },
+        {
+            key: 'average_submissions',
+            name: t('statistics.entries.exercises.average_number_of_submissions'),
+            data: (Submission.count.to_f / Exercise.count).round(2)
+        },
+        {
+            key: 'submissions_per_minute',
+            name: t('statistics.entries.exercises.submissions_per_minute'),
+            data: (Submission.where('created_at >= ?', DateTime.now - 1.hours).count.to_f / 60).round(2),
+            unit: '/min',
+            url: statistics_graphs_path
+        },
+        {
+            key: 'execution_environments',
+            name: t('activerecord.models.execution_environment.other'),
+            data: ExecutionEnvironment.count,
+            url: execution_environments_path
+        },
+        {
+            key: 'exercise_collections',
+            name: t('activerecord.models.exercise_collection.other'),
+            data: ExerciseCollection.count,
+            url: exercise_collections_path
+        }
+    ]
+  end
+
+  def rfc_statistics
+    rfc_activity_data + [
+        {
+            key: 'comments',
+            name: t('activerecord.models.comment.other'),
+            data: Comment.count
+        }
+    ]
+  end
+
+  def user_activity_live_data
+    [
+        {
+            key: 'active_in_last_hour',
+            name: t('statistics.entries.users.currently_active'),
+            data: ExternalUser.joins(:submissions)
+                      .where(['submissions.created_at >= ?', DateTime.now - 5.minutes])
+                      .distinct('external_users.id').count,
+        },
+        {
+            key: 'submissions_per_minute',
+            name: t('statistics.entries.exercises.submissions_per_minute'),
+            data: (Submission.where('created_at >= ?', DateTime.now - 1.hours).count.to_f / 60).round(2),
+            unit: '/min',
+            axis: 'right'
+        }
+    ]
+  end
+
+  def rfc_activity_data(from=DateTime.new(0), to=DateTime.now)
+    [
+        {
+            key: 'rfcs',
+            name: t('activerecord.models.request_for_comment.other'),
+            data: RequestForComment.in_range(from, to).count,
+            url: request_for_comments_path
+        },
+        {
+            key: 'percent_solved',
+            name: t('statistics.entries.request_for_comments.percent_solved'),
+            data: (100.0 / RequestForComment.in_range(from, to).count * RequestForComment.in_range(from, to).where(solved: true).count).round(1),
+            unit: '%',
+            axis: 'right',
+            url: statistics_graphs_path
+        },
+        {
+            key: 'percent_soft_solved',
+            name: t('statistics.entries.request_for_comments.percent_soft_solved'),
+            data: (100.0 / RequestForComment.in_range(from, to).count * RequestForComment.in_range(from, to).unsolved.where(full_score_reached: true).count).round(1),
+            unit: '%',
+            axis: 'right',
+            url: statistics_graphs_path
+        },
+        {
+            key: 'percent_unsolved',
+            name: t('statistics.entries.request_for_comments.percent_unsolved'),
+            data: (100.0 / RequestForComment.in_range(from, to).count * RequestForComment.in_range(from, to).unsolved.count).round(1),
+            unit: '%',
+            axis: 'right',
+            url: statistics_graphs_path
+        },
+        {
+            key: 'rfcs_with_comments',
+            name: t('statistics.entries.request_for_comments.with_comments'),
+            data: RequestForComment.in_range(from, to).joins('join "submissions" s on s.id = request_for_comments.submission_id
+                join "files" f on f.context_id = s.id and f.context_type = \'Submission\'
+                join "comments" c on c.file_id = f.id').group('request_for_comments.id').count.size,
+            url: statistics_graphs_path
+        }
+    ]
+  end
+
+  def ranged_rfc_data(interval='year', from=DateTime.new(0), to=DateTime.now)
+    [
+        {
+            key: 'rfcs',
+            name: t('activerecord.models.request_for_comment.other'),
+            data: RequestForComment.in_range(from, to)
+                      .select("date_trunc('#{interval}', created_at) AS \"key\", count(id) AS \"value\"")
+                      .group('key').order('key')
+        },
+        {
+            key: 'rfcs_solved',
+            name: t('statistics.entries.request_for_comments.percent_solved'),
+            data: RequestForComment.in_range(from, to)
+                      .where(solved: true)
+                      .select("date_trunc('#{interval}', created_at) AS \"key\", count(id) AS \"value\"")
+                      .group('key').order('key')
+        },
+        {
+            key: 'rfcs_soft_solved',
+            name: t('statistics.entries.request_for_comments.percent_soft_solved'),
+            data: RequestForComment.in_range(from, to).unsolved
+                      .where(full_score_reached: true)
+                      .select("date_trunc('#{interval}', created_at) AS \"key\", count(id) AS \"value\"")
+                      .group('key').order('key')
+        },
+        {
+            key: 'rfcs_unsolved',
+            name: t('statistics.entries.request_for_comments.percent_unsolved'),
+            data: RequestForComment.in_range(from, to).unsolved
+                      .select("date_trunc('#{interval}', created_at) AS \"key\", count(id) AS \"value\"")
+                      .group('key').order('key')
+        }
+    ]
+  end
+
+  def ranged_user_data(interval='year', from=DateTime.new(0), to=DateTime.now)
+    [
+        {
+            key: 'active',
+            name: t('statistics.entries.users.active'),
+            data: ExternalUser.joins(:submissions)
+                      .where(submissions: {created_at: from..to})
+                      .select("date_trunc('#{interval}', submissions.created_at) AS \"key\", count(distinct external_users.id) AS \"value\"")
+                      .group('key').order('key')
+        },
+        {
+            key: 'submissions',
+            name: t('statistics.entries.exercises.submissions'),
+            data: Submission.where(created_at: from..to)
+                      .select("date_trunc('#{interval}', created_at) AS \"key\", count(id) AS \"value\"")
+                      .group('key').order('key'),
+            axis: 'right'
+        }
+    ]
+  end
+
+end

app/helpers/time_helper.rb

@@ -0,0 +1,12 @@
+module TimeHelper
+
+  # convert timestamps ('12:34:56.789') to seconds
+  def time_to_f(timestamp)
+    unless timestamp.nil?
+      timestamp = timestamp.split(':')
+      return timestamp[0].to_i * 60 * 60 + timestamp[1].to_i * 60 + timestamp[2].to_f
+    end
+    nil
+  end
+
+end

app/mailers/user_mailer.rb

@@ -37,4 +37,18 @@ class UserMailer < ActionMailer::Base
     @rfc_link = request_for_comment_url(request_for_comments)
     mail(subject: t('mailers.user_mailer.send_thank_you_note.subject', author: @author), to: receiver.email)
   end
+  
+  def exercise_anomaly_detected(exercise_collection, anomalies)
+    @receiver_displayname = exercise_collection.user.displayname
+    @collection = exercise_collection
+    @anomalies = anomalies
+    mail(subject: t('mailers.user_mailer.exercise_anomaly_detected.subject'), to: exercise_collection.user.email)
+  end
+
+  def exercise_anomaly_needs_feedback(user, exercise, link)
+    @receiver_displayname = user.displayname
+    @exercise_title = exercise.title
+    @link = link
+    mail(subject: t('mailers.user_mailer.exercise_anomaly_needs_feedback.subject'), to: user.email)
+  end
 end

app/models/anomaly_notification.rb

@@ -0,0 +1,5 @@
+class AnomalyNotification < ActiveRecord::Base
+  belongs_to :user, polymorphic: true
+  belongs_to :exercise
+  belongs_to :exercise_collection
+end

app/models/exercise.rb

@@ -36,6 +36,7 @@ class Exercise < ActiveRecord::Base
   validates :token, presence: true, uniqueness: true
 
   @working_time_statistics = nil
+  attr_reader :working_time_statistics
 
   MAX_EXERCISE_FEEDBACKS = 20
 
@@ -65,21 +66,27 @@ class Exercise < ActiveRecord::Base
   end
 
   def user_working_time_query
-    """
+    "
       SELECT user_id,
-             sum(working_time_new) AS working_time
+             user_type,
+             SUM(working_time_new) AS working_time,
+             MAX(score) AS score
       FROM
         (SELECT user_id,
+                user_type,
+                score,
                 CASE WHEN working_time >= '0:05:00' THEN '0' ELSE working_time END AS working_time_new
          FROM
             (SELECT user_id,
+                    user_type,
+                    score,
                     id,
                     (created_at - lag(created_at) over (PARTITION BY user_id, exercise_id
                                                         ORDER BY created_at)) AS working_time
             FROM submissions
             WHERE exercise_id=#{id}) AS foo) AS bar
-      GROUP BY user_id
+      GROUP BY user_id, user_type
-    """
+    "
   end
 
   def get_quantiles(quantiles)
@@ -202,7 +209,7 @@ class Exercise < ActiveRecord::Base
   def retrieve_working_time_statistics
     @working_time_statistics = {}
     self.class.connection.execute(user_working_time_query).each do |tuple|
-      @working_time_statistics[tuple["user_id"].to_i] = tuple
+      @working_time_statistics[tuple['user_id'].to_i] = tuple
     end
   end
 
@@ -345,7 +352,11 @@ class Exercise < ActiveRecord::Base
   end
 
   def has_user_solved(user)
-    return maximum_score(user).to_i == maximum_score.to_i
+    maximum_score(user).to_i == maximum_score.to_i
+  end
+
+  def finishers
+    ExternalUser.joins(:submissions).where(submissions: {exercise_id: id, score: maximum_score, cause: %w(submit assess)}).distinct
   end
 
   def set_default_values
@@ -368,4 +379,15 @@ class Exercise < ActiveRecord::Base
     user_exercise_feedbacks.size <= MAX_EXERCISE_FEEDBACKS
   end
 
+  def last_submission_per_user
+    Submission.joins("JOIN (
+          SELECT
+              user_id,
+              user_type,
+              first_value(id) OVER (PARTITION BY user_id ORDER BY created_at DESC) AS fv
+          FROM submissions
+          WHERE exercise_id = #{id}
+        ) AS t ON t.fv = submissions.id").distinct
+  end
+
 end

app/models/exercise_collection.rb

@@ -1,6 +1,25 @@
 class ExerciseCollection < ActiveRecord::Base
+  include TimeHelper
 
   has_and_belongs_to_many :exercises
+  belongs_to :user, polymorphic: true
+
+  def exercise_working_times
+    working_times = {}
+    exercises.each do |exercise|
+      working_times[exercise.id] = time_to_f exercise.average_working_time
+    end
+    working_times
+  end
+
+  def average_working_time
+    if exercises.empty?
+      0
+    else
+      values = exercise_working_times.values.reject { |v| v.nil?}
+      values.reduce(:+) / exercises.size
+    end
+  end
 
   def to_s
     "#{I18n.t('activerecord.models.exercise_collection.one')}: #{name} (#{id})"

app/models/request_for_comment.rb

@@ -8,6 +8,7 @@ class RequestForComment < ActiveRecord::Base
   has_many :subscriptions
 
   scope :unsolved, -> { where(solved: [false, nil]) }
+  scope :in_range, -> (from, to) { where(created_at: from..to) }
 
     def self.last_per_user(n = 5)
       from("(#{row_number_user_sql}) as request_for_comments")

app/models/structured_error.rb

@@ -3,11 +3,21 @@ class StructuredError < ActiveRecord::Base
   belongs_to :submission
   belongs_to :file, class_name: 'CodeOcean::File'
 
+  has_many :structured_error_attributes
+
   def self.create_from_template(template, message_buffer, submission)
     instance = self.create(error_template: template, submission: submission)
-    template.error_template_attributes.each do |attribute|
+    template.error_template_attributes.each do | attribute |
       StructuredErrorAttribute.create_from_template(attribute, instance, message_buffer)
     end
     instance
   end
+
+  def hint
+    content = error_template.hint
+    structured_error_attributes.each do | attribute |
+      content.sub! "{{#{attribute.error_template_attribute.key}}}", attribute.value if attribute.match
+    end
+    content
+  end
 end

app/policies/exercise_collection_policy.rb

@@ -1,3 +1,7 @@
 class ExerciseCollectionPolicy < AdminOnlyPolicy
 
+  def statistics?
+    admin?
+  end
+
 end

app/policies/statistics_policy.rb

@@ -0,0 +1,7 @@
+class StatisticsPolicy < AdminOnlyPolicy
+
+  [:graphs?, :user_activity?, :user_activity_history?, :rfc_activity?, :rfc_activity_history?].each do |action|
+    define_method(action) { admin? }
+  end
+
+end

app/views/application/_navigation.html.slim

@@ -7,9 +7,11 @@
       ul.dropdown-menu role='menu'
         - if current_user.admin?
           li = link_to(t('breadcrumbs.dashboard.show'), admin_dashboard_path)
+          li = link_to(t('breadcrumbs.statistics.show'), statistics_path)
           li.divider
-        - models = [ExecutionEnvironment, Exercise, ExerciseCollection, ProxyExercise, Tag, Consumer, CodeHarborLink, UserExerciseFeedback,
+        = render('navigation_submenu', title: t('activerecord.models.exercise.other'), models: [Exercise, ExerciseCollection, ProxyExercise, Tag], link: exercises_path)
-                ErrorTemplate, ErrorTemplateAttribute, ExternalUser, FileType, FileTemplate, InternalUser].sort_by {|model| model.model_name.human(count: 2) }
+        = render('navigation_submenu', title: t('navigation.sections.users'), models: [InternalUser, ExternalUser])
-        - models.each do |model|
+        = render('navigation_collection_link', model: ExecutionEnvironment)
-          - if policy(model).index?
+        = render('navigation_submenu', title: t('navigation.sections.errors'), models: [ErrorTemplate, ErrorTemplateAttribute])
-            li = link_to(model.model_name.human(count: 2), send(:"#{model.model_name.collection}_path"))
+        = render('navigation_submenu', title: t('navigation.sections.files'), models: [FileType, FileTemplate])
+        = render('navigation_submenu', title: t('navigation.sections.integrations'), models: [Consumer, CodeHarborLink])

app/views/application/_navigation_collection_link.html.slim

@@ -0,0 +1,2 @@
+- if policy(model).index?
+  li = link_to(model.model_name.human(count: 2), send(:"#{model.model_name.collection}_path"))

app/views/application/_navigation_submenu.html.slim

@@ -0,0 +1,6 @@
+li.dropdown.dropdown-submenu
+  - link = link.nil? ? "#" : link
+  a href=link class="dropdown-toggle" data-toggle="dropdown" = title
+  ul class="dropdown-menu"
+    - models.each do |model|
+      = render('navigation_collection_link', model: model)

app/views/error_templates/_form.html.slim

@@ -16,4 +16,5 @@
   .form-group
     = f.label(:hint)
     = f.text_field(:hint, class: 'form-control')
+    .help-block == t('error_templates.hints.hint_templates')
   .actions = render('shared/submit_button', f: f, object: @error_template)

app/views/exercise_collections/_form.html.slim

@@ -1,11 +1,18 @@
 - exercises = Exercise.order(:title)
+- users = InternalUser.order(:name)
 
-= form_for(@exercise_collection, data: {exercises: exercises}, multipart: true) do |f|
+= form_for(@exercise_collection, data: {exercises: exercises, users: users}, multipart: true) do |f|
   = render('shared/form_errors', object: @exercise_collection)
   .form-group
-    = f.label(:name)
+    = f.label(t('activerecord.attributes.exercise_collections.name'))
     = f.text_field(:name, class: 'form-control', required: true)
   .form-group
-    = f.label(:exercises)
+    = f.label(t('activerecord.attributes.exercise_collections.use_anomaly_detection'))
+    = f.check_box(:use_anomaly_detection, {class: 'form-control'})
+  .form-group
+    = f.label(t('activerecord.attributes.exercise_collections.user'))
+    = f.collection_select(:user_id, users, :id, :name, {}, {class: 'form-control'})
+  .form-group
+    = f.label(t('activerecord.attributes.exercise_collections.exercises'))
     = f.collection_select(:exercise_ids, exercises, :id, :title, {}, {class: 'form-control', multiple: true})
   .actions = render('shared/submit_button', f: f, object: @exercise_collection)

app/views/exercise_collections/index.html.slim

@@ -8,7 +8,7 @@ h1 = ExerciseCollection.model_name.human(count: 2)
         th = t('activerecord.attributes.exercise_collections.name')
         th = t('activerecord.attributes.exercise_collections.updated_at')
         th = t('activerecord.attributes.exercise_collections.exercises')
-        th colspan=3 = t('shared.actions')
+        th colspan=4 = t('shared.actions')
     tbody
       - @exercise_collections.each do |collection|
         tr
@@ -18,6 +18,7 @@ h1 = ExerciseCollection.model_name.human(count: 2)
           td = collection.exercises.size
           td = link_to(t('shared.show'), collection)
           td = link_to(t('shared.edit'), edit_exercise_collection_path(collection))
+          td = link_to(t('shared.statistics'), statistics_exercise_collection_path(collection))
           td = link_to(t('shared.destroy'), collection, data: {confirm: t('shared.confirm_destroy')}, method: :delete)
 
 = render('shared/pagination', collection: @exercise_collections)

app/views/exercise_collections/show.html.slim

@@ -3,9 +3,25 @@ h1
   = render('shared/edit_button', object: @exercise_collection)
 
 = row(label: 'exercise_collections.name', value: @exercise_collection.name)
+= row(label: 'exercise_collections.user', value: link_to(@exercise_collection.user.name, @exercise_collection.user)) unless @exercise_collection.user.nil?
+= row(label: 'exercise_collections.use_anomaly_detection', value: @exercise_collection.use_anomaly_detection)
 = row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at)
 
 h4 = t('activerecord.attributes.exercise_collections.exercises')
-ul.list-unstyled
+.table-responsive
-  - @exercise_collection.exercises.sort_by{|c| c.title}.each do |exercise|
+  table.table
-    li = link_to(exercise, exercise)
+    thead
+      tr
+        th = t('activerecord.attributes.exercise.title')
+        th = t('activerecord.attributes.exercise.execution_environment')
+        th = t('activerecord.attributes.exercise.user')
+        th = t('shared.actions')
+    tbody
+      - @exercises.sort_by{|c| c.title}.each do |exercise|
+        tr
+          td = link_to(exercise.title, exercise)
+          td = link_to_if(exercise.execution_environment && policy(exercise.execution_environment).show?, exercise.execution_environment, exercise.execution_environment)
+          td = exercise.user.name
+          td = link_to(t('shared.statistics'), statistics_exercise_path(exercise))
+        
+= render('shared/pagination', collection: @exercises)

app/views/exercise_collections/statistics.html.slim

@@ -0,0 +1,17 @@
+h1 = @exercise_collection
+
+= row(label: 'exercise_collections.name', value: @exercise_collection.name)
+= row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at)
+= row(label: 'exercise_collections.exercises', value: @exercise_collection.exercises.count)
+= row(label: 'exercises.statistics.average_worktime', value: @exercise_collection.average_working_time.round(3).to_s + 's')
+
+#graph
+  #data.hidden(data-working-times=ActiveSupport::JSON.encode(@exercise_collection.exercise_working_times) data-average-working-time=@exercise_collection.average_working_time)
+  #legend
+    - {time: t('exercises.statistics.average_worktime'),
+        min: 'min. anomaly threshold',
+        avg: 'average time',
+        max: 'max. anomaly threshold'}.each_pair do |klass, label|
+      .legend-entry
+        div(class="box #{klass}")
+        .box-label = label

app/views/exercises/_editor_output.html.slim

@@ -47,9 +47,12 @@ div id='output_sidebar_uncollapsed' class='hidden col-sm-12 enforce-bottom-margi
       input#prompt-input.form-control type='text'
       span.input-group-btn
         button#prompt-submit.btn.btn-primary type="button" = t('exercises.editor.send')
+    #error-hints
+      .heading = t('exercises.implement.error_hints.heading')
+      ul.body
     #output
       pre = t('exercises.implement.no_output_yet')
     - if CodeOcean::Config.new(:code_ocean).read[:flowr][:enabled]
       #flowrHint.panel.panel-info data-url=CodeOcean::Config.new(:code_ocean).read[:flowr][:url] role='tab'
         .panel-heading = 'Gain more insights here'
-        .panel-body
+        .panel-body

app/views/exercises/external_users/statistics.html.slim

@@ -1,19 +1,16 @@
 h1 = "#{@exercise} (external user #{@external_user})"
-- submissions = Submission.where("user_id = ?  AND exercise_id = ?", @external_user.id, @exercise.id).order("created_at")
+- current_submission = @submissions.first
-- current_submission = submissions.first
-- submissions_and_interventions = (submissions + UserExerciseIntervention.where("user_id = ?  AND exercise_id = ?", @external_user.id, @exercise.id)).sort_by { |a| a.created_at }
-
 - if current_submission
   - initial_files = current_submission.files.to_a
 
   - all_files = []
   - file_types = Set.new()
-  - submissions.each do |submission|
+  - @submissions.each do |submission|
     - submission.files.each do |file|
       - file_types.add(ActiveSupport::JSON.encode(file.file_type))
     - all_files.push(submission.files)
 
-  .hidden#data data-submissions=ActiveSupport::JSON.encode(submissions) data-files=ActiveSupport::JSON.encode(all_files) data-file-types=ActiveSupport::JSON.encode(file_types)
+  .hidden#data data-submissions=ActiveSupport::JSON.encode(@submissions) data-files=ActiveSupport::JSON.encode(all_files) data-file-types=ActiveSupport::JSON.encode(file_types)
 
   #stats-editor.row
     - index = 0
@@ -27,14 +24,13 @@ h1 = "#{@exercise} (external user #{@external_user})"
     button.btn.btn-default id='play-button'
       span.fa.fa-play
     #submissions-slider.flex-item
-      input type='range' orient='horizontal' list='datapoints' min=0 max=submissions.length-1 value=0
+      input type='range' orient='horizontal' list='datapoints' min=0 max=@submissions.length-1 value=0
       datalist#datapoints
         - index=0
-        - submissions.each do |submission|
+        - @submissions.each do |submission|
           option data-submission=submission
             =index
           - index += 1
-  - working_times_until = Array.new
   #timeline
     .table-responsive
       table.table
@@ -43,28 +39,27 @@ h1 = "#{@exercise} (external user #{@external_user})"
             - ['.time', '.cause', '.score', '.tests', '.time_difference'].each do |title|
               th.header = t(title)
         tbody
-          - deltas = submissions.map.with_index {|item, index| delta = item.created_at - submissions[index - 1].created_at if index > 0; if delta == nil or delta > 10*60 then 0 else delta end}
+          - @submissions_and_interventions.each_with_index do |this, index|
-          - submissions_and_interventions.each_with_index do |submission_or_intervention, index|
+            - highlight = (index > 0 and @working_times_until[index] == @working_times_until[index - 1] and this.created_at > @submissions_and_interventions[index - 1].created_at)
-            tr data-id=submission_or_intervention.id
+            tr data-id=this.id class=('highlight' if highlight)
-              td.clickable = submission_or_intervention.created_at.strftime("%F %T")
+              td.clickable = this.created_at.strftime("%F %T")
-              - if submission_or_intervention.is_a?(Submission)
+              - if this.is_a?(Submission)
-                td = submission_or_intervention.cause
+                td = this.cause
-                td = submission_or_intervention.score
+                td = this.score
                 td
-                  -submission_or_intervention.testruns.each do |run|
+                  -this.testruns.each do |run|
                     - if run.passed
                       .unit-test-result.positive-result  title=run.output
                     - else
                       .unit-test-result.unknown-result title=run.output
-                td = Time.at(deltas[1..index].inject(:+)).utc.strftime("%H:%M:%S") if index > 0
+                td = @working_times_until[index] if index > 0
-                -working_times_until.push((Time.at(deltas[1..index].inject(:+)).utc.strftime("%H:%M:%S") if index > 0))
+              - elsif this.is_a? UserExerciseIntervention
-              - elsif submission_or_intervention.is_a? UserExerciseIntervention
+                td = this.intervention.name
-                td = submission_or_intervention.intervention.name
                 td =
                 td =
                 td =
     p = t('.addendum')
-    .hidden#wtimes data-working_times=ActiveSupport::JSON.encode(working_times_until);
+    .hidden#wtimes data-working_times=ActiveSupport::JSON.encode(@working_times_until);
     div#progress_chart.col-lg-12
       .graph-functions-2
 

app/views/exercises/statistics.html.slim

@@ -2,9 +2,15 @@ script src="http://labratrevenge.com/d3-tip/javascripts/d3.tip.v0.6.3.js"
 h1 = @exercise
 
 = row(label: '.participants', value: @exercise.users.distinct.count)
+
 - [:intermediate, :final].each do |scope|
   = row(label: ".#{scope}_submissions") do
     = "#{@exercise.submissions.send(scope).count} (#{t('.users', count: @exercise.submissions.send(scope).distinct.count(:user_id))})"
+
+= row(label: '.finishing_rate') do
+  p == @exercise.finishers.count ? "#{t('shared.out_of', maximum_value: @exercise.users.distinct.count, value: @exercise.finishers.count)} #{t('exercises.statistics.external_users')}" : empty
+  p = progress_bar((100.0 / @exercise.users.distinct.count * @exercise.finishers.count).round(2))
+
 = row(label: '.average_score') do
   p == @exercise.average_score ? t('shared.out_of', maximum_value: @exercise.maximum_score, value: @exercise.average_score.round(2)) : empty
   p = progress_bar(@exercise.average_percentage)
@@ -40,4 +46,4 @@ h1 = @exercise
             td = link_to_if symbol==:external_users, label, {controller: "exercises", action: "statistics", external_user_id: user.id, id: @exercise.id}
             td = us['maximum_score'] or 0
             td = us['runs']
-            td = @exercise.average_working_time_for(user.id) or 0
+            td = @exercise.average_working_time_for(user.id) or 0

app/views/request_for_comments/show.html.erb

@@ -43,10 +43,20 @@
         <% output_runs = testruns.select { |run| run.cause == 'run' } %>
         <% if output_runs.size > 0 %>
           <h5><%= t('request_for_comments.runtime_output') %></h5>
-          <div class="testrun-output text">
+          <div class="collapsed testrun-output text">
-            <span class="fa fa-chevron-up collapse-button"></span>
+            <span class="fa fa-chevron-down collapse-button"></span>
             <% output_runs.each do |testrun| %>
-              <pre><%= testrun.try(:output) or t('request_for_comments.no_output') %></pre>
+              <%
+                output = testrun.try(:output)
+                if output
+                  messages = output.scan(/{(?:(?:".+?":".+?")+?,?)+}/)
+                  messages.map! {|el| JSON.parse(el)}
+                  messages.keep_if {|message| message['cmd'] == 'write'}
+                  messages.map! {|message| message['data']}
+                  output = messages.join ''
+                end
+              %>
+              <pre><%= output or t('request_for_comments.no_output') %></pre>
             <% end %>
           </div>
         <% end %>
@@ -56,7 +66,13 @@
           <h5><%= t('request_for_comments.test_results') %></h5>
           <div class="testrun-assess-results">
             <% assess_runs.each do |testrun| %>
-                <div class="result <%= testrun.passed ? 'passed' : 'failed' %>" title="<%= testrun.output %>"></div>
+              <div class="testrun-container">
+                <div class="result <%= testrun.passed ? 'passed' : 'failed' %>"></div>
+                <div class="collapsed testrun-output text">
+                  <span class="fa fa-chevron-down collapse-button"></span>
+                  <pre><%= testrun.output or t('request_for_comments.no_output')%></pre>
+                </div>
+              </div>
             <% end %>
           </div>
         <% end %>

app/views/statistics/activity_history.html.slim

@@ -0,0 +1,22 @@
+- content_for :head do
+  = javascript_include_tag(asset_path('vis.min.js', type: :javascript))
+  = stylesheet_link_tag(asset_path('vis.min.css', type: :stylesheet))
+
+.group
+  .title
+    h1 = t("statistics.graphs.#{resource}_activity")
+  .spinner
+  .graph id="#{resource}-activity-history"
+  form
+    .form-group
+      label for="from-date" = t('.from')
+      input type="date" class="form-control" id="from-date" name="from" value=(params[:from] || DateTime.new(2016).to_date)
+    .form-group
+      label for="to-date" = t('.to')
+      input type="date" class="form-control" id="to-date" name="to" value=(params[:to] || DateTime.now.to_date)
+    .form-group
+      label for="interval" = t('.interval')
+      select class="form-control" id="interval" name="interval"
+        = [:year, :quarter, :month, :day, :hour, :minute, :second].each do | key |
+          option selected=(key.to_s == params[:interval]) = key
+    button type="submit" class="btn btn-primary" = t('.update')

app/views/statistics/graphs.html.slim

@@ -0,0 +1,17 @@
+- content_for :head do
+  = javascript_include_tag(asset_path('vis.min.js', type: :javascript))
+  = stylesheet_link_tag(asset_path('vis.min.css', type: :stylesheet))
+
+.group
+  .title
+    h1 = t('.user_activity')
+    a href=statistics_graphs_user_activity_history_path = t('.history')
+  .spinner
+  .graph#user-activity
+
+.group
+  .title
+    h1 = t('.rfc_activity')
+    a href=statistics_graphs_rfc_activity_history_path = t('.history')
+  .spinner
+  .graph#rfc-activity

app/views/statistics/show.html.slim

@@ -0,0 +1,12 @@
+
+#statistics-container
+  - statistics_data.each do | section |
+    h2 = section[:name]
+    .statistics-wrapper data-key=section[:key]
+      - section[:entries].each do | entry |
+        a href=entry[:url]
+          div data-key=entry[:key]
+            .title = entry[:name]
+            .data
+              span = entry[:data].to_s
+              span.unit = entry[:unit] if entry.key? :unit

app/views/user_mailer/exercise_anomaly_detected.html.slim

@@ -0,0 +1,38 @@
+== t('mailers.user_mailer.exercise_anomaly_detected.body1',
+        receiver_displayname: @receiver_displayname,
+        collection_name: @collection.name)
+
+table(border=1)
+  thead
+    tr
+      td = t('activerecord.attributes.exercise.title', locale: :de)
+      td = t('exercises.statistics.average_worktime', locale: :de)
+      td = t('shared.actions', locale: :de)
+  tbody
+    - @anomalies.keys.each do | id |
+      - exercise = Exercise.find(id)
+      tr
+        td = link_to(exercise.title, exercise_path(exercise))
+        td = @anomalies[id]
+        td = link_to(t('shared.statistics', locale: :de), statistics_exercise_path(exercise))
+
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body2',
+        receiver_displayname: @receiver_displayname,
+        collection_name: @collection.name)
+
+table(border=1)
+  thead
+    tr
+      td = t('activerecord.attributes.exercise.title', locale: :en)
+      td = t('exercises.statistics.average_worktime', locale: :en)
+      td = t('shared.actions', locale: :en)
+  tbody
+    - @anomalies.keys.each do | id |
+      - exercise = Exercise.find(id)
+      tr
+        td = link_to(exercise.title, exercise_path(exercise))
+        td = @anomalies[id]
+        td = link_to(t('shared.statistics', locale: :en), statistics_exercise_path(exercise))
+
+== t('mailers.user_mailer.exercise_anomaly_detected.body3')

app/views/user_mailer/exercise_anomaly_needs_feedback.html.slim

@@ -0,0 +1 @@
+== t('mailers.user_mailer.exercise_anomaly_needs_feedback.body', receiver_displayname: @receiver_displayname, exercise: @exercise_title, link: link_to(@link, @link))

config/deploy.rb

@@ -9,6 +9,8 @@ set :log_level, :info
 set :puma_threads, [0, 16]
 set :repo_url, 'git@github.com:openHPI/codeocean.git'
 
+set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" }
+
 namespace :deploy do
   before 'check:linked_files', 'config:push'
 
@@ -21,3 +23,11 @@ namespace :deploy do
     end
   end
 end
+
+namespace :whenever do
+  task :update_crontab do
+    run 'bundle exec whenever --update-crontab'
+  end
+end
+
+after 'deploy', 'whenever:update_crontab'

config/docker.yml.erb.example

@@ -9,7 +9,7 @@ development:
   <<: *default
   host: tcp://127.0.0.1:2376
   ws_host: ws://127.0.0.1:2376 #url to connect rails server to docker host
-  ws_client_protocol: ws:// #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
+  ws_client_protocol: 'ws:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
   workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %>
   pool:
     active: true
@@ -32,7 +32,7 @@ production:
       timeout: 60
   workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %>
   ws_host: ws://localhost:4243 #url to connect rails server to docker host
-  ws_client_protocol: wss:// #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
+  ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
 
 staging:
   <<: *default

config/docker.yml.erb.travis

@@ -9,7 +9,7 @@ development:
   <<: *default
   host: tcp://127.0.0.1:2376
   ws_host: ws://127.0.0.1:2376 #url to connect rails server to docker host
-  ws_client_protocol: ws:// #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
+  ws_client_protocol: 'ws:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
   workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %>
   pool:
     active: true
@@ -32,7 +32,7 @@ production:
       timeout: 60
   workspace_root: <%= Rails.root.join('tmp', 'files', Rails.env) %>
   ws_host: ws://localhost:4243 #url to connect rails server to docker host
-  ws_client_protocol: wss:// #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
+  ws_client_protocol: 'wss:' #set the websocket protocol to be used by the client to connect to the rails server (ws on development, wss on production)
 
 staging:
   <<: *default

config/locales/de.yml

@@ -123,6 +123,8 @@ de:
       exercise_collections:
         id: "ID"
         name: "Name"
+        user: "Verantwortlicher"
+        use_anomaly_detection: "Abweichungen in der Arbeitszeit erkennen"
         updated_at: "Letzte Änderung"
         exercises: "Aufgaben"
       user_exercise_feedback:
@@ -217,6 +219,11 @@ de:
       show: Dashboard
     sessions:
       destroy_through_lti: Code-Abgabe
+    statistics:
+      show: "Statistiken"
+      graphs: "Visualisierungen"
+      user_activity_history: Nutzeraktivitätshistorie
+      rfc_activity_history: Kommentaranfragenhistorie
   consumers:
     show:
       link: Konsument
@@ -327,6 +334,8 @@ de:
       break_intervention:
         title: "Pause"
         text: "Uns ist aufgefallen, dass du schon lange an dieser Aufgabe arbeitest. Möchtest du vielleicht später weiter machen um erstmal auf neue Gedanken zu kommen?"
+      error_hints:
+        heading: "Hinweise"
     index:
       clone: Duplizieren
       implement: Implementieren
@@ -344,7 +353,8 @@ de:
       worktime: Arbeitszeit
       average_worktime: Durchschnittliche Arbeitszeit
       internal_users: Interne Nutzer
-      external_user: Externe Nutzer
+      external_users: Externe Nutzer
+      finishing_rate: Abschlussrate
     submit:
       failure: Beim Übermitteln Ihrer Punktzahl ist ein Fehler aufgetreten. Bitte versuchen Sie es später erneut.
       full_score_redirect_to_rfc: Herzlichen Glückwunsch! Sie haben die maximale Punktzahl für diese Aufgabe an den Kurs übertragen. Ein anderer Teilnehmer hat eine Frage zu der von Ihnen gelösten Aufgabe. Er würde sich sicherlich sehr über ihre Hilfe und Kommentare freuen.
@@ -357,7 +367,7 @@ de:
         score: Punktzahl
         tests: Unit Tests
         time_difference: 'Arbeitszeit bis hier*'
-        addendum: '* Differenzen von mehr als 30 Minuten werden ignoriert.'
+        addendum: '* Differenzen von mehr als 10 Minuten werden ignoriert.'
   proxy_exercises:
       index:
         clone: Duplizieren
@@ -518,6 +528,64 @@ de:
           <br>
           This mail was automatically sent by CodeOcean. <br>
         subject: "%{author_displayname} hat einen neuen Kommentar in einer Diskussion veröffentlicht, die Sie abonniert haben."
+      exercise_anomaly_detected:
+        subject: "Unregelmäßigkeiten in Aufgaben Ihrer Aufgabensammlung"
+        body1: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+          <br>
+          Die Aufgaben sind:
+          <br>
+        body2: |
+          <br>
+          Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+          <br>
+          The exercises are:
+          <br>
+        body3: |
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+      exercise_anomaly_needs_feedback:
+        body: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+        subject: "Eine Aufgabe auf CodeOcean benötigt Ihr Feedback"
   request_for_comments:
     click_here: Zum Kommentieren auf die Seitenleiste klicken!
     comments: Kommentare
@@ -567,7 +635,7 @@ de:
       score: Ihre Punktzahl
       success_with_outcome: 'Ihr Code wurde erfolgreich bewertet. Ihre Bewertung wurde an %{consumer} übermittelt.'
       success_without_outcome: Ihr Code wurde erfolgreich bewertet.
-      do_not_use_backbutton: Benutzen Sie nicht den "Zurück" Button des Browsers, um zu CodeOcean zurück zu kehren. Übungen müssen immer aus dem %{consumer} Kontext gestartet werden.
+      do_not_use_backbutton: 'Benutzen Sie nicht den "Zurück" Button des Browsers, um zu CodeOcean zurück zu kehren. Übungen müssen immer aus dem %{consumer} Kontext gestartet werden.'
     new:
       forgot_password: Passwort vergessen?
       headline: Anmelden
@@ -666,6 +734,7 @@ de:
   error_templates:
     hints:
       signature: "Ein regulärer Ausdruck in Ruby-Syntax und ohne führende und schließende \"/\""
+      hint_templates: 'Attributnamen in {{doppelten geschweiften Klammern}} werden zur Laufzeit durch die jeweiligen Attributwerte ersetzt. Beispiel: "Der Fehler ist in Zeile {{Line}}." --(StructuredError: {Line: 4})--> "Der Fehler ist in Zeile 4."'
     attributes: "Attribute"
     add_attribute: "Attribut hinzufügen"
   comments:
@@ -674,3 +743,37 @@ de:
   subscriptions:
     successfully_unsubscribed: "Ihr Abonnement für weitere Kommentare auf dieser Kommentaranfrage wurde erfolgreich beendet."
     subscription_not_existent: "Das Abonnement, von dem Sie sich abmelden wollen, existiert nicht."
+  statistics:
+    sections:
+      users: "Benutzer"
+      exercises: "Aufgaben"
+      request_for_comments: "Kommentaranfragen"
+    entries:
+      exercises:
+        average_number_of_submissions: "Durchschnittliche Zahl von Abgaben"
+        submissions_per_minute: "Aktuelle Abgabenhäufigkeit (1h)"
+        submissions: "Abgaben"
+      request_for_comments:
+        percent_solved: "Beantwortete Anfragen"
+        percent_unsolved: "Unbeantwortete Anfragen"
+        percent_soft_solved: "Ungelöst mit voller Punktzahl"
+        with_comments: "Anfragen mit Kommentaren"
+      users:
+        currently_active: "Aktiv (5 Minuten)"
+        currently_active60: "Aktiv (60 Minuten)"
+        active: "Aktive Nutzer"
+    graphs:
+      user_activity: "Nutzeraktivität"
+      rfc_activity: "Kommentaranfragenaktivität"
+      history: "Historie"
+    activity_history:
+      from: "Von"
+      to: "Bis"
+      interval: "Intervall"
+      update: "Aktualisieren"
+  navigation:
+    sections:
+      errors: "Fehler"
+      files: "Dateien"
+      users: "Benutzer"
+      integrations: "Integrationen"

config/locales/en.yml

@@ -123,6 +123,8 @@ en:
       exercise_collections:
         id: "ID"
         name: "Name"
+        user: "Associated User"
+        use_anomaly_detection: "Enable Worktime Anomaly Detection"
         updated_at: "Last Update"
         exercises: "Exercises"
       user_exercise_feedback:
@@ -217,6 +219,11 @@ en:
       show: Dashboard
     sessions:
       destroy_through_lti: Code Submission
+    statistics:
+      show: "Statistics"
+      graphs: "Graphs"
+      user_activity_history: User Activity History
+      rfc_activity_history: RfC Activity History
   consumers:
     show:
       link: Consumer
@@ -327,6 +334,8 @@ en:
       break_intervention:
         title: "Break"
         text: "We recognized that you are already working quite a while on this exercise. We would like to encourage you to take a break and come back later."
+      error_hints:
+        heading: "Hints"
     index:
       clone: Duplicate
       implement: Implement
@@ -345,6 +354,7 @@ en:
       average_worktime: Average Working Time
       internal_users: Internal Users
       external_users: External Users
+      finishing_rate: Finishing Rate
     submit:
       failure: An error occured while transmitting your score. Please try again later.
       full_score_redirect_to_rfc: Congratulations! You achieved and submitted the highest possible score for this exercise. Another participant has a question concerning the exercise you just solved. Your help and comments will be greatly appreciated!
@@ -357,7 +367,7 @@ en:
         score: Score
         tests: Unit Test Results
         time_difference: 'Working Time until here*'
-        addendum: '* Deltas longer than 30 minutes are ignored.'
+        addendum: '* Deltas longer than 10 minutes are ignored.'
   proxy_exercises:
       index:
         clone: Duplicate
@@ -518,6 +528,64 @@ en:
           <br>
           This mail was automatically sent by CodeOcean. <br>
         subject: "%{author_displayname} has posted a new comment to a discussion you subscribed to on CodeOcean."
+      exercise_anomaly_detected:
+        subject: "Anomalies in exercises of your exercise collection"
+        body1: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          eine oder mehrere Aufgaben Ihrer Aufgabensammlung "%{collection_name}" zeigen Unregelmäßigkeiten in der Bearbeitungszeit. Möglicherweise sind sie zu schwer oder zu leicht.
+          <br>
+          Die Aufgaben sind:
+          <br>
+        body2: |
+          <br>
+          Falls Sie beim Klick auf einen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          at least one exercise in your exercise collection "%{collection_name}" has a much longer or much shorter average working time than the average. Perhaps they are too difficult or too easy.
+          <br>
+          The exercises are:
+          <br>
+        body3: |
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking a link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+      exercise_anomaly_needs_feedback:
+        body: |
+          English version below <br>
+          _________________________<br>
+          <br>
+          Hallo %{receiver_displayname}, <br>
+          <br>
+          um die Aufgaben auf CodeOcean weiter zu verbessern, benötigen wir Ihre Mithilfe. Bitte nehmen Sie sich ein paar Minuten Zeit um ein kurzes Feedback zu folgender Aufgabe zu geben:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          Falls Sie beim Klick auf diesen Link eine Fehlermeldung erhalten, dass Sie nicht berechtigt wären diese Aktion auszuführen, öffnen Sie bitte eine beliebige Programmieraufgabe aus einem Kurs heraus und klicken den Link danach noch einmal.<br>
+          <br>
+          Diese Mail wurde automatisch von CodeOcean verschickt.<br>
+          <br>
+          _________________________<br>
+          <br>
+          Dear %{receiver_displayname}, <br>
+          <br>
+          we need your help to improve the quality of the exercises on CodeOcean. Please take a few minutes to give us feedback for the following exercise:
+          <br>
+          %{exercise} - %{link}
+          <br>
+          If you receive an error that you are not authorized to perform this action when clicking the link, please log-in through any course exercise beforehand and click the link again. <br>
+          <br>
+          This mail was automatically sent by CodeOcean. <br>
+        subject: "An exercise on CodeOcean needs your feedback"
   request_for_comments:
     click_here: Click on this sidebar to comment!
     comments: Comments
@@ -553,7 +621,7 @@ en:
       failure: Invalid email or password.
       success: Successfully signed in.
     create_through_lti:
-      session_with_outcome: 'Please click "Submit Code for Assessment" after scoring to send your score %{consumer}.'
+      session_with_outcome: 'Please click "Submit Code for Assessment" after scoring to send your score to %{consumer}.'
       session_without_outcome: 'This is a practice session. Your grade will not be transmitted to %{consumer}.'
     destroy:
       link: Sign out
@@ -567,7 +635,7 @@ en:
       score: Your Score
       success_with_outcome: 'Your code has been successfully assessed. Your grade has been transmitted to %{consumer}.'
       success_without_outcome: Your code has been successfully assessed.
-      do_not_use_backbutton: Never use the browser's "Back" button to get back to CodeOcean. Always start an exercise from within %{consumer}.
+      do_not_use_backbutton: "Never use the browser's \"Back\" button to get back to CodeOcean. Always start an exercise from within %{consumer}."
     new:
       forgot_password: Forgot password?
       headline: Sign In
@@ -666,6 +734,7 @@ en:
   error_templates:
     hints:
       signature: "A regular expression in Ruby syntax without leading and trailing \"/\""
+      hint_templates: 'Attribute names in {{double curly braces}} are replaced by the corresponding attribute value at runtime, e.g. "The error occurs in line {{Line}}." --(StructuredError: {Line: 4})--> "The error occurs in line 4."'
     attributes: "Attributes"
     add_attribute: "Add attribute"
   comments:
@@ -674,3 +743,37 @@ en:
   subscriptions:
     successfully_unsubscribed: "You successfully unsubscribed from this Request for Comment"
     subscription_not_existent: "The subscription you want to unsubscribe from does not exist."
+  statistics:
+    sections:
+      users: "Users"
+      exercises: "Exercises"
+      request_for_comments: "Requests for Comment"
+    entries:
+      exercises:
+        average_number_of_submissions: "Average Number of Submissions"
+        submissions_per_minute: "Current Submission Volume (1h)"
+        submissions: "Submissions"
+      request_for_comments:
+        percent_solved: "Solved Requests"
+        percent_unsolved: "Unsolved Requests"
+        percent_soft_solved: "Unsolved with full score"
+        with_comments: "RfCs with Comments"
+      users:
+        currently_active: "Active (5 minutes)"
+        currently_active60: "Active (60 minutes)"
+        active: "Active Users"
+    graphs:
+      user_activity: "User Activity"
+      rfc_activity: "RfC Activity"
+      history: "History"
+    activity_history:
+      from: "From"
+      to: "To"
+      interval: "Interval"
+      update: "Update"
+  navigation:
+    sections:
+      errors: "Errors"
+      files: "Files"
+      users: "Users"
+      integrations: "Integrations"

config/routes.rb

@@ -42,6 +42,13 @@ Rails.application.routes.draw do
 
   get '/help', to: 'application#help'
 
+  get 'statistics/', to: 'statistics#show'
+  get 'statistics/graphs', to: 'statistics#graphs'
+  get 'statistics/graphs/user-activity', to: 'statistics#user_activity'
+  get 'statistics/graphs/user-activity-history', to: 'statistics#user_activity_history'
+  get 'statistics/graphs/rfc-activity', to: 'statistics#rfc_activity'
+  get 'statistics/graphs/rfc-activity-history', to: 'statistics#rfc_activity_history'
+
   concern :statistics do
     member do
       get :statistics
@@ -82,7 +89,11 @@ Rails.application.routes.draw do
     end
   end
 
-  resources :exercise_collections
+  resources :exercise_collections do
+    member do
+      get :statistics
+    end
+  end
 
   resources :proxy_exercises do
     member do

config/schedule.rb

@@ -0,0 +1,27 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+set :output, Whenever.path + '/log/whenever/whenever_$(date +%Y%m%d%H%M%S).log'
+set :environment, ENV['RAILS_ENV'] if ENV['RAILS_ENV']
+
+every 1.day, at: '3:00 am' do
+  rake 'detect_exercise_anomalies:with_at_least[50,50]'
+end

db/migrate/20171115121125_add_anomaly_detection_flag_to_exercise_collection.rb

@@ -0,0 +1,5 @@
+class AddAnomalyDetectionFlagToExerciseCollection < ActiveRecord::Migration
+  def change
+    add_column :exercise_collections, :use_anomaly_detection, :boolean, :default => false
+  end
+end

db/migrate/20171122124222_add_index_to_exercises.rb

@@ -0,0 +1,5 @@
+class AddIndexToExercises < ActiveRecord::Migration
+  def change
+    add_index :exercises, :id
+  end
+end

db/migrate/20171210172208_add_user_to_exercise_collection.rb

@@ -0,0 +1,5 @@
+class AddUserToExerciseCollection < ActiveRecord::Migration
+  def change
+    add_reference :exercise_collections, :user,  polymorphic: true, index: true
+  end
+end

db/migrate/20180226131340_create_anomaly_notifications.rb

@@ -0,0 +1,11 @@
+class CreateAnomalyNotifications < ActiveRecord::Migration
+  def change
+    create_table :anomaly_notifications do |t|
+      t.belongs_to :user, polymorphic: true, index: true
+      t.belongs_to :exercise, index: true
+      t.belongs_to :exercise_collection, index: true
+      t.string :reason
+      t.timestamps
+    end
+  end
+end

db/migrate/20180515110030_remove_file_id_from_structured_errors.rb

@@ -0,0 +1,5 @@
+class RemoveFileIdFromStructuredErrors < ActiveRecord::Migration
+  def change
+    remove_column :structured_errors, :file_id
+  end
+end

db/schema.rb

@@ -11,11 +11,25 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180222145909) do
+ActiveRecord::Schema.define(version: 20180515110030) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "anomaly_notifications", force: :cascade do |t|
+    t.integer  "user_id"
+    t.string   "user_type"
+    t.integer  "exercise_id"
+    t.integer  "exercise_collection_id"
+    t.string   "reason"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "anomaly_notifications", ["exercise_collection_id"], name: "index_anomaly_notifications_on_exercise_collection_id", using: :btree
+  add_index "anomaly_notifications", ["exercise_id"], name: "index_anomaly_notifications_on_exercise_id", using: :btree
+  add_index "anomaly_notifications", ["user_type", "user_id"], name: "index_anomaly_notifications_on_user_type_and_user_id", using: :btree
+
   create_table "code_harbor_links", force: :cascade do |t|
     t.string   "oauth2token", limit: 255
     t.datetime "created_at"
@@ -104,8 +118,13 @@ ActiveRecord::Schema.define(version: 20180222145909) do
     t.string   "name"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.boolean  "use_anomaly_detection", default: false
+    t.integer  "user_id"
+    t.string   "user_type"
   end
 
+  add_index "exercise_collections", ["user_type", "user_id"], name: "index_exercise_collections_on_user_type_and_user_id", using: :btree
+
   create_table "exercise_collections_exercises", id: false, force: :cascade do |t|
     t.integer "exercise_collection_id"
     t.integer "exercise_id"
@@ -137,6 +156,8 @@ ActiveRecord::Schema.define(version: 20180222145909) do
     t.integer  "expected_difficulty",                  default: 1
   end
 
+  add_index "exercises", ["id"], name: "index_exercises_on_id", using: :btree
+
   create_table "exercises_proxy_exercises", id: false, force: :cascade do |t|
     t.integer  "proxy_exercise_id"
     t.integer  "exercise_id"
@@ -304,7 +325,6 @@ ActiveRecord::Schema.define(version: 20180222145909) do
 
   create_table "structured_errors", force: :cascade do |t|
     t.integer  "error_template_id"
-    t.integer  "file_id"
     t.datetime "created_at",        null: false
     t.datetime "updated_at",        null: false
     t.integer  "submission_id"

lib/tasks/detect_exercise_anomalies.rake

@@ -0,0 +1,159 @@
+include Rails.application.routes.url_helpers
+
+namespace :detect_exercise_anomalies do
+  # uncomment for debug logging:
+  # logger           = Logger.new(STDOUT)
+  # logger.level     = Logger::DEBUG
+  # Rails.logger     = logger
+
+  # These factors determine if an exercise is an anomaly, given the average working time (avg):
+  # (avg * MIN_TIME_FACTOR) <= working_time <= (avg * MAX_TIME_FACTOR)
+  MIN_TIME_FACTOR = 0.1
+  MAX_TIME_FACTOR = 2
+
+  # Determines how many users are picked from the best/average/worst performers of each anomaly for feedback
+  NUMBER_OF_USERS_PER_CLASS = 10
+
+  # Determines margin below which user working times will be considered data errors (e.g. copy/paste solutions)
+  MIN_USER_WORKING_TIME = 0.0
+
+  # Cache exercise working times, because queries are expensive and values do not change between collections
+  WORKING_TIME_CACHE = {}
+  AVERAGE_WORKING_TIME_CACHE = {}
+
+  task :with_at_least, [:number_of_exercises, :number_of_solutions] => :environment do |task, args|
+    include TimeHelper
+
+    number_of_exercises = args[:number_of_exercises]
+    number_of_solutions = args[:number_of_solutions]
+
+    puts "Searching for exercise collections with at least #{number_of_exercises} exercises and #{number_of_solutions} users."
+    # Get all exercise collections that have at least the specified amount of exercises and at least the specified
+    # number of submissions AND are flagged for anomaly detection
+    collections = get_collections(number_of_exercises, number_of_solutions)
+    puts "Found #{collections.length}."
+
+    collections.each do |collection|
+      puts "\t- #{collection}"
+      anomalies = find_anomalies(collection)
+
+      if anomalies.length > 0 and not collection.user.nil?
+        notify_collection_author(collection, anomalies)
+        notify_users(collection, anomalies)
+        reset_anomaly_detection_flag(collection)
+      end
+    end
+    puts 'Done.'
+  end
+
+  def get_collections(number_of_exercises, number_of_solutions)
+    ExerciseCollection
+      .where(:use_anomaly_detection => true)
+      .joins("join exercise_collections_exercises ece on exercise_collections.id = ece.exercise_collection_id
+                            join
+                              (select e.id
+                               from exercises e
+                                 join submissions s on s.exercise_id = e.id
+                               group by e.id
+                               having count(s.user_id) > #{ExerciseCollection.sanitize(number_of_solutions)}
+                              ) as exercises_with_submissions on exercises_with_submissions.id = ece.exercise_id")
+      .group('exercise_collections.id')
+      .having('count(exercises_with_submissions.id) > ?', number_of_exercises)
+  end
+
+  def find_anomalies(collection)
+    working_times = {}
+    collection.exercises.each do |exercise|
+      puts "\t\t> #{exercise.title}"
+      working_times[exercise.id] = get_average_working_time(exercise)
+    end
+    average = working_times.values.reduce(:+) / working_times.size
+    working_times.select do |exercise_id, working_time|
+      working_time > average * MAX_TIME_FACTOR or working_time < average * MIN_TIME_FACTOR
+    end
+  end
+
+  def get_average_working_time(exercise)
+    unless AVERAGE_WORKING_TIME_CACHE.key?(exercise.id)
+      seconds = time_to_f exercise.average_working_time
+      AVERAGE_WORKING_TIME_CACHE[exercise.id] = seconds
+    end
+    AVERAGE_WORKING_TIME_CACHE[exercise.id]
+  end
+
+  def get_user_working_times(exercise)
+    unless WORKING_TIME_CACHE.key?(exercise.id)
+      exercise.retrieve_working_time_statistics
+      WORKING_TIME_CACHE[exercise.id] = exercise.working_time_statistics
+    end
+    WORKING_TIME_CACHE[exercise.id]
+  end
+
+  def notify_collection_author(collection, anomalies)
+    puts "\t\tSending E-Mail to author (#{collection.user.displayname} <#{collection.user.email}>)..."
+    UserMailer.exercise_anomaly_detected(collection, anomalies).deliver_now
+  end
+
+  def notify_users(collection, anomalies)
+    by_id_and_type = proc { |u| {user_id: u[:user_id], user_type: u[:user_type]} }
+
+    puts "\t\tSending E-Mails to best and worst performing users of each anomaly..."
+    anomalies.each do |exercise_id, average_working_time|
+      puts "\t\tAnomaly in exercise #{exercise_id} (avg: #{average_working_time} seconds):"
+      exercise = Exercise.find(exercise_id)
+      users_to_notify = []
+
+      users = {}
+      [:performers_by_time, :performers_by_score].each do |method|
+        # merge users found by multiple methods returning a hash {best: [], worst: []}
+        users = users.merge(send(method, exercise, NUMBER_OF_USERS_PER_CLASS)) {|key, this, other| this + other}
+      end
+
+      # write reasons for feedback emails to db
+      users.keys.each do |key|
+        segment = users[key].uniq &by_id_and_type
+        users_to_notify += segment
+        segment.each do |user|
+          reason = "{\"segment\": \"#{key.to_s}\", \"feature\": \"#{user[:reason]}\", value: \"#{user[:value]}\"}"
+          AnomalyNotification.create(user_id: user[:user_id], user_type: user[:user_type],
+                                     exercise: exercise, exercise_collection: collection, reason: reason)
+        end
+      end
+
+      users_to_notify.uniq! &by_id_and_type
+      users_to_notify.each do |u|
+        user = u[:user_type] == InternalUser.name ? InternalUser.find(u[:user_id]) : ExternalUser.find(u[:user_id])
+        host = CodeOcean::Application.config.action_mailer.default_url_options[:host]
+        feedback_link = url_for(action: :new, controller: :user_exercise_feedbacks, exercise_id: exercise.id, host: host)
+        UserMailer.exercise_anomaly_needs_feedback(user, exercise, feedback_link).deliver
+      end
+      puts "\t\tAsked #{users_to_notify.size} users for feedback."
+    end
+  end
+
+  def performers_by_score(exercise, n)
+    submissions = exercise.last_submission_per_user.where('score is not null').order(score: :desc)
+    map_block = proc {|item| {user_id: item.user_id, user_type: item.user_type, value: item.score, reason: 'score'}}
+    best_performers = submissions.first(n).to_a.map &map_block
+    worst_performers = submissions.last(n).to_a.map &map_block
+    return {:best => best_performers, :worst => worst_performers}
+  end
+
+  def performers_by_time(exercise, n)
+    working_times = get_user_working_times(exercise).values.map do |item|
+      {user_id: item['user_id'], user_type: item['user_type'], score: item['score'].to_f,
+       value: time_to_f(item['working_time']), reason: 'time'}
+    end
+    avg_score = exercise.average_score
+    working_times.reject! {|item| item[:value].nil? or item[:value] <= MIN_USER_WORKING_TIME or item[:score] < avg_score}
+    working_times.sort_by! {|item| item[:value]}
+    return {:best => working_times.first(n), :worst => working_times.last(n)}
+  end
+
+  def reset_anomaly_detection_flag(collection)
+    puts "\t\tResetting flag..."
+    collection.use_anomaly_detection = false
+    collection.save!
+  end
+
+end

log/whenever/.gitignore

@@ -0,0 +1 @@
+*.log

provision.sh

@@ -2,71 +2,87 @@
 # rvm/rails installation from https://gorails.com/setup/ubuntu/14.04
 # passenger installation from https://www.phusionpassenger.com/library/install/nginx/install/oss/trusty/
 
+######## VERSION INFORMATION ########
+
+postgres_version=10
+ruby_version=2.3.6
+rails_version=4.2.10
+
+########## INSTALL SCRIPT ###########
+
+# PostgreSQL
+sudo add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+
 # passenger
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7
-apt-get install -y apt-transport-https ca-certificates
+sudo apt-get -qq -y install apt-transport-https ca-certificates
-sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list'
+sudo sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger trusty main > /etc/apt/sources.list.d/passenger.list'
 
 # rails
-add-apt-repository ppa:chris-lea/node.js
+sudo add-apt-repository -y ppa:chris-lea/node.js
 
-apt-get update
+sudo apt-get -qq update
 
 # code_ocean
-apt-get install -y postgresql-client postgresql-10 postgresql-server-dev-10 vagrant
+sudo apt-get -qq -y install postgresql-client postgresql-$postgres_version postgresql-server-dev-$postgres_version vagrant
 
 # Docker
 if [ ! -f /etc/default/docker ]
 then
-    curl -sSL https://get.docker.com/ | sh
+    curl -sSL https://get.docker.com/ | sudo sh
 fi
 if ! grep code_ocean /etc/default/docker
 then
-    cat >>/etc/default/docker <<EOF
+    sudo tee -a /etc/default/docker <<EOF
 
 # code_ocean: enable TCP
 DOCKER_OPTS="-H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock"
 EOF
-    service docker restart
+    sudo service docker restart
 fi
 
 # run docker without sudo
-sudo groupadd docker
 sudo gpasswd -a ${USER} docker
-newgrp docker
 sudo service docker restart
 
-docker pull openhpi/docker_java
+sudo docker pull openhpi/docker_java
-docker pull openhpi/docker_ruby
+sudo docker pull openhpi/docker_ruby
-docker pull openhpi/docker_python
+sudo docker pull openhpi/docker_python
-docker pull openhpi/co_execenv_python
+sudo docker pull openhpi/co_execenv_python
-docker pull openhpi/co_execenv_java
+sudo docker pull openhpi/co_execenv_java
-docker pull openhpi/co_execenv_java_antlr
+sudo docker pull openhpi/co_execenv_java_antlr
 
 # rvm
-apt-get install -y git-core curl zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libcurl4-openssl-dev python-software-properties libffi-dev
+sudo apt-get -qq -y install git-core curl zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libcurl4-openssl-dev python-software-properties libffi-dev libgdbm-dev libncurses5-dev automake libtool bison libffi-dev
-apt-get install -y libgdbm-dev libncurses5-dev automake libtool bison libffi-dev
 gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3
-curl -L https://get.rvm.io | bash -s stable
+curl -sSL https://get.rvm.io | sudo bash -s stable
+
+# access rvm installation without sudo
+sudo gpasswd -a ${USER} rvm
+
+# ruby
 source /etc/profile.d/rvm.sh
-rvm install 2.3.6
+sg rvm "rvm install $ruby_version"
-rvm use 2.3.6 --default
+rvm use $ruby_version --default
+sudo /usr/local/rvm/bin/rvm alias create default $ruby_version
 ruby -v
 
 # rails
-apt-get -y install nodejs
+sudo apt-get -qq -y install nodejs
-gem install rails -v 4.2.10
+sg rvm "/usr/local/rvm/rubies/ruby-$ruby_version/bin/gem install rails -v $rails_version"
+# sudo gem install bundler
 
 # drop postgres access control
-if  ! grep -q code_ocean /etc/postgresql/10/main/pg_hba.conf
+if  ! sudo grep -q code_ocean /etc/postgresql/$postgres_version/main/pg_hba.conf
 then
-  cat >/etc/postgresql/10/main/pg_hba.conf <<EOF
+  sudo tee /etc/postgresql/$postgres_version/main/pg_hba.conf <<EOF
 # code_ocean: drop access control
 local all all trust
 host  all all 127.0.0.1/32 trust
 host  all all ::1/128 trust
 EOF
-  service postgresql restart
+  sudo service postgresql restart
 fi
 
 # create database
@@ -74,9 +90,20 @@ if ! (sudo -u postgres psql -l | grep -q code_ocean_development)
 then
   sudo -u postgres createdb code_ocean_development || true
 fi
+if ! (sudo -u postgres psql -l | grep -q code_ocean_test)
+then
+  sudo -u postgres createdb code_ocean_test || true
+fi
+
+# Selenium tests
+sudo apt-get -qq -y install xvfb firefox
+wget --quiet -O ~/geckodriverdownload.tar.gz https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz
+sudo tar -xzf ~/geckodriverdownload.tar.gz -C /usr/local/bin
+rm ~/geckodriverdownload.tar.gz
+sudo chmod +x /usr/local/bin/geckodriver
 
 # nginx and passenger
-apt-get install -y nginx-extras passenger
+sudo apt-get -qq -y install nginx-extras passenger
 
 ############# codeocean install ###########################
 cd /vagrant
@@ -91,33 +118,36 @@ do
 done
 
 # install code
-bundle install
+sg rvm 'bundle install'
 
 # create database
 export RAILS_ENV=development
 rake db:schema:load
 rake db:migrate
-rake db:seed
+sg docker 'rake db:seed'
 sudo mkdir -p /shared
-chown -R vagrant /shared
+sudo chown -R vagrant /shared
 ln -sf /shared tmp/files #make sure you are running vagrant with admin privileges
 
 # NGINX
 if [ ! -L /etc/nginx/sites-enabled/code_ocean ]
 then
-    cat > /etc/nginx/sites-available/code_ocean <<EOF
+    sudo tee /etc/nginx/sites-available/code_ocean <<EOF
 passenger_root /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini;
 server {
     server_name codeocean.local;
     root /vagrant/public;
-    passenger_ruby /usr/local/rvm/gems/ruby-2.3.6/wrappers/ruby;
+    passenger_ruby /usr/local/rvm/gems/ruby-$ruby_version/wrappers/ruby;
     passenger_sticky_sessions on;
     passenger_enabled on;
     passenger_app_env development;
 }
 EOF
-    rm -f /etc/nginx/sites-enabled/default
+    sudo rm -f /etc/nginx/sites-enabled/default
-    ln -s /etc/nginx/sites-available/code_ocean /etc/nginx/sites-enabled
+    sudo ln -s /etc/nginx/sites-available/code_ocean /etc/nginx/sites-enabled
-    #service nginx restart
+    #sudo service nginx restart
     #cd /vagrant/ && rails s 
 fi
+
+# Always set language to English
+sudo locale-gen en_US en_US.UTF-8

spec/controllers/statistics_controller_spec.rb

@@ -0,0 +1,34 @@
+require 'rails_helper'
+
+describe StatisticsController do
+  let(:user) { FactoryBot.create(:admin) }
+  before(:each) { allow(controller).to receive(:current_user).and_return(user) }
+
+  [:show, :graphs].each do |route|
+    describe "GET ##{route}" do
+      before(:each) { get route }
+
+      expect_status(200)
+      expect_template(route)
+    end
+  end
+
+  [:user_activity_history, :rfc_activity_history].each do |route|
+    describe "GET ##{route}" do
+      before(:each) { get route }
+
+      expect_status(200)
+      expect_template(:activity_history)
+    end
+  end
+
+  [:show, :user_activity, :user_activity_history, :rfc_activity, :rfc_activity_history].each do |route|
+    describe "GET ##{route}.json" do
+      before(:each) { get route, format: :json }
+
+      expect_status(200)
+      expect_json
+    end
+  end
+
+end

spec/features/editor_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 describe 'Editor', js: true do
-  let(:exercise) { FactoryBot.create(:audio_video, instructions: Forgery(:lorem_ipsum).sentence) }
+  let(:exercise) { FactoryBot.create(:audio_video, description: Forgery(:lorem_ipsum).sentence) }
   let(:user) { FactoryBot.create(:teacher) }
 
   before(:each) do
@@ -9,94 +9,74 @@ describe 'Editor', js: true do
     fill_in('email', with: user.email)
     fill_in('password', with: FactoryBot.attributes_for(:teacher)[:password])
     click_button(I18n.t('sessions.new.link'))
+    expect_any_instance_of(LtiHelper).to receive(:lti_outcome_service?).and_return(true)
     visit(implement_exercise_path(exercise))
   end
 
-  skip "is skipped" do
+  it 'displays the exercise title' do
-    # selenium tests are currently not working locally.
+    expect(page).to have_content(exercise.title)
-    it 'displays the exercise title' do
+  end
-      expect(page).to have_content(exercise.title)
+
+  it 'displays the exercise description' do
+    expect(page).to have_content(exercise.description)
+  end
+
+  it 'displays all visible files in a file tree' do
+    within('#files') do
+      exercise.files.select(&:visible).each do |file|
+        expect(page).to have_content(file.name_with_extension)
+      end
     end
   end
 
-  describe 'Instructions Tab' do
+  it "displays the main file's code" do
-    skip "is skipped" do
+    expect(page).to have_css(".frame[data-filename='#{exercise.files.detect(&:main_file?).name_with_extension}']")
+  end
 
-    before(:each) { click_link(I18n.t('activerecord.attributes.exercise.instructions')) }
+  context 'when selecting a file' do
-
+    before(:each) do
-    it 'displays the exercise instructions' do
+      within('#files') { click_link(file.name_with_extension) }
-      expect(page).to have_content(exercise.instructions)
     end
+
+    context 'when selecting a binary file' do
+      context 'when selecting an audio file' do
+        let(:file) { exercise.files.detect { |file| file.file_type.audio? } }
+
+        it 'contains an <audio> tag' do
+          expect(page).to have_css("audio[src='#{file.native_file.url}']")
+        end
+      end
+
+      context 'when selecting an image file' do
+        let(:file) { exercise.files.detect { |file| file.file_type.image? } }
+
+        it 'contains an <img> tag' do
+          expect(page).to have_css("img[src='#{file.native_file.url}']")
+        end
+      end
+
+      context 'when selecting a video file' do
+        let(:file) { exercise.files.detect { |file| file.file_type.video? } }
+
+        it 'contains a <video> tag' do
+          expect(page).to have_css("video[src='#{file.native_file.url}']")
+        end
+      end
+    end
+
+    context 'when selecting a non-binary file' do
+      let(:file) { exercise.files.detect { |file| !file.file_type.binary? && !file.hidden? } }
+
+      it "displays the file's code" do
+        expect(page).to have_css(".frame[data-filename='#{file.name_with_extension}']")
+      end
     end
   end
 
-  describe 'Workspace Tab' do
+  it 'does not contains a button for submitting the exercise' do
-    skip "is skipped" do
+    click_button(I18n.t('exercises.editor.score'))
-
+    click_button('toggle-sidebar-output-collapsed')
-    before(:each) { click_link(I18n.t('exercises.implement.workspace')) }
+    expect(page).not_to have_css('#submit_outdated')
-
+    expect(page).to have_css('#submit')
-    it 'displays all visible files in a file tree' do
-      within('#files') do
-        exercise.files.select(&:visible).each do |file|
-          expect(page).to have_content(file.name_with_extension)
-        end
-      end
-    end
-
-    it "displays the main file's code" do
-      expect(page).to have_css(".frame[data-filename='#{exercise.files.detect(&:main_file?).name_with_extension}']")
-    end
-
-    context 'when selecting a file' do
-      before(:each) do
-        within('#files') { click_link(file.name_with_extension) }
-      end
-
-      context 'when selecting a binary file' do
-        context 'when selecting an audio file' do
-          let(:file) { exercise.files.detect { |file| file.file_type.audio? } }
-
-          it 'contains an <audio> tag' do
-            expect(page).to have_css("audio[src='#{file.native_file.url}']")
-          end
-        end
-
-        context 'when selecting an image file' do
-          let(:file) { exercise.files.detect { |file| file.file_type.image? } }
-
-          it 'contains an <img> tag' do
-            expect(page).to have_css("img[src='#{file.native_file.url}']")
-          end
-        end
-
-        context 'when selecting a video file' do
-          let(:file) { exercise.files.detect { |file| file.file_type.video? } }
-
-          it 'contains a <video> tag' do
-            expect(page).to have_css("video[src='#{file.native_file.url}']")
-          end
-        end
-      end
-
-      context 'when selecting a non-binary file' do
-        let(:file) { exercise.files.detect { |file| !file.file_type.binary? } }
-
-        it "displays the file's code" do
-          expect(page).to have_css(".frame[data-filename='#{file.name_with_extension}']")
-        end
-      end
-    end
-    end
-  end
-
-  describe 'Progress Tab' do
-    skip "is skipped" do
-      before(:each) { click_link(I18n.t('exercises.implement.progress')) }
-
-      it 'does not contains a button for submitting the exercise' do
-        # pending("the button is only displayed when an correct LTI handshake to a running course happened. This is not the case in the test")
-        expect(page).not_to have_css('#submit')
-      end
-    end
   end
 end

spec/support/selenium.rb

@@ -0,0 +1,21 @@
+require 'capybara/rspec'
+require 'selenium/webdriver'
+
+if ENV['HEADLESS_TEST'] == 'true' || ENV['USER'] == 'vagrant'
+  require 'headless'
+
+  headless = Headless.new
+  headless.start
+end
+
+Capybara.register_driver :selenium do |app|
+  profile = Selenium::WebDriver::Firefox::Profile.new
+  profile['intl.accept_languages'] = 'en'
+  capabilities = Selenium::WebDriver::Remote::Capabilities.firefox(elementScrollBehavior: 1)
+  options = Selenium::WebDriver::Firefox::Options.new
+  options.profile = profile
+  driver = Capybara::Selenium::Driver.new(app, browser: :firefox, desired_capabilities: capabilities, options: options)
+  driver.browser.manage.window.resize_to(1280, 960)
+  driver
+end
+Capybara.javascript_driver = :selenium

test/factories/structured_errors.rb

@@ -1,6 +1,6 @@
 FactoryBot.define do
   factory :structured_error do
     error_template nil
-    file nil
+    submission nil
   end
 end

test/mailers/previews/user_mailer_preview.rb

@@ -0,0 +1,7 @@
+class UserMailerPreview < ActionMailer::Preview
+  def exercise_anomaly_detected()
+    collection = ExerciseCollection.new(name: 'Hello World', user: FactoryBot.create(:admin))
+    anomalies = {49 => 879.325828, 51 => 924.870057, 31 => 1031.21233, 69 => 2159.182116}
+    UserMailer.exercise_anomaly_detected(collection, anomalies)
+  end
+end