merged with current master

.gitignore

@@ -4,9 +4,11 @@
 /config/secrets.yml
 /config/sendmail.yml
 /config/smtp.yml
-/config/docker.yml.erb
+/config/docker.yml*.erb
 /config/*.production.yml
 /config/*.staging.yml
+/config/*.staging-epic.yml
+/config/deploy/staging-epic.rb
 /coverage
 /log
 /public/assets
@@ -14,6 +16,7 @@
 /rubocop.html
 /tmp
 /vagrant/
+/.capistrano
 /.vagrant
 *.sublime-*
 /.idea

Gemfile

@@ -8,7 +8,7 @@ gem 'coffee-rails', '~> 4.0.0'
 gem 'concurrent-ruby', '~> 1.0.1'
 gem 'concurrent-ruby-ext', '~> 1.0.1', platform: :ruby
 gem 'docker-api','~> 1.25.0',  require: 'docker'
-gem 'factory_girl_rails', '~> 4.0'
+gem 'factory_bot_rails', '~> 4.8.2'
 gem 'forgery'
 gem 'highline'
 gem 'jbuilder', '~> 2.0'

Gemfile.lock

@@ -125,10 +125,10 @@ GEM
     eventmachine (1.0.9.1-java)
     excon (0.54.0)
     execjs (2.6.0)
-    factory_girl (4.5.0)
+    factory_bot (4.8.2)
       activesupport (>= 3.0.0)
-    factory_girl_rails (4.6.0)
-      factory_girl (~> 4.5.0)
+    factory_bot_rails (4.8.2)
+      factory_bot (~> 4.8.2)
       railties (>= 3.0.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
@@ -401,7 +401,7 @@ DEPENDENCIES
   d3-rails
   database_cleaner
   docker-api (~> 1.25.0)
-  factory_girl_rails (~> 4.0)
+  factory_bot_rails (~> 4.8.2)
   faye-websocket
   forgery
   highline

app/assets/javascripts/error_templates.js

@@ -0,0 +1,18 @@
+$(function() {
+    if ($.isController('error_templates')) {
+        $('#add-attribute').find('button').on('click', function () {
+            $.ajax(location + '/attribute.json', {
+                method: 'POST',
+                data: {
+                    _method: 'PUT',
+                    dataType: 'json',
+                    error_template_attribute_id: $('#add-attribute').find('select').val()
+                }
+            }).success(function () {
+                location.reload();
+            }).error(function (error) {
+                $.flash.danger({text: error.statusText});
+            });
+        });
+    }
+});

app/assets/stylesheets/error_templates.scss

@@ -0,0 +1,9 @@
+#add-attribute {
+  display: flex;
+  max-width: 400px;
+  margin-top: 30px;
+
+  button {
+    margin-left: 10px;
+  }
+}

app/assets/stylesheets/exercises.css.scss

@@ -94,3 +94,15 @@ a.file-heading {
   top: 100%;
   left: 0;
 }
+
+.feedback {
+  .text {
+    margin-bottom: 10px;
+  }
+  .difficulty {
+    font-weight: bold;
+  }
+  .worktime {
+    font-weight: bold;
+  }
+}

app/controllers/concerns/submission_scoring.rb

@@ -8,7 +8,15 @@ module SubmissionScoring
         output = execute_test_file(file, submission)
         assessment = assessor.assess(output)
         passed = ((assessment[:passed] == assessment[:count]) and (assessment[:score] > 0))
-        testrun_output = passed ? nil : output[:stderr]
+        testrun_output = passed ? nil : 'message: ' + output[:message].to_s + "\n stdout: " + output[:stdout].to_s + "\n stderr: " + output[:stderr].to_s
+        if !testrun_output.blank?
+          submission.exercise.execution_environment.error_templates.each do |template|
+            pattern = Regexp.new(template.signature).freeze
+            if pattern.match(testrun_output)
+              StructuredError.create_from_template(template, testrun_output)
+            end
+          end
+        end
         Testrun.new(submission: submission, cause: 'assess', file: file, passed: passed, output: testrun_output).save
         output.merge!(assessment)
         output.merge!(filename: file.name_with_extension, message: feedback_message(file, output[:score]), weight: file.weight)

app/controllers/error_template_attributes_controller.rb

@@ -0,0 +1,86 @@
+class ErrorTemplateAttributesController < ApplicationController
+  before_action :set_error_template_attribute, only: [:show, :edit, :update, :destroy]
+
+  def authorize!
+    authorize(@error_template_attributes || @error_template_attribute)
+  end
+  private :authorize!
+
+  # GET /error_template_attributes
+  # GET /error_template_attributes.json
+  def index
+    @error_template_attributes = ErrorTemplateAttribute.all.order('important DESC', :key, :id).paginate(page: params[:page])
+    authorize!
+  end
+
+  # GET /error_template_attributes/1
+  # GET /error_template_attributes/1.json
+  def show
+    authorize!
+  end
+
+  # GET /error_template_attributes/new
+  def new
+    @error_template_attribute = ErrorTemplateAttribute.new
+    authorize!
+  end
+
+  # GET /error_template_attributes/1/edit
+  def edit
+    authorize!
+  end
+
+  # POST /error_template_attributes
+  # POST /error_template_attributes.json
+  def create
+    @error_template_attribute = ErrorTemplateAttribute.new(error_template_attribute_params)
+    authorize!
+
+    respond_to do |format|
+      if @error_template_attribute.save
+        format.html { redirect_to @error_template_attribute, notice: 'Error template attribute was successfully created.' }
+        format.json { render :show, status: :created, location: @error_template_attribute }
+      else
+        format.html { render :new }
+        format.json { render json: @error_template_attribute.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # PATCH/PUT /error_template_attributes/1
+  # PATCH/PUT /error_template_attributes/1.json
+  def update
+    authorize!
+    respond_to do |format|
+      if @error_template_attribute.update(error_template_attribute_params)
+        format.html { redirect_to @error_template_attribute, notice: 'Error template attribute was successfully updated.' }
+        format.json { render :show, status: :ok, location: @error_template_attribute }
+      else
+        format.html { render :edit }
+        format.json { render json: @error_template_attribute.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # DELETE /error_template_attributes/1
+  # DELETE /error_template_attributes/1.json
+  def destroy
+    authorize!
+    @error_template_attribute.destroy
+    respond_to do |format|
+      format.html { redirect_to error_template_attributes_url, notice: 'Error template attribute was successfully destroyed.' }
+      format.json { head :no_content }
+    end
+  end
+
+  private
+    # Use callbacks to share common setup or constraints between actions.
+    def set_error_template_attribute
+      @error_template_attribute = ErrorTemplateAttribute.find(params[:id])
+    end
+
+    # Never trust parameters from the scary internet, only allow the white list through.
+    def error_template_attribute_params
+      params[:error_template_attribute].permit(:key, :description, :regex, :important)
+    end
+end

app/controllers/error_templates_controller.rb

@@ -0,0 +1,104 @@
+class ErrorTemplatesController < ApplicationController
+  before_action :set_error_template, only: [:show, :edit, :update, :destroy, :add_attribute, :remove_attribute]
+
+  def authorize!
+    authorize(@error_templates || @error_template)
+  end
+  private :authorize!
+
+  # GET /error_templates
+  # GET /error_templates.json
+  def index
+    @error_templates = ErrorTemplate.all.order(:execution_environment_id, :name).paginate(page: params[:page])
+    authorize!
+  end
+
+  # GET /error_templates/1
+  # GET /error_templates/1.json
+  def show
+    authorize!
+  end
+
+  # GET /error_templates/new
+  def new
+    @error_template = ErrorTemplate.new
+    authorize!
+  end
+
+  # GET /error_templates/1/edit
+  def edit
+    authorize!
+  end
+
+  # POST /error_templates
+  # POST /error_templates.json
+  def create
+    @error_template = ErrorTemplate.new(error_template_params)
+    authorize!
+
+    respond_to do |format|
+      if @error_template.save
+        format.html { redirect_to @error_template, notice: 'Error template was successfully created.' }
+        format.json { render :show, status: :created, location: @error_template }
+      else
+        format.html { render :new }
+        format.json { render json: @error_template.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # PATCH/PUT /error_templates/1
+  # PATCH/PUT /error_templates/1.json
+  def update
+    authorize!
+    respond_to do |format|
+      if @error_template.update(error_template_params)
+        format.html { redirect_to @error_template, notice: 'Error template was successfully updated.' }
+        format.json { render :show, status: :ok, location: @error_template }
+      else
+        format.html { render :edit }
+        format.json { render json: @error_template.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # DELETE /error_templates/1
+  # DELETE /error_templates/1.json
+  def destroy
+    authorize!
+    @error_template.destroy
+    respond_to do |format|
+      format.html { redirect_to error_templates_url, notice: 'Error template was successfully destroyed.' }
+      format.json { head :no_content }
+    end
+  end
+
+  def add_attribute
+    authorize!
+    @error_template.error_template_attributes << ErrorTemplateAttribute.find(params['error_template_attribute_id'])
+    respond_to do |format|
+      format.html { redirect_to @error_template }
+      format.json { head :no_content }
+    end
+  end
+
+  def remove_attribute
+    authorize!
+    @error_template.error_template_attributes.delete(ErrorTemplateAttribute.find(params['error_template_attribute_id']))
+    respond_to do |format|
+      format.html { redirect_to @error_template }
+      format.json { head :no_content }
+    end
+  end
+
+  private
+    # Use callbacks to share common setup or constraints between actions.
+    def set_error_template
+      @error_template = ErrorTemplate.find(params[:id])
+    end
+
+    # Never trust parameters from the scary internet, only allow the white list through.
+    def error_template_params
+      params[:error_template].permit(:name, :execution_environment_id, :signature, :description, :hint)
+    end
+end

app/controllers/exercise_collections_controller.rb

@@ -0,0 +1,51 @@
+class ExerciseCollectionsController < ApplicationController
+  include CommonBehavior
+
+  before_action :set_exercise_collection, only: [:show, :edit, :update, :destroy]
+
+  def index
+    @exercise_collections = ExerciseCollection.all.paginate(:page => params[:page])
+    authorize!
+  end
+
+  def show
+  end
+
+  def new
+    @exercise_collection = ExerciseCollection.new
+    authorize!
+  end
+
+  def create
+    @exercise_collection = ExerciseCollection.new(exercise_collection_params)
+    authorize!
+    create_and_respond(object: @exercise_collection)
+  end
+
+  def destroy
+    authorize!
+    destroy_and_respond(object: @exercise_collection)
+  end
+
+  def edit
+  end
+
+  def update
+    update_and_respond(object: @exercise_collection, params: exercise_collection_params)
+  end
+
+  private
+
+  def set_exercise_collection
+    @exercise_collection = ExerciseCollection.find(params[:id])
+    authorize!
+  end
+
+  def authorize!
+    authorize(@exercise_collection || @exercise_collections)
+  end
+
+  def exercise_collection_params
+    params[:exercise_collection].permit(:name, :exercise_ids => [])
+  end
+end

app/controllers/exercises_controller.rb

@@ -6,7 +6,7 @@ class ExercisesController < ApplicationController
 
   before_action :handle_file_uploads, only: [:create, :update]
   before_action :set_execution_environments, only: [:create, :edit, :new, :update]
-  before_action :set_exercise, only: MEMBER_ACTIONS + [:clone, :implement, :working_times, :intervention, :search, :run, :statistics, :submit, :reload]
+  before_action :set_exercise, only: MEMBER_ACTIONS + [:clone, :implement, :working_times, :intervention, :search, :run, :statistics, :submit, :reload, :feedback]
   before_action :set_external_user, only: [:statistics]
   before_action :set_file_types, only: [:create, :edit, :new, :update]
   before_action :set_course_token, only: [:implement]
@@ -28,7 +28,6 @@ class ExercisesController < ApplicationController
     1
   end
 
-
   def java_course_token
     "702cbd2a-c84c-4b37-923a-692d7d1532d0"
   end
@@ -94,6 +93,11 @@ class ExercisesController < ApplicationController
     collect_set_and_unset_exercise_tags
   end
 
+  def feedback
+    authorize!
+    @feedbacks = @exercise.user_exercise_feedbacks.paginate(page: params[:page])
+  end
+
   def import_proforma_xml
     begin
       user = user_for_oauth2_request()
@@ -147,8 +151,7 @@ class ExercisesController < ApplicationController
   private :user_by_code_harbor_token
 
   def exercise_params
-    params[:exercise][:expected_worktime_seconds] = params[:exercise][:expected_worktime_minutes].to_i * 60
-    params[:exercise].permit(:description, :execution_environment_id, :file_id, :instructions, :public, :hide_file_tree, :allow_file_creation, :allow_auto_completion, :title, :expected_difficulty, :expected_worktime_seconds, files_attributes: file_attributes, :tag_ids => []).merge(user_id: current_user.id, user_type: current_user.class.name)
+    params[:exercise].permit(:description, :execution_environment_id, :file_id, :instructions, :public, :hide_file_tree, :allow_file_creation, :allow_auto_completion, :title, :expected_difficulty, files_attributes: file_attributes, :tag_ids => []).merge(user_id: current_user.id, user_type: current_user.class.name)
   end
   private :exercise_params
 
@@ -388,10 +391,13 @@ class ExercisesController < ApplicationController
       # otherwise an internal user could be shown a false rfc here, since current_user.id is polymorphic, but only makes sense for external users when used with rfcs.)
       # redirect 10 percent pseudorandomly to the feedback page
       if current_user.respond_to? :external_id
-        if ((current_user.id + @submission.exercise.created_at.to_i) % 10 == 1)
+        if @submission.redirect_to_feedback?
           redirect_to_user_feedback
           return
-        elsif rfc = RequestForComment.unsolved.where(exercise_id: @submission.exercise, user_id: current_user.id).first
+        end
+
+        rfc = @submission.own_unsolved_rfc
+        if rfc
           # set a message that informs the user that his own RFC should be closed.
           flash[:notice] = I18n.t('exercises.submit.full_score_redirect_to_own_rfc')
           flash.keep(:notice)
@@ -401,24 +407,30 @@ class ExercisesController < ApplicationController
             format.json { render(json: {redirect: url_for(rfc)}) }
           end
           return
+        end
 
         # else: show open rfc for same exercise if available
-        elsif rfc = RequestForComment.unsolved.where(exercise_id: @submission.exercise).where.not(question: nil).order("RANDOM()").find { | rfc_element |(rfc_element.comments_count < 5) }
+        rfc = @submission.unsolved_rfc
+        unless rfc.nil?
           # set a message that informs the user that his score was perfect and help in RFC is greatly appreciated.
           flash[:notice] = I18n.t('exercises.submit.full_score_redirect_to_rfc')
           flash.keep(:notice)
 
           respond_to do |format|
-            format.html { redirect_to(rfc) }
-            format.json { render(json: {redirect: url_for(rfc)}) }
+            format.html {redirect_to(rfc)}
+            format.json {render(json: {redirect: url_for(rfc)})}
           end
           return
         end
       end
     else
       # redirect to feedback page if score is less than 100 percent
-       redirect_to_user_feedback
-      return
+       if @exercise.needs_more_feedback?
+         redirect_to_user_feedback
+       else
+         redirect_to_lti_return_path
+       end
+       return
     end
     redirect_to_lti_return_path
   end

app/controllers/submissions_controller.rb

@@ -6,7 +6,7 @@ class SubmissionsController < ApplicationController
   include SubmissionScoring
   include Tubesock::Hijack
 
-  before_action :set_submission, only: [:download, :download_file, :render_file, :run, :score, :show, :statistics, :stop, :test]
+  before_action :set_submission, only: [:download, :download_file, :render_file, :run, :score, :extract_errors, :show, :statistics, :stop, :test]
   before_action :set_docker_client, only: [:run, :test]
   before_action :set_files, only: [:download, :download_file, :render_file, :show]
   before_action :set_file, only: [:download_file, :render_file]
@@ -191,6 +191,9 @@ class SubmissionsController < ApplicationController
   end
 
   def kill_socket(tubesock)
+    # search for errors and save them as StructuredError (for scoring runs see submission_scoring.rb)
+    extract_errors
+
     # save the output of this "run" as a "testrun" (scoring runs are saved in submission_scoring.rb)
     save_run_output
 
@@ -199,6 +202,17 @@ class SubmissionsController < ApplicationController
     tubesock.close
   end
 
+  def extract_errors
+    if !@message_buffer.blank?
+      @submission.exercise.execution_environment.error_templates.each do |template|
+        pattern = Regexp.new(template.signature).freeze
+        if pattern.match(@message_buffer)
+          StructuredError.create_from_template(template, @message_buffer)
+        end
+      end
+    end
+  end
+
   def handle_message(message, tubesock, container)
     @run_output ||= ""
     # Handle special commands first

app/controllers/user_exercise_feedbacks_controller.rb

@@ -2,6 +2,7 @@ class UserExerciseFeedbacksController < ApplicationController
   include CommonBehavior
 
   before_action :set_user_exercise_feedback, only: [:edit, :update]
+  before_action :set_user_exercise_feedback_by_id, only: [:show, :destroy]
 
   def comment_presets
     [[0,t('user_exercise_feedback.difficulty_easy')],
@@ -19,10 +20,15 @@ class UserExerciseFeedbacksController < ApplicationController
      [4,t('user_exercise_feedback.estimated_time_more_30')]]
   end
 
-  def authorize!
-    authorize(@uef)
+  def index
+    @search = UserExerciseFeedback.all.search params[:q]
+    @uefs = @search.result.includes(:execution_environment).order(:id).paginate(page: params[:page])
+    authorize!
+  end
+
+  def show
+    authorize!
   end
-  private :authorize!
 
   def create
     @exercise = Exercise.find(uef_params[:exercise_id])
@@ -49,7 +55,8 @@ class UserExerciseFeedbacksController < ApplicationController
   end
 
   def destroy
-    destroy_and_respond(object: @tag)
+    authorize!
+    destroy_and_respond(object: @uef)
   end
 
   def edit
@@ -58,11 +65,6 @@ class UserExerciseFeedbacksController < ApplicationController
     authorize!
   end
 
-  def uef_params
-    params[:user_exercise_feedback].permit(:feedback_text, :difficulty, :exercise_id, :user_estimated_worktime).merge(user_id: current_user.id, user_type: current_user.class.name)
-  end
-  private :uef_params
-
   def new
     @texts = comment_presets.to_a
     @times = time_presets.to_a
@@ -89,6 +91,12 @@ class UserExerciseFeedbacksController < ApplicationController
     end
   end
 
+  private
+
+  def authorize!
+    authorize(@uef || @uefs)
+  end
+
   def to_s
     name
   end
@@ -98,6 +106,14 @@ class UserExerciseFeedbacksController < ApplicationController
     @uef = UserExerciseFeedback.find_by(exercise_id: params[:user_exercise_feedback][:exercise_id], user: current_user)
   end
 
+  def set_user_exercise_feedback_by_id
+    @uef = UserExerciseFeedback.find(params[:id])
+  end
+
+  def uef_params
+    params[:user_exercise_feedback].permit(:feedback_text, :difficulty, :exercise_id, :user_estimated_worktime).merge(user_id: current_user.id, user_type: current_user.class.name)
+  end
+
   def validate_inputs(uef_params)
     begin
       if uef_params[:difficulty].to_i < 0 || uef_params[:difficulty].to_i >= comment_presets.size

app/helpers/error_template_attributes_helper.rb

@@ -0,0 +1,2 @@
+module ErrorTemplateAttributesHelper
+end

app/helpers/error_templates_helper.rb

@@ -0,0 +1,2 @@
+module ErrorTemplatesHelper
+end

app/models/error_template.rb

@@ -0,0 +1,8 @@
+class ErrorTemplate < ActiveRecord::Base
+  belongs_to :execution_environment
+  has_and_belongs_to_many :error_template_attributes
+
+  def to_s
+    "#{id} [#{name}]"
+  end
+end

app/models/error_template_attribute.rb

@@ -0,0 +1,7 @@
+class ErrorTemplateAttribute < ActiveRecord::Base
+  has_and_belongs_to_many :error_template
+
+  def to_s
+    "#{id} [#{key}]"
+  end
+end

app/models/execution_environment.rb

@@ -11,6 +11,7 @@ class ExecutionEnvironment < ActiveRecord::Base
   has_many :exercises
   belongs_to :file_type
   has_many :hints
+  has_many :error_templates
 
   scope :with_exercises, -> { where('id IN (SELECT execution_environment_id FROM exercises)') }
 

app/models/exercise.rb

@@ -20,6 +20,7 @@ class Exercise < ActiveRecord::Base
   has_many :exercise_tags
   has_many :tags, through: :exercise_tags
   accepts_nested_attributes_for :exercise_tags
+  has_many :user_exercise_feedbacks
 
   has_many :external_users, source: :user, source_type: ExternalUser, through: :submissions
   has_many :internal_users, source: :user, source_type: InternalUser, through: :submissions
@@ -36,6 +37,8 @@ class Exercise < ActiveRecord::Base
 
   @working_time_statistics = nil
 
+  MAX_EXERCISE_FEEDBACKS = 20
+
 
   def average_percentage
     if average_score and maximum_score != 0.0 and submissions.exists?(cause: 'submit')
@@ -361,4 +364,8 @@ class Exercise < ActiveRecord::Base
   end
   private :valid_main_file?
 
+  def needs_more_feedback
+    user_exercise_feedbacks.size <= MAX_EXERCISE_FEEDBACKS
+  end
+
 end

app/models/exercise_collection.rb

@@ -2,4 +2,8 @@ class ExerciseCollection < ActiveRecord::Base
 
   has_and_belongs_to_many :exercises
 
+  def to_s
+    "#{I18n.t('activerecord.models.exercise_collection.one')}: #{name} (#{id})"
+  end
+
 end

app/models/proxy_exercise.rb

@@ -36,8 +36,8 @@ class ProxyExercise < ActiveRecord::Base
           Rails.logger.debug("retrieved assigned exercise for user #{user.id}: Exercise #{assigned_user_proxy_exercise.exercise}" )
           assigned_user_proxy_exercise.exercise
         else
+          Rails.logger.debug("find new matching exercise for user #{user.id}" )
           matching_exercise =
-              Rails.logger.debug("find new matching exercise for user #{user.id}" )
               begin
                 find_matching_exercise(user)
               rescue => e #fallback
@@ -72,7 +72,7 @@ class ProxyExercise < ActiveRecord::Base
 
           # find exercises
           potential_recommended_exercises = []
-          exercises.where("expected_difficulty > 1").each do |ex|
+          exercises.where("expected_difficulty >= 1").each do |ex|
             ## find exercises which have only tags the user has already seen
             if (ex.tags - tags_user_has_seen).empty?
               potential_recommended_exercises << ex
@@ -85,8 +85,7 @@ class ProxyExercise < ActiveRecord::Base
             @reason[:reason] = "easiest exercise in pool. empty potential exercises"
             select_easiest_exercise(exercises)
           else
-            recommended_exercise = select_best_matching_exercise(user, exercises_user_has_accessed, potential_recommended_exercises)
-            recommended_exercise
+            select_best_matching_exercise(user, exercises_user_has_accessed, potential_recommended_exercises)
           end
       end
 

app/models/structured_error.rb

@@ -0,0 +1,12 @@
+class StructuredError < ActiveRecord::Base
+  belongs_to :error_template
+  belongs_to :file, class_name: 'CodeOcean::File'
+
+  def self.create_from_template(template, message_buffer)
+    instance = self.create(error_template: template)
+    template.error_template_attributes.each do |attribute|
+      StructuredErrorAttribute.create_from_template(attribute, instance, message_buffer)
+    end
+    instance
+  end
+end

app/models/structured_error_attribute.rb

@@ -0,0 +1,17 @@
+class StructuredErrorAttribute < ActiveRecord::Base
+  belongs_to :structured_error
+  belongs_to :error_template_attribute
+
+  def self.create_from_template(attribute, structured_error, message_buffer)
+    match = false
+    value = nil
+    result = message_buffer.match(attribute.regex)
+    if result != nil
+      match = true
+      if result.captures.size > 0
+        value = result.captures[0]
+      end
+    end
+    self.create(structured_error: structured_error, error_template_attribute: attribute, value: value, match: match)
+  end
+end

app/models/submission.rb

@@ -18,6 +18,8 @@ class Submission < ActiveRecord::Base
   validates :cause, inclusion: {in: CAUSES}
   validates :exercise_id, presence: true
 
+  MAX_COMMENTS_ON_RECOMMENDED_RFC = 5
+
   def build_files_hash(files, attribute)
     files.map(&attribute.to_proc).zip(files).to_h
   end
@@ -53,4 +55,16 @@ class Submission < ActiveRecord::Base
   def to_s
     Submission.model_name.human
   end
+
+  def redirect_to_feedback?
+    ((user_id + exercise.created_at.to_i) % 10 == 1) && exercise.needs_more_feedback
+  end
+
+  def own_unsolved_rfc
+    RequestForComment.unsolved.where(exercise_id: exercise, user_id: user_id).first
+  end
+
+  def unsolved_rfc
+    RequestForComment.unsolved.where(exercise_id: exercise).where.not(question: nil).order("RANDOM()").find { | rfc_element |(rfc_element.comments_count < MAX_COMMENTS_ON_RECOMMENDED_RFC) }
+  end
 end

app/models/user_exercise_feedback.rb

@@ -2,6 +2,7 @@ class UserExerciseFeedback < ActiveRecord::Base
   include Creation
 
   belongs_to :exercise
+  has_one :execution_environment, through: :exercise
 
   validates :user_id, uniqueness: { scope: [:exercise_id, :user_type] }
 

app/policies/error_template_attribute_policy.rb

@@ -0,0 +1,3 @@
+class ErrorTemplateAttributePolicy < AdminOnlyPolicy
+
+end

app/policies/error_template_policy.rb

@@ -0,0 +1,9 @@
+class ErrorTemplatePolicy < AdminOnlyPolicy
+  def add_attribute?
+    admin?
+  end
+
+  def remove_attribute?
+    admin?
+  end
+end

app/policies/exercise_collection_policy.rb

@@ -0,0 +1,3 @@
+class ExerciseCollectionPolicy < AdminOnlyPolicy
+
+end

app/policies/exercise_policy.rb

@@ -12,7 +12,7 @@ class ExercisePolicy < AdminOrAuthorPolicy
     @user.internal_user?
   end
 
-  [:clone?, :destroy?, :edit?, :statistics?, :update?].each do |action|
+  [:clone?, :destroy?, :edit?, :statistics?, :update?, :feedback?].each do |action|
     define_method(action) { admin? || author?}
   end
 

app/policies/user_exercise_feedback_policy.rb

@@ -1,8 +1,4 @@
-class UserExerciseFeedbackPolicy < ApplicationPolicy
-  def author?
-    @user == @record.author
-  end
-  private :author?
+class UserExerciseFeedbackPolicy < AdminOrAuthorPolicy
 
   def create?
     everyone
@@ -12,8 +8,4 @@ class UserExerciseFeedbackPolicy < ApplicationPolicy
     everyone
   end
 
-  [:show? ,:destroy?, :edit?, :update?].each do |action|
-    define_method(action) { admin? || author?}
-  end
-
 end

app/views/application/_navigation.html.slim

@@ -8,7 +8,8 @@
         - if current_user.admin?
           li = link_to(t('breadcrumbs.dashboard.show'), admin_dashboard_path)
           li.divider
-        - models = [ExecutionEnvironment, Exercise, ProxyExercise, Tag, Consumer, CodeHarborLink, ExternalUser, FileType, FileTemplate, InternalUser].sort_by { |model| model.model_name.human(count: 2) }
+        - models = [ExecutionEnvironment, Exercise, ExerciseCollection, ProxyExercise, Tag, Consumer, CodeHarborLink, UserExerciseFeedback,
+                ErrorTemplate, ErrorTemplateAttribute, ExternalUser, FileType, FileTemplate, InternalUser].sort_by {|model| model.model_name.human(count: 2) }
         - models.each do |model|
           - if policy(model).index?
             li = link_to(model.model_name.human(count: 2), send(:"#{model.model_name.collection}_path"))

app/views/error_template_attributes/_form.html.slim

@@ -0,0 +1,16 @@
+= form_for(@error_template_attribute) do |f|
+  = render('shared/form_errors', object: @error_template_attribute)
+  .form-group
+    = f.label(:key)
+    = f.text_field(:key, class: 'form-control', required: true)
+  .form-group
+    = f.label(:description)
+    = f.text_field(:description, class: 'form-control')
+  .form-group
+    = f.label(:regex)
+    = f.text_field(:regex, class: 'form-control', required: true)
+    .help-block == t('error_templates.hints.signature')
+  .form-group
+    = f.check_box(:important)
+    = t('activerecord.attributes.error_template_attribute.important')
+  .actions = render('shared/submit_button', f: f, object: @error_template_attribute)

app/views/error_template_attributes/edit.html.slim

@@ -0,0 +1,3 @@
+h1 = @error_template_attribute
+
+= render('form')

app/views/error_template_attributes/index.html.slim

@@ -0,0 +1,28 @@
+h1 = ErrorTemplateAttribute.model_name.human(count: 2)
+
+.table-responsive
+  table.sortable.table
+    thead
+      tr
+        th
+        th = t('activerecord.attributes.error_template_attribute.key')
+        th = t('activerecord.attributes.error_template_attribute.description')
+        th = t('activerecord.attributes.error_template_attribute.regex')
+        th colspan=5 = t('shared.actions')
+    tbody
+      - @error_template_attributes.each do |error_template_attribute|
+        tr
+          td
+            - if error_template_attribute.important
+              span class="fa fa-star" aria-hidden="true"
+            - else
+              span class="fa fa-star-o" aria-hidden="true"
+          td = error_template_attribute.key
+          td = error_template_attribute.description
+          td = error_template_attribute.regex
+          td = link_to(t('shared.show'), error_template_attribute)
+          td = link_to(t('shared.edit'), edit_error_template_attribute_path(error_template_attribute))
+          td = link_to(t('shared.destroy'), error_template_attribute, data: {confirm: t('shared.confirm_destroy')}, method: :delete)
+
+= render('shared/pagination', collection: @error_template_attributes)
+p = render('shared/new_button', model: ErrorTemplateAttribute)

app/views/error_template_attributes/new.html.slim

@@ -0,0 +1,3 @@
+h1 = t('shared.new_model', model: ErrorTemplateAttribute.model_name.human)
+
+= render('form')

app/views/error_template_attributes/show.html.slim

@@ -0,0 +1,8 @@
+h1
+  = @error_template_attribute
+  = render('shared/edit_button', object: @error_template_attribute)
+
+- [:key, :description, :regex, :important].each do |attribute|
+  = row(label: "error_template_attribute.#{attribute}", value: @error_template_attribute.send(attribute))
+
+// todo: used by

app/views/error_templates/_form.html.slim

@@ -0,0 +1,19 @@
+= form_for(@error_template) do |f|
+  = render('shared/form_errors', object: @error_template)
+  .form-group
+    = f.label(:name)
+    = f.text_field(:name, class: 'form-control', required: true)
+  .form-group
+    = f.label(:execution_environment_id)
+    = f.collection_select(:execution_environment_id, ExecutionEnvironment.all.order(:name), :id, :name, {include_blank: false}, class: 'form-control')
+  .form-group
+    = f.label(:signature)
+    = f.text_field(:signature, class: 'form-control')
+    .help-block == t('error_templates.hints.signature')
+  .form-group
+    = f.label(:description)
+    = f.text_field(:description, class: 'form-control')
+  .form-group
+    = f.label(:hint)
+    = f.text_field(:hint, class: 'form-control')
+  .actions = render('shared/submit_button', f: f, object: @error_template)

app/views/error_templates/edit.html.slim

@@ -0,0 +1,3 @@
+h1 = @error_template
+
+= render('form')

app/views/error_templates/index.html.slim

@@ -0,0 +1,22 @@
+h1 = ErrorTemplate.model_name.human(count: 2)
+
+.table-responsive
+  table.sortable.table
+    thead
+      tr
+        th = t('activerecord.attributes.error_template.name')
+        th = t('activerecord.attributes.error_template.description')
+        th = t('activerecord.attributes.exercise.execution_environment')
+        th colspan=3 = t('shared.actions')
+    tbody
+      - @error_templates.each do |error_template|
+        tr
+          td = error_template.name
+          td = error_template.description
+          td = link_to(error_template.execution_environment)
+          td = link_to(t('shared.show'), error_template)
+          td = link_to(t('shared.edit'), edit_error_template_path(error_template))
+          td = link_to(t('shared.destroy'), error_template, data: {confirm: t('shared.confirm_destroy')}, method: :delete)
+
+= render('shared/pagination', collection: @error_templates)
+p = render('shared/new_button', model: ErrorTemplate)

app/views/error_templates/new.html.slim

@@ -0,0 +1,3 @@
+h1 = t('shared.new_model', model: ErrorTemplate.model_name.human)
+
+= render('form')

app/views/error_templates/show.html.slim

@@ -0,0 +1,40 @@
+h1
+  = @error_template
+  = render('shared/edit_button', object: @error_template)
+
+= row(label: 'error_template.name', value: @error_template.name)
+= row(label: 'exercise.execution_environment', value: link_to(@error_template.execution_environment))
+- [:signature, :description, :hint].each do |attribute|
+  = row(label: "error_template.#{attribute}", value: @error_template.send(attribute))
+
+h3
+  = t 'error_templates.attributes'
+
+.table-responsive
+  table.sortable.table
+    thead
+      tr
+        th
+        th = t('activerecord.attributes.error_template_attribute.key')
+        th = t('activerecord.attributes.error_template_attribute.description')
+        th = t('activerecord.attributes.error_template_attribute.regex')
+        th colspan=3 = t('shared.actions')
+    tbody
+      - @error_template.error_template_attributes.order('important DESC', :key).each do |attribute|
+        tr
+          td
+            - if attribute.important
+              span class="fa fa-star" aria-hidden="true"
+            - else
+              span class="fa fa-star-o" aria-hidden="true"
+          td = attribute.key
+          td = attribute.description
+          td = attribute.regex
+          td = link_to(t('shared.show'), attribute)
+          td = link_to(t('shared.destroy'), attribute_error_template_url(:error_template_attribute_id => attribute.id), :method => :delete)
+
+#add-attribute
+  = collection_select({}, :error_template_attribute_id,
+          ErrorTemplateAttribute.where.not(id: @error_template.error_template_attributes.select(:id).to_a).order('important DESC', :key),
+          :id, :key, {include_blank: false}, class: '')
+  button.btn.btn-default = t('error_templates.add_attribute')

app/views/exercise_collections/_form.html.slim

@@ -0,0 +1,11 @@
+- exercises = Exercise.order(:title)
+
+= form_for(@exercise_collection, data: {exercises: exercises}, multipart: true) do |f|
+  = render('shared/form_errors', object: @exercise_collection)
+  .form-group
+    = f.label(:name)
+    = f.text_field(:name, class: 'form-control', required: true)
+  .form-group
+    = f.label(:exercises)
+    = f.collection_select(:exercise_ids, exercises, :id, :title, {}, {class: 'form-control', multiple: true})
+  .actions = render('shared/submit_button', f: f, object: @exercise_collection)

app/views/exercise_collections/edit.html.slim

@@ -0,0 +1,3 @@
+h1 = @exercise_collection
+
+= render('form')

app/views/exercise_collections/index.html.slim

@@ -0,0 +1,24 @@
+h1 = ExerciseCollection.model_name.human(count: 2)
+
+.table-responsive
+  table.table
+    thead
+      tr
+        th = t('activerecord.attributes.exercise_collections.id')
+        th = t('activerecord.attributes.exercise_collections.name')
+        th = t('activerecord.attributes.exercise_collections.updated_at')
+        th = t('activerecord.attributes.exercise_collections.exercises')
+        th colspan=3 = t('shared.actions')
+    tbody
+      - @exercise_collections.each do |collection|
+        tr
+          td = collection.id
+          td = link_to(collection.name, collection)
+          td = collection.updated_at
+          td = collection.exercises.size
+          td = link_to(t('shared.show'), collection)
+          td = link_to(t('shared.edit'), edit_exercise_collection_path(collection))
+          td = link_to(t('shared.destroy'), collection, data: {confirm: t('shared.confirm_destroy')}, method: :delete)
+
+= render('shared/pagination', collection: @exercise_collections)
+p = render('shared/new_button', model: ExerciseCollection)

app/views/exercise_collections/new.html.slim

@@ -0,0 +1,3 @@
+h1 = t('shared.new_model', model: ExerciseCollection.model_name.human)
+
+= render('form')

app/views/exercise_collections/show.html.slim

@@ -0,0 +1,11 @@
+h1
+  = @exercise_collection
+  = render('shared/edit_button', object: @exercise_collection)
+
+= row(label: 'exercise_collections.name', value: @exercise_collection.name)
+= row(label: 'exercise_collections.updated_at', value: @exercise_collection.updated_at)
+
+h4 = t('activerecord.attributes.exercise_collections.exercises')
+ul.list-unstyled
+  - @exercise_collection.exercises.sort_by{|c| c.title}.each do |exercise|
+    li = link_to(exercise, exercise)

app/views/exercises/_editor.html.slim

@@ -3,7 +3,7 @@
 - consumer_id = @current_user.respond_to?(:external_id) ? @current_user.consumer_id : '' #'tests' #(@current_user.uuid.present? ? @current_user.uuid : '')
 - show_break_interventions = @show_break_interventions || "false"
 - show_rfc_interventions = @show_rfc_interventions || "false"
-#editor.row data-exercise-id=exercise.id data-message-depleted=t('exercises.editor.depleted') data-message-timeout=t('exercises.editor.timeout',  permitted_execution_time: @exercise.execution_environment.permitted_execution_time) data-errors-url=execution_environment_errors_path(exercise.execution_environment) data-submissions-url=submissions_path data-user-id=@current_user.id data-user-external-id=external_user_external_id data-working-times-url=working_times_exercise_path data-intervention-save-url=intervention_exercise_path data-rfc-interventions=show_rfc_interventions data-break-interventions=show_break_interventions data-course_token=@course_token data-search-save-url=search_exercise_path
+#editor.row data-exercise-id=@exercise.id data-message-depleted=t('exercises.editor.depleted') data-message-timeout=t('exercises.editor.timeout',  permitted_execution_time: @exercise.execution_environment.permitted_execution_time) data-errors-url=execution_environment_errors_path(exercise.execution_environment) data-submissions-url=submissions_path data-user-id=@current_user.id data-user-external-id=external_user_external_id data-working-times-url=working_times_exercise_path(@exercise) data-intervention-save-url=intervention_exercise_path(@exercise) data-rfc-interventions=show_rfc_interventions data-break-interventions=show_break_interventions data-course_token=@course_token data-search-save-url=search_exercise_path(@exercise)
   div id="sidebar" class=(@exercise.hide_file_tree ? 'sidebar-col-collapsed' : 'sidebar-col') = render('editor_file_tree', exercise: @exercise, files: @files)
   div id='output_sidebar' class='output-col-collapsed' = render('exercises/editor_output', external_user_id: external_user_id, consumer_id: consumer_id )
   div id='frames' class='editor-col'

app/views/exercises/_form.html.slim

@@ -35,9 +35,6 @@
   .form-group
     = f.label(t('activerecord.attributes.exercise.difficulty'))
     = f.number_field :expected_difficulty, in: 1..10, step: 1
-  .form-group
-    = f.label(t('activerecord.attributes.exercise.worktime'))
-    = f.number_field "expected_worktime_minutes", value: @exercise.expected_worktime_seconds / 60, in: 1..1000, step: 1
 
   h2 = t('exercises.form.tags')
   ul.list-unstyled.panel-group

app/views/exercises/feedback.html.slim

@@ -0,0 +1,15 @@
+h1 = @exercise
+
+ul.list-unstyled.panel-group#files
+  - @feedbacks.each do |feedback|
+    li.panel.panel-default
+      .panel-heading role="tab" id="heading"
+        div.clearfix
+          span = feedback.user.name
+      .panel-collapse role="tabpanel"
+        .panel-body.feedback
+          .text = feedback.feedback_text
+          .difficulty = "#{t('user_exercise_feedback.difficulty')} #{feedback.difficulty}" if feedback.difficulty
+          .worktime = "#{t('user_exercise_feedback.working_time')} #{feedback.user_estimated_worktime}" if feedback.user_estimated_worktime
+
+= render('shared/pagination', collection: @feedbacks)

app/views/exercises/index.html.slim

@@ -18,7 +18,6 @@ h1 = Exercise.model_name.human(count: 2)
         th = t('activerecord.attributes.exercise.maximum_score')
         th = t('activerecord.attributes.exercise.tags')
         th = t('activerecord.attributes.exercise.difficulty')
-        th = t('activerecord.attributes.exercise.worktime')
         th
           = t('activerecord.attributes.exercise.public')
           - if policy(Exercise).batch_update?
@@ -34,7 +33,6 @@ h1 = Exercise.model_name.human(count: 2)
           td = exercise.maximum_score
           td = exercise.exercise_tags.count
           td = exercise.expected_difficulty
-          td = (exercise.expected_worktime_seconds / 60).ceil
           td.public data-value=exercise.public? = symbol_for(exercise.public?)
           td = link_to(t('shared.edit'), edit_exercise_path(exercise)) if policy(exercise).edit?
           td = link_to(t('.implement'), implement_exercise_path(exercise)) if policy(exercise).implement?
@@ -47,6 +45,7 @@ h1 = Exercise.model_name.human(count: 2)
               span.sr-only Toggle Dropdown
               ul.dropdown-menu.pull-right role="menu"
                 li = link_to(t('shared.show'), exercise) if policy(exercise).show?
+                li = link_to(t('activerecord.models.user_exercise_feedback.other'), feedback_exercise_path(exercise)) if policy(exercise).feedback?
                 li = link_to(t('shared.destroy'), exercise, data: {confirm: t('shared.confirm_destroy')}, method: :delete) if policy(exercise).destroy?
                 li = link_to(t('.clone'), clone_exercise_path(exercise), data: {confirm: t('shared.confirm_destroy')}, method: :post) if policy(exercise).clone?
 

app/views/exercises/show.html.slim

@@ -20,7 +20,6 @@ h1
 = row(label: 'exercise.embedding_parameters') do
   = content_tag(:input, nil, class: 'form-control', readonly: true, value: embedding_parameters(@exercise))
 = row(label: 'exercise.difficulty', value: @exercise.expected_difficulty)
-= row(label: 'exercise.worktime', value: "#{@exercise.expected_worktime_seconds/60} min")
 = row(label: 'exercise.tags', value: @exercise.exercise_tags.map{|et| "#{et.tag.name} (#{et.factor})"}.sort.join(", "))
 
 h2 = t('activerecord.attributes.exercise.files')

app/views/request_for_comments/show.html.erb

@@ -29,7 +29,8 @@
         <%= t('activerecord.attributes.request_for_comments.question')%>
       </h5>
       <div class="text">
-        <%= @request_for_comment.question or t('request_for_comments.no_question')%>
+        <% question = @request_for_comment.question %>
+        <%= question.nil? or question.empty? ? t('request_for_comments.no_question') : question %>
       </div>
     </div>
 

app/views/user_exercise_feedbacks/index.html.slim

@@ -0,0 +1,27 @@
+h1 = UserExerciseFeedback.model_name.human(count: 2)
+
+= render(layout: 'shared/form_filters') do |f|
+  .form-group
+    = f.label(:execution_environment_id_eq, t('activerecord.attributes.exercise.execution_environment'), class: 'sr-only')
+    = f.collection_select(:execution_environment_id_eq, ExecutionEnvironment.with_exercises, :id, :name, class: 'form-control', prompt: t('activerecord.attributes.exercise.execution_environment'))
+  .form-group
+    = f.label(:exercise_title_cont, t('activerecord.attributes.request_for_comments.exercise'), class: 'sr-only')
+    = f.search_field(:exercise_title_cont, class: 'form-control', placeholder: t('activerecord.attributes.request_for_comments.exercise'))
+
+.table-responsive
+  table.table
+    thead
+      tr
+        th colspan=2 = t('activerecord.attributes.user_exercise_feedback.user')
+        th = t('activerecord.attributes.user_exercise_feedback.exercise')
+        th colspan=2 = t('shared.actions')
+    tbody
+      - @uefs.each do |uef|
+        tr
+          td = uef.user.id
+          td = uef.user.name
+          td = link_to(uef.exercise.title, uef.exercise)
+          td = link_to(t('shared.show'), uef)
+          td = link_to(t('shared.destroy'), uef, data: {confirm: t('shared.confirm_destroy')}, method: :delete)
+
+= render('shared/pagination', collection: @uefs)

app/views/user_exercise_feedbacks/show.html.slim

@@ -0,0 +1,7 @@
+h2 = @uef
+
+= row(label: 'activerecord.attributes.user_exercise_feedback.exercise', value: link_to(@uef.exercise.title, @uef.exercise))
+= row(label: 'user_exercise_feedback.user', value: @uef.user)
+= row(label: 'activerecord.attributes.user_exercise_feedback.feedback_text', value: @uef.feedback_text)
+= row(label: 'user_exercise_feedback.difficulty', value: @uef.difficulty)
+= row(label: 'user_exercise_feedback.working_time', value: @uef.user_estimated_worktime)

config/environments/test.rb

@@ -39,4 +39,8 @@ Rails.application.configure do
 
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
+
+  #config.logger = Logger.new(STDOUT)
+  # Set log level
+  #config.log_level = :DEBUG
 end

config/locales/de.yml

@@ -41,7 +41,6 @@ de:
         allow_auto_completion: "Autovervollständigung aktivieren"
         allow_file_creation: "Dateierstellung erlauben"
         difficulty: Schwierigkeitsgrad
-        worktime: "vermutete Arbeitszeit in Minuten"
         token: "Aufgaben-Token"
       proxy_exercise:
         title: Title
@@ -111,6 +110,25 @@ de:
         name: "Name"
         file_type: "Dateityp"
         content: "Code"
+      error_template:
+        name: Name
+        signature: Regulärer Ausdruck
+        description: Beschreibung
+        hint: Hinweis
+      error_template_attribute:
+        important: "Wichtig"
+        key: "Name"
+        description: "Beschreibung"
+        regex: "Regulärer Ausdruck"
+      exercise_collections:
+        id: "ID"
+        name: "Name"
+        updated_at: "Letzte Änderung"
+        exercises: "Aufgaben"
+      user_exercise_feedback:
+        user: "Autor"
+        exercise: "Aufgabe"
+        feedback_text: "Feedback Text"
     models:
       code_harbor_link:
         one: CodeHarbor-Link
@@ -121,12 +139,21 @@ de:
       error:
         one: Fehler
         other: Fehler
+      error_template:
+        one: Fehlertemplate
+        other: Fehlertemplates
+      error_template_attribute:
+        one: Fehlertemplatettribut
+        other: Fehlertemplatettribute
       execution_environment:
         one: Ausführungsumgebung
         other: Ausführungsumgebungen
       exercise:
         one: Aufgabe
         other: Aufgaben
+      exercise_collection:
+        one: Aufgabesammlung
+        other: Aufgabensammlungen
       proxy_exercise:
         one: Proxy Aufgabe
         other: Proxy Aufgaben
@@ -304,6 +331,7 @@ de:
       clone: Duplizieren
       implement: Implementieren
       test_files: Test-Dateien
+      feedback: Feedback
     statistics:
       average_score: Durchschnittliche Punktzahl
       final_submissions: Finale Abgaben
@@ -634,10 +662,14 @@ de:
     estimated_time_20_to_30: "zwischen 20 und 30 Minuten"
     estimated_time_more_30: "mehr als 30 Minuten"
     working_time: "Geschätze Bearbeitungszeit für diese Aufgabe:"
+  error_templates:
+    hints:
+      signature: "Ein regulärer Ausdruck in Ruby-Syntax und ohne führende und schließende \"/\""
+    attributes: "Attribute"
+    add_attribute: "Attribut hinzufügen"
   comments:
     deleted: "Gelöscht"
     save_update: "Speichern"
   subscriptions:
     successfully_unsubscribed: "Ihr Abonnement für weitere Kommentare auf dieser Kommentaranfrage wurde erfolgreich beendet."
     subscription_not_existent: "Das Abonnement, von dem Sie sich abmelden wollen, existiert nicht."
-

config/locales/en.yml

@@ -41,7 +41,6 @@ en:
         allow_auto_completion: "Allow auto completion"
         allow_file_creation: "Allow file creation"
         difficulty: Difficulty
-        worktime: "Expected worktime in minutes"
         token: "Exercise Token"
       proxy_exercise:
         title: Title
@@ -111,6 +110,25 @@ en:
         name: "Name"
         file_type: "File Type"
         content: "Content"
+      error_template:
+        name: Name
+        signature: Signature Regular Expression
+        description: Description
+        hint: Hint
+      error_template_attribute:
+        important: "Important"
+        key: "Identifier"
+        description: "Description"
+        regex: "Regular Expression"
+      exercise_collections:
+        id: "ID"
+        name: "Name"
+        updated_at: "Last Update"
+        exercises: "Exercises"
+      user_exercise_feedback:
+        user: "Author"
+        exercise: "Exercise"
+        feedback_text: "Feedback Text"
     models:
       code_harbor_link:
         one: CodeHarbor Link
@@ -121,12 +139,21 @@ en:
       error:
         one: Error
         other: Errors
+      error_template:
+        one: Error Template
+        other: Error Templates
+      error_template_attribute:
+        one: Error Template Attribute
+        other: Error Template Attributes
       execution_environment:
         one: Execution Environment
         other: Execution Environments
       exercise:
         one: Exercise
         other: Exercises
+      exercise_collection:
+        one: Exercise Collection
+        other: Exercise Collections
       proxy_exercise:
         one: Proxy Exercise
         other: Proxy Exercises
@@ -304,6 +331,7 @@ en:
       clone: Duplicate
       implement: Implement
       test_files: Test Files
+      feedback: Feedback
     statistics:
       average_score: Average Score
       final_submissions: Final Submissions
@@ -634,6 +662,11 @@ en:
     estimated_time_20_to_30: "between 20 and 30 minutes"
     estimated_time_more_30: "more than 30 minutes"
     working_time: "Estimated time working on this exercise:"
+  error_templates:
+    hints:
+      signature: "A regular expression in Ruby syntax without leading and trailing \"/\""
+    attributes: "Attributes"
+    add_attribute: "Add attribute"
   comments:
     deleted: "Deleted"
     save_update: "Save"

config/routes.rb

@@ -1,6 +1,13 @@
 FILENAME_REGEXP = /[\w\.]+/ unless Kernel.const_defined?(:FILENAME_REGEXP)
 
 Rails.application.routes.draw do
+  resources :error_template_attributes
+  resources :error_templates do
+    member do
+      put 'attribute', to: 'error_templates#add_attribute'
+      delete 'attribute', to: 'error_templates#remove_attribute'
+    end
+  end
   resources :file_templates do
     collection do
       get 'by_file_type/:file_type_id', as: :by_file_type, action: :by_file_type
@@ -69,11 +76,14 @@ Rails.application.routes.draw do
       post :intervention
       post :search
       get :statistics
+      get :feedback
       get :reload
       post :submit
     end
   end
 
+  resources :exercise_collections
+
   resources :proxy_exercises do
     member do
       post :clone

db/migrate/20160204111716_add_user_to_code_harbor_link.rb

@@ -1,5 +1,5 @@
 class AddUserToCodeHarborLink < ActiveRecord::Migration
   def change
-    add_reference :code_harbor_links, :user, index: true, foreign_key: true
+    add_reference :code_harbor_links, :user,  polymorphic: true, index: true
   end
 end

db/migrate/20170703075832_create_error_templates.rb

@@ -0,0 +1,11 @@
+class CreateErrorTemplates < ActiveRecord::Migration
+  def change
+    create_table :error_templates do |t|
+      t.belongs_to :execution_environment
+      t.string :name
+      t.string :signature
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20170703075959_create_error_template_attributes.rb

@@ -0,0 +1,11 @@
+class CreateErrorTemplateAttributes < ActiveRecord::Migration
+  def change
+    create_table :error_template_attributes do |t|
+      t.belongs_to :error_template
+      t.string :key
+      t.string :regex
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20170703080205_create_structured_errors.rb

@@ -0,0 +1,10 @@
+class CreateStructuredErrors < ActiveRecord::Migration
+  def change
+    create_table :structured_errors do |t|
+      t.references :error_template
+      t.belongs_to :file
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20170703080355_create_structured_error_attributes.rb

@@ -0,0 +1,11 @@
+class CreateStructuredErrorAttributes < ActiveRecord::Migration
+  def change
+    create_table :structured_error_attributes do |t|
+      t.belongs_to :structured_error
+      t.references :error_template_attribute
+      t.string :value
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20170711170456_add_description_and_hint_to_error_template.rb

@@ -0,0 +1,9 @@
+class AddDescriptionAndHintToErrorTemplate < ActiveRecord::Migration
+  def change
+    add_column :error_templates, :description, :text
+    add_column :error_templates, :hint, :text
+
+    add_column :error_template_attributes, :description, :text
+    add_column :error_template_attributes, :important, :boolean
+  end
+end

db/migrate/20170711170928_change_error_template_attribute_relationship_to_n_to_m.rb

@@ -0,0 +1,6 @@
+class ChangeErrorTemplateAttributeRelationshipToNToM < ActiveRecord::Migration
+  def change
+    remove_belongs_to :error_template_attributes, :error_template
+    create_join_table :error_templates, :error_template_attributes
+  end
+end

db/migrate/20170719133351_add_match_to_structured_error_attribute.rb

@@ -0,0 +1,5 @@
+class AddMatchToStructuredErrorAttribute < ActiveRecord::Migration
+  def change
+    add_column :structured_error_attributes, :match, :boolean
+  end
+end

db/migrate/20171002131135_remove_expected_working_time.rb

@@ -0,0 +1,5 @@
+class RemoveExpectedWorkingTime < ActiveRecord::Migration
+  def change
+    remove_column :exercises, :expected_worktime_seconds
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170920145852) do
+ActiveRecord::Schema.define(version: 20171002131135) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -47,6 +47,30 @@ ActiveRecord::Schema.define(version: 20170920145852) do
     t.string   "oauth_secret", limit: 255
   end
 
+  create_table "error_template_attributes", force: :cascade do |t|
+    t.string   "key"
+    t.string   "regex"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+    t.text     "description"
+    t.boolean  "important"
+  end
+
+  create_table "error_template_attributes_templates", id: false, force: :cascade do |t|
+    t.integer "error_template_id",           null: false
+    t.integer "error_template_attribute_id", null: false
+  end
+
+  create_table "error_templates", force: :cascade do |t|
+    t.integer  "execution_environment_id"
+    t.string   "name"
+    t.string   "signature"
+    t.datetime "created_at",               null: false
+    t.datetime "updated_at",               null: false
+    t.text     "description"
+    t.text     "hint"
+  end
+
   create_table "errors", force: :cascade do |t|
     t.integer  "execution_environment_id"
     t.text     "message"
@@ -109,9 +133,8 @@ ActiveRecord::Schema.define(version: 20170920145852) do
     t.string   "token",                     limit: 255
     t.boolean  "hide_file_tree"
     t.boolean  "allow_file_creation"
-    t.boolean  "allow_auto_completion",                 default: false
-    t.integer  "expected_worktime_seconds",             default: 60
-    t.integer  "expected_difficulty",                   default: 1
+    t.boolean  "allow_auto_completion",                default: false
+    t.integer  "expected_difficulty",                  default: 1
   end
 
   create_table "exercises_proxy_exercises", id: false, force: :cascade do |t|
@@ -268,6 +291,22 @@ ActiveRecord::Schema.define(version: 20170920145852) do
     t.datetime "updated_at"
   end
 
+  create_table "structured_error_attributes", force: :cascade do |t|
+    t.integer  "structured_error_id"
+    t.integer  "error_template_attribute_id"
+    t.string   "value"
+    t.datetime "created_at",                  null: false
+    t.datetime "updated_at",                  null: false
+    t.boolean  "match"
+  end
+
+  create_table "structured_errors", force: :cascade do |t|
+    t.integer  "error_template_id"
+    t.integer  "file_id"
+    t.datetime "created_at",        null: false
+    t.datetime "updated_at",        null: false
+  end
+
   create_table "submissions", force: :cascade do |t|
     t.integer  "exercise_id"
     t.float    "score"

db/seeds.rb

@@ -1,5 +1,5 @@
 def find_factories_by_class(klass)
-  FactoryGirl.factories.select do |factory|
+  FactoryBot.factories.select do |factory|
     factory.instance_variable_get(:@class_name) == klass || factory.instance_variable_get(:@name) == klass.model_name.singular.to_sym
   end
 end
@@ -9,7 +9,7 @@ module ActiveRecord
     [:build, :create].each do |strategy|
       define_singleton_method("#{strategy}_factories") do |attributes = {}|
         find_factories_by_class(self).map(&:name).map do |factory_name|
-          FactoryGirl.send(strategy, factory_name, attributes)
+          FactoryBot.send(strategy, factory_name, attributes)
         end
       end
     end

db/seeds/development.rb

@@ -1,9 +1,9 @@
 # consumers
-FactoryGirl.create(:consumer)
-FactoryGirl.create(:consumer, name: 'openSAP')
+FactoryBot.create(:consumer)
+FactoryBot.create(:consumer, name: 'openSAP')
 
 # users
-[:admin, :external_user, :teacher].each { |factory_name| FactoryGirl.create(factory_name) }
+[:admin, :external_user, :teacher].each { |factory_name| FactoryBot.create(factory_name) }
 
 # execution environments
 ExecutionEnvironment.create_factories
@@ -12,7 +12,7 @@ ExecutionEnvironment.create_factories
 Error.create_factories
 
 # exercises
-@exercises = find_factories_by_class(Exercise).map(&:name).map { |factory_name| [factory_name, FactoryGirl.create(factory_name)] }.to_h
+@exercises = find_factories_by_class(Exercise).map(&:name).map { |factory_name| [factory_name, FactoryBot.create(factory_name)] }.to_h
 
 # file types
 FileType.create_factories
@@ -21,4 +21,4 @@ FileType.create_factories
 Hint.create_factories
 
 # submissions
-FactoryGirl.create(:submission, exercise: @exercises[:fibonacci])
+FactoryBot.create(:submission, exercise: @exercises[:fibonacci])

db/seeds/production.rb

@@ -1,7 +1,7 @@
 require 'highline/import'
 
 # consumers
-FactoryGirl.create(:consumer)
+FactoryBot.create(:consumer)
 
 # users
 email = ask('Enter admin email: ')
@@ -11,7 +11,7 @@ passwords = ['password', 'password confirmation'].map do |attribute|
 end
 
 if passwords.uniq.length == 1
-  FactoryGirl.create(:admin, email: email, name: 'Administrator', password: passwords.first)
+  FactoryBot.create(:admin, email: email, name: 'Administrator', password: passwords.first)
 else
   abort('Passwords do not match!')
 end

spec/concerns/lti_spec.rb

@@ -11,7 +11,7 @@ describe Lti do
   describe '#build_tool_provider' do
     it 'instantiates a tool provider' do
       expect(IMS::LTI::ToolProvider).to receive(:new)
-      controller.send(:build_tool_provider, consumer: FactoryGirl.build(:consumer), parameters: {})
+      controller.send(:build_tool_provider, consumer: FactoryBot.build(:consumer), parameters: {})
     end
   end
 
@@ -25,31 +25,23 @@ describe Lti do
 
   describe '#external_user_name' do
     let(:first_name) { 'Jane' }
-    let(:full_name) { 'John Doe' }
     let(:last_name) { 'Doe' }
+    let(:full_name) { 'John Doe' }
     let(:provider) { double }
+    let(:provider_full) { double(:lis_person_name_full => full_name) }
 
     context 'when a full name is provided' do
       it 'returns the full name' do
-        expect(provider).to receive(:lis_person_name_full).twice.and_return(full_name)
-        expect(controller.send(:external_user_name, provider)).to eq(full_name)
-      end
-    end
-
-    context 'when first and last name are provided' do
-      it 'returns the concatenated names' do
-        expect(provider).to receive(:lis_person_name_full)
-        expect(provider).to receive(:lis_person_name_given).twice.and_return(first_name)
-        expect(provider).to receive(:lis_person_name_family).twice.and_return(last_name)
-        expect(controller.send(:external_user_name, provider)).to eq("#{first_name} #{last_name}")
+        expect(provider_full).to receive(:lis_person_name_full).twice.and_return(full_name)
+        expect(controller.send(:external_user_name, provider_full)).to eq(full_name)
       end
     end
 
     context 'when only partial information is provided' do
       it 'returns the first available name' do
         expect(provider).to receive(:lis_person_name_full)
-        expect(provider).to receive(:lis_person_name_given).twice.and_return(first_name)
-        expect(provider).to receive(:lis_person_name_family)
+        expect(provider).to receive(:lis_person_name_given).and_return(first_name)
+        expect(provider).not_to receive(:lis_person_name_family)
         expect(controller.send(:external_user_name, provider)).to eq(first_name)
       end
     end
@@ -103,10 +95,10 @@ describe Lti do
   end
 
   describe '#send_score' do
-    let(:consumer) { FactoryGirl.create(:consumer) }
+    let(:consumer) { FactoryBot.create(:consumer) }
     let(:score) { 0.5 }
-    let(:submission) { FactoryGirl.create(:submission) }
-    let!(:lti_parameter) { FactoryGirl.create(:lti_parameter)}
+    let(:submission) { FactoryBot.create(:submission) }
+    let!(:lti_parameter) { FactoryBot.create(:lti_parameter)}
 
     context 'with an invalid score' do
       it 'raises an exception' do
@@ -122,6 +114,7 @@ describe Lti do
 
         context 'when grading is not supported' do
           it 'returns a corresponding status' do
+            skip('ralf: this does not work, since send_score pulls data from the database, which then returns an empty array. On this is called .first, which returns nil and lets the test fail. Before Toms changes, this was taken from the session, which could be mocked')
             expect_any_instance_of(IMS::LTI::ToolProvider).to receive(:outcome_service?).and_return(false)
             expect(controller.send(:send_score, submission.exercise_id, score, submission.user_id)[:status]).to eq('unsupported')
           end
@@ -140,10 +133,12 @@ describe Lti do
           end
 
           it 'sends the score' do
+            skip('ralf: this does not work, since send_score pulls data from the database, which then returns an empty array. On this is called .first, which returns nil and lets the test fail. Before Toms changes, this was taken from the session, which could be mocked')
             controller.send(:send_score, submission.exercise_id, score, submission.user_id)
           end
 
           it 'returns code, message, and status' do
+            skip('ralf: this does not work, since send_score pulls data from the database, which then returns an empty array. On this is called .first, which returns nil and lets the test fail. Before Toms changes, this was taken from the session, which could be mocked')
             result = controller.send(:send_score, submission.exercise_id, score, submission.user_id)
             expect(result[:code]).to eq(response.response_code)
             expect(result[:message]).to eq(response.body)
@@ -164,18 +159,18 @@ describe Lti do
     let(:parameters) { {} }
 
     it 'stores data in the session' do
-      controller.instance_variable_set(:@current_user, FactoryGirl.create(:external_user))
-      controller.instance_variable_set(:@exercise, FactoryGirl.create(:fibonacci))
+      controller.instance_variable_set(:@current_user, FactoryBot.create(:external_user))
+      controller.instance_variable_set(:@exercise, FactoryBot.create(:fibonacci))
       expect(controller.session).to receive(:[]=).with(:consumer_id, anything)
       expect(controller.session).to receive(:[]=).with(:external_user_id, anything)
-      controller.send(:store_lti_session_data, consumer: FactoryGirl.build(:consumer), parameters: parameters)
+      controller.send(:store_lti_session_data, consumer: FactoryBot.build(:consumer), parameters: parameters)
     end
 
     it 'it creates an LtiParameter Object' do
       before_count = LtiParameter.count
-      controller.instance_variable_set(:@current_user, FactoryGirl.create(:external_user))
-      controller.instance_variable_set(:@exercise, FactoryGirl.create(:fibonacci))
-      controller.send(:store_lti_session_data, consumer: FactoryGirl.build(:consumer), parameters: parameters)
+      controller.instance_variable_set(:@current_user, FactoryBot.create(:external_user))
+      controller.instance_variable_set(:@exercise, FactoryBot.create(:fibonacci))
+      controller.send(:store_lti_session_data, consumer: FactoryBot.build(:consumer), parameters: parameters)
       expect(LtiParameter.count).to eq(before_count + 1)
     end
   end

spec/concerns/submission_scoring_spec.rb

@@ -6,8 +6,8 @@ end
 
 describe SubmissionScoring do
   let(:controller) { Controller.new }
-  before(:all) { @submission = FactoryGirl.create(:submission, cause: 'submit') }
-  before(:each) { controller.instance_variable_set(:@current_user, FactoryGirl.create(:external_user)) }
+  before(:all) { @submission = FactoryBot.create(:submission, cause: 'submit') }
+  before(:each) { controller.instance_variable_set(:@current_user, FactoryBot.create(:external_user)) }
 
   describe '#collect_test_results' do
     after(:each) { controller.send(:collect_test_results, @submission) }

spec/controllers/admin/dashboard_controller_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 describe Admin::DashboardController do
-  before(:each) { allow(controller).to receive(:current_user).and_return(FactoryGirl.build(:admin)) }
+  before(:each) { allow(controller).to receive(:current_user).and_return(FactoryBot.build(:admin)) }
 
   describe 'GET #show' do
     describe 'with format HTML' do

spec/controllers/application_controller_spec.rb

@@ -3,7 +3,7 @@ require 'rails_helper'
 describe ApplicationController do
   describe '#current_user' do
     context 'with an external user' do
-      let(:external_user) { FactoryGirl.create(:external_user) }
+      let(:external_user) { FactoryBot.create(:external_user) }
       before(:each) { session[:external_user_id] = external_user.id }
 
       it 'returns the external user' do
@@ -12,7 +12,7 @@ describe ApplicationController do
     end
 
     context 'without an external user' do
-      let(:internal_user) { FactoryGirl.create(:teacher) }
+      let(:internal_user) { FactoryBot.create(:teacher) }
       before(:each) { login_user(internal_user) }
 
       it 'returns the internal user' do

spec/controllers/code_ocean/files_controller_spec.rb

@@ -1,14 +1,14 @@
 require 'rails_helper'
 
 describe CodeOcean::FilesController do
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
-    let(:submission) { FactoryGirl.create(:submission, user: user) }
+    let(:submission) { FactoryBot.create(:submission, user: user) }
 
     context 'with a valid file' do
-      let(:request) { proc { post :create, code_ocean_file: FactoryGirl.build(:file, context: submission).attributes, format: :json } }
+      let(:request) { proc { post :create, code_ocean_file: FactoryBot.build(:file, context: submission).attributes, format: :json } }
       before(:each) { request.call }
 
       expect_assigns(file: CodeOcean::File)
@@ -31,14 +31,14 @@ describe CodeOcean::FilesController do
   end
 
   describe 'DELETE #destroy' do
-    let(:exercise) { FactoryGirl.create(:fibonacci) }
+    let(:exercise) { FactoryBot.create(:fibonacci) }
     let(:request) { proc { delete :destroy, id: exercise.files.first.id } }
     before(:each) { request.call }
 
     expect_assigns(file: CodeOcean::File)
 
     it 'destroys the file' do
-      FactoryGirl.create(:fibonacci)
+      FactoryBot.create(:fibonacci)
       expect { request.call }.to change(CodeOcean::File, :count).by(-1)
     end
 

spec/controllers/consumers_controller_spec.rb

@@ -1,13 +1,13 @@
 require 'rails_helper'
 
 describe ConsumersController do
-  let(:consumer) { FactoryGirl.create(:consumer) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:consumer) { FactoryBot.create(:consumer) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
     context 'with a valid consumer' do
-      let(:request) { proc { post :create, consumer: FactoryGirl.attributes_for(:consumer) } }
+      let(:request) { proc { post :create, consumer: FactoryBot.attributes_for(:consumer) } }
       before(:each) { request.call }
 
       expect_assigns(consumer: Consumer)
@@ -34,7 +34,7 @@ describe ConsumersController do
     expect_assigns(consumer: Consumer)
 
     it 'destroys the consumer' do
-      consumer = FactoryGirl.create(:consumer)
+      consumer = FactoryBot.create(:consumer)
       expect { delete :destroy, id: consumer.id }.to change(Consumer, :count).by(-1)
     end
 
@@ -50,7 +50,7 @@ describe ConsumersController do
   end
 
   describe 'GET #index' do
-    let!(:consumers) { FactoryGirl.create_pair(:consumer) }
+    let!(:consumers) { FactoryBot.create_pair(:consumer) }
     before(:each) { get :index }
 
     expect_assigns(consumers: Consumer.all)
@@ -76,7 +76,7 @@ describe ConsumersController do
 
   describe 'PUT #update' do
     context 'with a valid consumer' do
-      before(:each) { put :update, consumer: FactoryGirl.attributes_for(:consumer), id: consumer.id }
+      before(:each) { put :update, consumer: FactoryBot.attributes_for(:consumer), id: consumer.id }
 
       expect_assigns(consumer: Consumer)
       expect_redirect(:consumer)

spec/controllers/execution_environments_controller_spec.rb

@@ -1,15 +1,15 @@
 require 'rails_helper'
 
 describe ExecutionEnvironmentsController do
-  let(:execution_environment) { FactoryGirl.create(:ruby) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:execution_environment) { FactoryBot.create(:ruby) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
     before(:each) { expect(DockerClient).to receive(:image_tags).at_least(:once).and_return([]) }
 
     context 'with a valid execution environment' do
-      let(:request) { proc { post :create, execution_environment: FactoryGirl.attributes_for(:ruby) } }
+      let(:request) { proc { post :create, execution_environment: FactoryBot.attributes_for(:ruby) } }
       before(:each) { request.call }
 
       expect_assigns(docker_images: Array)
@@ -37,7 +37,7 @@ describe ExecutionEnvironmentsController do
     expect_assigns(execution_environment: :execution_environment)
 
     it 'destroys the execution environment' do
-      execution_environment = FactoryGirl.create(:ruby)
+      execution_environment = FactoryBot.create(:ruby)
       expect { delete :destroy, id: execution_environment.id }.to change(ExecutionEnvironment, :count).by(-1)
     end
 
@@ -72,7 +72,7 @@ describe ExecutionEnvironmentsController do
   end
 
   describe 'GET #index' do
-    before(:all) { FactoryGirl.create_pair(:ruby) }
+    before(:all) { FactoryBot.create_pair(:ruby) }
     before(:each) { get :index }
 
     expect_assigns(execution_environments: ExecutionEnvironment.all)
@@ -150,7 +150,7 @@ describe ExecutionEnvironmentsController do
     context 'with a valid execution environment' do
       before(:each) do
         expect(DockerClient).to receive(:image_tags).at_least(:once).and_return([])
-        put :update, execution_environment: FactoryGirl.attributes_for(:ruby), id: execution_environment.id
+        put :update, execution_environment: FactoryBot.attributes_for(:ruby), id: execution_environment.id
       end
 
       expect_assigns(docker_images: Array)

spec/controllers/exercises_controller_spec.rb

@@ -1,8 +1,8 @@
 require 'rails_helper'
 
 describe ExercisesController do
-  let(:exercise) { FactoryGirl.create(:dummy) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:exercise) { FactoryBot.create(:dummy) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'PUT #batch_update' do
@@ -52,7 +52,7 @@ describe ExercisesController do
   end
 
   describe 'POST #create' do
-    let(:exercise_attributes) { FactoryGirl.build(:dummy).attributes }
+    let(:exercise_attributes) { FactoryBot.build(:dummy).attributes }
 
     context 'with a valid exercise' do
       let(:request) { proc { post :create, exercise: exercise_attributes } }
@@ -71,7 +71,7 @@ describe ExercisesController do
       let(:request) { proc { post :create, exercise: exercise_attributes.merge(files_attributes: files_attributes) } }
 
       context 'when specifying the file content within the form' do
-        let(:files_attributes) { {'0' => FactoryGirl.build(:file).attributes} }
+        let(:files_attributes) { {'0' => FactoryBot.build(:file).attributes} }
 
         it 'creates the file' do
           expect { request.call }.to change(CodeOcean::File, :count)
@@ -79,11 +79,11 @@ describe ExercisesController do
       end
 
       context 'when uploading a file' do
-        let(:files_attributes) { {'0' => FactoryGirl.build(:file, file_type: file_type).attributes.merge(content: uploaded_file)} }
+        let(:files_attributes) { {'0' => FactoryBot.build(:file, file_type: file_type).attributes.merge(content: uploaded_file)} }
 
         context 'when uploading a binary file' do
           let(:file_path) { Rails.root.join('db', 'seeds', 'audio_video', 'devstories.mp4') }
-          let(:file_type) { FactoryGirl.create(:dot_mp4) }
+          let(:file_type) { FactoryBot.create(:dot_mp4) }
           let(:uploaded_file) { Rack::Test::UploadedFile.new(file_path, 'video/mp4', true) }
 
           it 'creates the file' do
@@ -98,7 +98,7 @@ describe ExercisesController do
 
         context 'when uploading a non-binary file' do
           let(:file_path) { Rails.root.join('db', 'seeds', 'fibonacci', 'exercise.rb') }
-          let(:file_type) { FactoryGirl.create(:dot_rb) }
+          let(:file_type) { FactoryBot.create(:dot_rb) }
           let(:uploaded_file) { Rack::Test::UploadedFile.new(file_path, 'text/x-ruby', false) }
 
           it 'creates the file' do
@@ -128,7 +128,7 @@ describe ExercisesController do
     expect_assigns(exercise: :exercise)
 
     it 'destroys the exercise' do
-      exercise = FactoryGirl.create(:dummy)
+      exercise = FactoryBot.create(:dummy)
       expect { delete :destroy, id: exercise.id }.to change(Exercise, :count).by(-1)
     end
 
@@ -147,13 +147,13 @@ describe ExercisesController do
     let(:request) { proc { get :implement, id: exercise.id } }
 
     context 'with an exercise with visible files' do
-      let(:exercise) { FactoryGirl.create(:fibonacci) }
+      let(:exercise) { FactoryBot.create(:fibonacci) }
       before(:each) { request.call }
 
       expect_assigns(exercise: :exercise)
 
       context 'with an existing submission' do
-        let!(:submission) { FactoryGirl.create(:submission, exercise_id: exercise.id, user_id: user.id, user_type: user.class.name) }
+        let!(:submission) { FactoryBot.create(:submission, exercise_id: exercise.id, user_id: user.id, user_type: user.class.name) }
 
         it "populates the editors with the submission's files' content" do
           request.call
@@ -182,7 +182,7 @@ describe ExercisesController do
 
   describe 'GET #index' do
     let(:scope) { Pundit.policy_scope!(user, Exercise) }
-    before(:all) { FactoryGirl.create_pair(:dummy) }
+    before(:all) { FactoryBot.create_pair(:dummy) }
     before(:each) { get :index }
 
     expect_assigns(exercises: :scope)
@@ -230,8 +230,8 @@ describe ExercisesController do
   describe 'POST #submit' do
     let(:output) { {} }
     let(:request) { post :submit, format: :json, id: exercise.id, submission: {cause: 'submit', exercise_id: exercise.id} }
-    let!(:external_user) { FactoryGirl.create(:external_user) }
-    let!(:lti_parameter) { FactoryGirl.create(:lti_parameter, external_user: external_user, exercise: exercise) }
+    let!(:external_user) { FactoryBot.create(:external_user) }
+    let!(:lti_parameter) { FactoryBot.create(:lti_parameter, external_user: external_user, exercise: exercise) }
 
     before(:each) do
       allow_any_instance_of(Submission).to receive(:normalized_score).and_return(1)
@@ -298,7 +298,7 @@ describe ExercisesController do
 
   describe 'PUT #update' do
     context 'with a valid exercise' do
-      let(:exercise_attributes) { FactoryGirl.build(:dummy).attributes }
+      let(:exercise_attributes) { FactoryBot.build(:dummy).attributes }
       before(:each) { put :update, exercise: exercise_attributes, id: exercise.id }
 
       expect_assigns(exercise: Exercise)

spec/controllers/external_users_controller_spec.rb

@@ -1,8 +1,8 @@
 require 'rails_helper'
 
 describe ExternalUsersController do
-  let(:user) { FactoryGirl.build(:admin) }
-  let!(:users) { FactoryGirl.create_pair(:external_user) }
+  let(:user) { FactoryBot.build(:admin) }
+  let!(:users) { FactoryBot.create_pair(:external_user) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'GET #index' do

spec/controllers/file_types_controller_spec.rb

@@ -1,13 +1,13 @@
 require 'rails_helper'
 
 describe FileTypesController do
-  let(:file_type) { FactoryGirl.create(:dot_rb) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:file_type) { FactoryBot.create(:dot_rb) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
     context 'with a valid file type' do
-      let(:request) { proc { post :create, file_type: FactoryGirl.attributes_for(:dot_rb) } }
+      let(:request) { proc { post :create, file_type: FactoryBot.attributes_for(:dot_rb) } }
       before(:each) { request.call }
 
       expect_assigns(editor_modes: Array)
@@ -36,7 +36,7 @@ describe FileTypesController do
     expect_assigns(file_type: FileType)
 
     it 'destroys the file type' do
-      file_type = FactoryGirl.create(:dot_rb)
+      file_type = FactoryBot.create(:dot_rb)
       expect { delete :destroy, id: file_type.id }.to change(FileType, :count).by(-1)
     end
 
@@ -53,7 +53,7 @@ describe FileTypesController do
   end
 
   describe 'GET #index' do
-    before(:all) { FactoryGirl.create_pair(:dot_rb) }
+    before(:all) { FactoryBot.create_pair(:dot_rb) }
     before(:each) { get :index }
 
     expect_assigns(file_types: FileType.all)
@@ -80,7 +80,7 @@ describe FileTypesController do
 
   describe 'PUT #update' do
     context 'with a valid file type' do
-      before(:each) { put :update, file_type: FactoryGirl.attributes_for(:dot_rb), id: file_type.id }
+      before(:each) { put :update, file_type: FactoryBot.attributes_for(:dot_rb), id: file_type.id }
 
       expect_assigns(editor_modes: Array)
       expect_assigns(file_type: FileType)

spec/controllers/hints_controller_spec.rb

@@ -1,14 +1,14 @@
 require 'rails_helper'
 
 describe HintsController do
-  let(:execution_environment) { FactoryGirl.create(:ruby) }
-  let(:hint) { FactoryGirl.create(:ruby_syntax_error) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:execution_environment) { FactoryBot.create(:ruby) }
+  let(:hint) { FactoryBot.create(:ruby_syntax_error) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
     context 'with a valid hint' do
-      let(:request) { proc { post :create, execution_environment_id: execution_environment.id, hint: FactoryGirl.attributes_for(:ruby_syntax_error) } }
+      let(:request) { proc { post :create, execution_environment_id: execution_environment.id, hint: FactoryBot.attributes_for(:ruby_syntax_error) } }
       before(:each) { request.call }
 
       expect_assigns(execution_environment: :execution_environment)
@@ -38,7 +38,7 @@ describe HintsController do
     expect_assigns(hint: Hint)
 
     it 'destroys the hint' do
-      hint = FactoryGirl.create(:ruby_syntax_error)
+      hint = FactoryBot.create(:ruby_syntax_error)
       expect { delete :destroy, execution_environment_id: execution_environment.id, id: hint.id }.to change(Hint, :count).by(-1)
     end
 
@@ -55,7 +55,7 @@ describe HintsController do
   end
 
   describe 'GET #index' do
-    before(:all) { FactoryGirl.create_pair(:ruby_syntax_error) }
+    before(:all) { FactoryBot.create_pair(:ruby_syntax_error) }
     before(:each) { get :index, execution_environment_id: execution_environment.id }
 
     expect_assigns(execution_environment: :execution_environment)
@@ -84,7 +84,7 @@ describe HintsController do
 
   describe 'PUT #update' do
     context 'with a valid hint' do
-      before(:each) { put :update, execution_environment_id: execution_environment.id, hint: FactoryGirl.attributes_for(:ruby_syntax_error), id: hint.id }
+      before(:each) { put :update, execution_environment_id: execution_environment.id, hint: FactoryBot.attributes_for(:ruby_syntax_error), id: hint.id }
 
       expect_assigns(execution_environment: :execution_environment)
       expect_assigns(hint: Hint)

spec/controllers/internal_users_controller_spec.rb

@@ -1,11 +1,11 @@
 require 'rails_helper'
 
 describe InternalUsersController do
-  let(:user) { FactoryGirl.build(:admin) }
-  let!(:users) { FactoryGirl.create_pair(:teacher) }
+  let(:user) { FactoryBot.build(:admin) }
+  let!(:users) { FactoryBot.create_pair(:teacher) }
 
   describe 'GET #activate' do
-    let(:user) { InternalUser.create(FactoryGirl.attributes_for(:teacher)) }
+    let(:user) { InternalUser.create(FactoryBot.attributes_for(:teacher)) }
 
     before(:each) do
       user.send(:setup_activation)
@@ -37,7 +37,7 @@ describe InternalUsersController do
   end
 
   describe 'PUT #activate' do
-    let(:user) { InternalUser.create(FactoryGirl.attributes_for(:teacher)) }
+    let(:user) { InternalUser.create(FactoryBot.attributes_for(:teacher)) }
     let(:password) { SecureRandom.hex }
 
     before(:each) do
@@ -103,7 +103,7 @@ describe InternalUsersController do
     before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
     context 'with a valid internal user' do
-      let(:request) { proc { post :create, internal_user: FactoryGirl.attributes_for(:teacher) } }
+      let(:request) { proc { post :create, internal_user: FactoryBot.attributes_for(:teacher) } }
       before(:each) { request.call }
 
       expect_assigns(user: InternalUser)
@@ -303,7 +303,7 @@ describe InternalUsersController do
     before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
     context 'with a valid internal user' do
-      before(:each) { put :update, internal_user: FactoryGirl.attributes_for(:teacher), id: users.first.id }
+      before(:each) { put :update, internal_user: FactoryBot.attributes_for(:teacher), id: users.first.id }
 
       expect_assigns(user: InternalUser)
       expect_redirect { user }

spec/controllers/sessions_controller_spec.rb

@@ -1,12 +1,12 @@
 require 'rails_helper'
 
 describe SessionsController do
-  let(:consumer) { FactoryGirl.create(:consumer) }
+  let(:consumer) { FactoryBot.create(:consumer) }
 
   describe 'POST #create' do
     let(:password) { user_attributes[:password] }
     let(:user) { InternalUser.create(user_attributes) }
-    let(:user_attributes) { FactoryGirl.attributes_for(:teacher) }
+    let(:user_attributes) { FactoryBot.attributes_for(:teacher) }
 
     context 'with valid credentials' do
       before(:each) do
@@ -27,8 +27,8 @@ describe SessionsController do
   end
 
   describe 'POST #create_through_lti' do
-    let(:exercise) { FactoryGirl.create(:dummy) }
-    let(:exercise2) { FactoryGirl.create(:dummy) }
+    let(:exercise) { FactoryBot.create(:dummy) }
+    let(:exercise2) { FactoryBot.create(:dummy) }
     let(:nonce) { SecureRandom.hex }
     before(:each) { I18n.locale = I18n.default_locale }
 
@@ -73,7 +73,7 @@ describe SessionsController do
     context 'with valid launch parameters' do
       let(:locale) { :de }
       let(:request) { post :create_through_lti, custom_locale: locale, custom_token: exercise.token, oauth_consumer_key: consumer.oauth_key, oauth_nonce: nonce, oauth_signature: SecureRandom.hex, user_id: user.external_id }
-      let(:user) { FactoryGirl.create(:external_user, consumer_id: consumer.id) }
+      let(:user) { FactoryBot.create(:external_user, consumer_id: consumer.id) }
       before(:each) { expect_any_instance_of(IMS::LTI::ToolProvider).to receive(:valid_request?).and_return(true) }
 
       it 'assigns the current user' do
@@ -132,14 +132,14 @@ describe SessionsController do
       end
 
       it 'redirects to recommended exercise if requested token of proxy exercise' do
-        FactoryGirl.create(:proxy_exercise, exercises: [exercise])
+        FactoryBot.create(:proxy_exercise, exercises: [exercise])
         post :create_through_lti, custom_locale: locale, custom_token: ProxyExercise.first.token, oauth_consumer_key: consumer.oauth_key, oauth_nonce: nonce, oauth_signature: SecureRandom.hex, user_id: user.external_id
         expect(controller).to redirect_to(implement_exercise_path(exercise.id))
       end
 
       it 'recommends only exercises who are 1 degree more complicated than what user has seen' do
         # dummy user has no exercises finished, therefore his highest difficulty is 0
-        FactoryGirl.create(:proxy_exercise, exercises: [exercise, exercise2])
+        FactoryBot.create(:proxy_exercise, exercises: [exercise, exercise2])
         exercise.expected_difficulty = 3
         exercise.save
         exercise2.expected_difficulty = 1
@@ -191,7 +191,7 @@ describe SessionsController do
 
   describe 'GET #destroy_through_lti' do
     let(:request) { proc { get :destroy_through_lti, consumer_id: consumer.id, submission_id: submission.id } }
-    let(:submission) { FactoryGirl.create(:submission, exercise: FactoryGirl.create(:dummy)) }
+    let(:submission) { FactoryBot.create(:submission, exercise: FactoryBot.create(:dummy)) }
 
     before(:each) do
       #Todo replace session with lti_parameter
@@ -225,7 +225,7 @@ describe SessionsController do
 
     context 'when a user is already logged in' do
       before(:each) do
-        expect(controller).to receive(:current_user).and_return(FactoryGirl.build(:teacher))
+        expect(controller).to receive(:current_user).and_return(FactoryBot.build(:teacher))
         get :new
       end
 

spec/controllers/submissions_controller_spec.rb

@@ -1,8 +1,8 @@
 require 'rails_helper'
 
 describe SubmissionsController do
-  let(:submission) { FactoryGirl.create(:submission) }
-  let(:user) { FactoryGirl.create(:admin) }
+  let(:submission) { FactoryBot.create(:submission) }
+  let(:user) { FactoryBot.create(:admin) }
   before(:each) { allow(controller).to receive(:current_user).and_return(user) }
 
   describe 'POST #create' do
@@ -11,8 +11,8 @@ describe SubmissionsController do
     end
 
     context 'with a valid submission' do
-      let(:exercise) { FactoryGirl.create(:hello_world) }
-      let(:request) { proc { post :create, format: :json, submission: FactoryGirl.attributes_for(:submission, exercise_id: exercise.id) } }
+      let(:exercise) { FactoryBot.create(:hello_world) }
+      let(:request) { proc { post :create, format: :json, submission: FactoryBot.attributes_for(:submission, exercise_id: exercise.id) } }
       before(:each) { request.call }
 
       expect_assigns(submission: Submission)
@@ -42,7 +42,7 @@ describe SubmissionsController do
     end
 
     context 'with a valid filename' do
-      let(:submission) { FactoryGirl.create(:submission, exercise: FactoryGirl.create(:audio_video)) }
+      let(:submission) { FactoryBot.create(:submission, exercise: FactoryBot.create(:audio_video)) }
       before(:each) { get :download_file, filename: file.name_with_extension, id: submission.id }
 
       context 'for a binary file' do
@@ -74,7 +74,7 @@ describe SubmissionsController do
   end
 
   describe 'GET #index' do
-    before(:all) { FactoryGirl.create_pair(:submission) }
+    before(:all) { FactoryBot.create_pair(:submission) }
     before(:each) { get :index }
 
     expect_assigns(submissions: Submission.all)
@@ -92,7 +92,7 @@ describe SubmissionsController do
     end
 
     context 'with a valid filename' do
-      let(:submission) { FactoryGirl.create(:submission, exercise: FactoryGirl.create(:audio_video)) }
+      let(:submission) { FactoryBot.create(:submission, exercise: FactoryBot.create(:audio_video)) }
       before(:each) { get :render_file, filename: file.name_with_extension, id: submission.id }
 
       context 'for a binary file' do
@@ -183,6 +183,41 @@ describe SubmissionsController do
     expect_template(:show)
   end
 
+  describe 'GET #show.json' do
+    # Render views requested in controller tests in order to get json responses
+    # https://github.com/rails/jbuilder/issues/32
+    render_views
+
+    before(:each) { get :show, id: submission.id, format: :json }
+    expect_assigns(submission: :submission)
+    expect_status(200)
+
+    [:render, :run, :test].each do |action|
+      describe "##{action}_url" do
+        let(:url) { JSON.parse(response.body).with_indifferent_access.fetch("#{action}_url") }
+
+        it "starts like the #{action} path" do
+          filename = File.basename(__FILE__)
+          expect(url).to start_with(Rails.application.routes.url_helpers.send(:"#{action}_submission_path", submission, filename).sub(filename, ''))
+        end
+
+        it 'ends with a placeholder' do
+          expect(url).to end_with(Submission::FILENAME_URL_PLACEHOLDER)
+        end
+      end
+    end
+
+    [:score, :stop].each do |action|
+      describe "##{action}_url" do
+        let(:url) { JSON.parse(response.body).with_indifferent_access.fetch("#{action}_url") }
+
+        it "corresponds to the #{action} path" do
+          expect(url).to eq(Rails.application.routes.url_helpers.send(:"#{action}_submission_path", submission))
+        end
+      end
+    end
+  end
+
   describe 'GET #score' do
     let(:request) { proc { get :score, id: submission.id } }
     before(:each) { request.call }

spec/factories/code_ocean/file.rb

@@ -1,7 +1,7 @@
 require 'seeds_helper'
 
 module CodeOcean
-  FactoryGirl.define do
+  FactoryBot.define do
     factory :file, class: CodeOcean::File do
       content ''
       association :context, factory: :submission

spec/factories/consumer.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :consumer do
     name 'openHPI'
     oauth_key { SecureRandom.hex }

spec/factories/error.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :error, class: Error do
     association :execution_environment, factory: :ruby
     message "exercise.rb:4:in `<main>': undefined local variable or method `foo' for main:Object (NameError)"

spec/factories/execution_environment.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :coffee_script, class: ExecutionEnvironment do
     created_by_teacher
     default_memory_limit

spec/factories/exercise.rb

@@ -2,7 +2,7 @@ require 'seeds_helper'
 
 def create_seed_file(exercise, path, file_attributes = {})
   file_extension = File.extname(path)
-  file_type = FactoryGirl.create(file_attributes[:file_type] || :"dot_#{file_extension.gsub('.', '')}")
+  file_type = FactoryBot.create(file_attributes[:file_type] || :"dot_#{file_extension.gsub('.', '')}")
   name = File.basename(path).gsub(file_extension, '')
   file_attributes.merge!(file_type: file_type, name: name, path: path.split('/')[1..-2].join('/'), role: file_attributes[:role] || 'regular_file')
   if file_type.binary?
@@ -13,7 +13,7 @@ def create_seed_file(exercise, path, file_attributes = {})
   exercise.add_file!(file_attributes)
 end
 
-FactoryGirl.define do
+FactoryBot.define do
   factory :audio_video, class: Exercise do
     created_by_teacher
     description "Try HTML's audio and video capabilities."
@@ -38,6 +38,24 @@ FactoryGirl.define do
     association :execution_environment, factory: :ruby
     instructions
     title 'Dummy'
+
+    factory :dummy_with_user_feedbacks do
+      # user_feedbacks_count is declared as a transient attribute and available in
+      # attributes on the factory, as well as the callback via the evaluator
+      transient do
+        user_feedbacks_count 5
+      end
+
+      # the after(:create) yields two values; the exercise instance itself and the
+      # evaluator, which stores all values from the factory, including transient
+      # attributes; `create_list`'s second argument is the number of records
+      # to create and we make sure the user_exercise_feedback is associated properly to the exercise
+      after(:create) do |exercise, evaluator|
+        create_list(:user_exercise_feedback, evaluator.user_feedbacks_count, exercise: exercise)
+      end
+
+    end
+
   end
 
   factory :even_odd, class: Exercise do

spec/factories/external_user.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :external_user do
     association :consumer
     generated_email

spec/factories/file_type.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :dot_coffee, class: FileType do
     created_by_admin
     editor_mode 'ace/mode/coffee'

spec/factories/hint.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :node_js_invalid_assignment, class: Hint do
     association :execution_environment, factory: :node_js
     english

spec/factories/internal_user.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :admin, class: InternalUser do
     activated_user
     email 'admin@example.org'

spec/factories/lti_parameter.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
 
   LTI_PARAMETERS = {
       lis_result_sourcedid: "c2db0c7c-4411-4b27-a52b-ddfc3dc32065",

spec/factories/proxy_exercise.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :proxy_exercise, class: ProxyExercise do
     token 'dummytoken'
     title 'Dummy'

spec/factories/shared_traits.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   [:admin, :external_user, :teacher].each do |factory_name|
     trait :"created_by_#{factory_name}" do
       association :user, factory: factory_name

spec/factories/submission.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   factory :submission do
     cause 'save'
     created_by_external_user

spec/factories/user_exercise_feedback.rb

@@ -0,0 +1,7 @@
+FactoryBot.define do
+  factory :user_exercise_feedback, class: UserExerciseFeedback do
+    created_by_external_user
+    feedback_text 'Most suitable exercise ever'
+  end
+
+end

spec/features/authentication_spec.rb

@@ -1,8 +1,8 @@
 require 'rails_helper'
 
 describe 'Authentication' do
-  let(:user) { FactoryGirl.create(:admin) }
-  let(:password) { FactoryGirl.attributes_for(:admin)[:password] }
+  let(:user) { FactoryBot.create(:admin) }
+  let(:password) { FactoryBot.attributes_for(:admin)[:password] }
 
   context 'when signed out' do
     before(:each) { visit(root_path) }