Implement working time graph for study group dashboard

(so far, without live update)

app/assets/javascripts/channels/la_exercises.js

@@ -18,16 +18,233 @@ $(document).on('turbolinks:load', function() {
 
             received: function (data) {
                 // Called when there's incoming data on the websocket for this channel
-                let $row = $('tr[data-id="' + data.id + '"]');
-                if ($row.length === 0) {
-                    $row = $($('#posted_rfcs')[0].insertRow(0));
+                if (data.type === 'rfc') {
+                    handleNewRfCdata(data);
+                } else if (data.type === 'working_times') {
+                    handleWorkingTimeUpdate(data.working_time_data)
                 }
-                $row = $row.replaceWithPush(data.html);
-                $row.find('time').timeago();
-                $row.click(function () {
-                    Turbolinks.visit($(this).data("href"));
-                });
             }
         });
+
+        function handleNewRfCdata(data) {
+            let $row = $('tr[data-id="' + data.id + '"]');
+            if ($row.length === 0) {
+                $row = $($('#posted_rfcs')[0].insertRow(0));
+            }
+            $row = $row.replaceWithPush(data.html);
+            $row.find('time').timeago();
+            $row.click(function () {
+                Turbolinks.visit($(this).data("href"));
+            });
+        }
+
+        function handleWorkingTimeUpdate(data) {
+            const user_progress = data['user_progress'];
+            const additional_user_data = data['additional_user_data'];
+
+            const user = additional_user_data[additional_user_data.length - 1][0];
+            const position = userPosition[user.type + user.id]; // TODO validate: will result in undef. if not existent.
+            // TODO: Do update
+        }
+
+        const graph_data = $('#initial_graph_data').data('graph_data');
+        let userPosition = {};
+
+        drawGraph(graph_data);
+
+        function drawGraph(graph_data) {
+            const user_progress = graph_data['user_progress'];
+            const additional_user_data = graph_data['additional_user_data'];
+
+            function get_minutes (time_stamp) {
+                try {
+                    hours = time_stamp.split(":")[0];
+                    minutes = time_stamp.split(":")[1];
+                    seconds = time_stamp.split(":")[2];
+                    seconds /= 60;
+                    minutes = parseFloat(hours * 60) + parseInt(minutes) + seconds;
+                    if (minutes > 0){
+                        return minutes;
+                    } else{
+                        return parseFloat(seconds/60);
+                    }
+                } catch (err) {
+                    return 0;
+                }
+            }
+
+            function learners_name(index) {
+                return additional_user_data[additional_user_data.length - 1][index]["name"] + ", ID: " + additional_user_data[additional_user_data.length - 1][index]["id"];
+            }
+
+            function learners_time(group, index) {
+                if (user_progress[group] !== null && user_progress[group] !== undefined && user_progress[group][index] !== null) {
+                    return user_progress[group][index]
+                } else {
+                    return 0;
+                }
+            }
+
+            if (user_progress.length === 0) {
+                // No data available
+                $('#no_chart_data').removeClass("d-none");
+                return;
+            }
+
+            const margin = ({top: 20, right: 20, bottom: 150, left: 80});
+            const width = $('#chart_stacked').width();
+            const height = 500;
+            const users = user_progress[0].length; // # of users
+            const n = user_progress.length; // # of different sub bars, called buckets
+
+            let working_times_in_minutes = d3.range(n).map((index) => {
+                if (user_progress[index] !== null) {
+                    return user_progress[index].map((time) => get_minutes(time))
+                } else return new Array(users).fill(0);
+            });
+
+            let xAxis = svg => svg.append("g")
+                .attr("transform", `translate(0,${height - margin.bottom})`)
+                .call(d3.axisBottom(x).tickSizeOuter(0).tickFormat((index) => learners_name(index)));
+
+            let yAxis = svg => svg.append("g")
+                .attr("transform", `translate(${margin.left}, 0)`)
+                .call(d3.axisLeft(y).tickSizeOuter(0).tickFormat((index) => index));
+
+            let color = d3.scaleSequential(d3.interpolateRdYlGn)
+                .domain([-0.5 * n, 1.5 * n]);
+
+            let userAxis = d3.range(users); // the x-values shared by all series
+
+            // Calculate the corresponding start and end value of each value;
+            const yBarValuesGrouped = d3.stack()
+                .keys(d3.range(n))
+                (d3.transpose(working_times_in_minutes)) // stacked working_times_in_minutes
+                .map((data, i) => data.map(([y0, y1]) => [y0, y1, i]));
+
+            const maxYSingleBar = d3.max(working_times_in_minutes, y => d3.max(y));
+
+            const maxYBarStacked = d3.max(yBarValuesGrouped, y => d3.max(y, d => d[1]));
+
+            let x = d3.scaleBand()
+                .domain(userAxis)
+                .rangeRound([margin.left, width - margin.right])
+                .padding(0.08);
+
+            let y = d3.scaleLinear()
+                .domain([0, maxYBarStacked])
+                .range([height - margin.bottom, margin.top]);
+
+            const svg = d3.select("#chart_stacked")
+                .append("svg")
+                .attr("width", '100%')
+                .attr("height", '100%')
+                .attr("viewBox", `0 0 ${width} ${height}`)
+                .attr("preserveAspectRatio","xMinYMin meet");
+
+            const rect = svg.selectAll("g")
+                .data(yBarValuesGrouped)
+                .enter().append("g")
+                .attr("fill", (d, i) => color(i))
+                .selectAll("rect")
+                .data(d => d)
+                .join("rect")
+                .attr("x", (d, i) => x(i))
+                .attr("y", height - margin.bottom)
+                .attr("width", x.bandwidth())
+                .attr("height", 0)
+                .attr("class", (d) => "bar-stacked-"+d[2]);
+
+            svg.append("g")
+                .attr("class", "x axis")
+                .call(xAxis)
+                .selectAll("text")
+                .style("text-anchor", "end")
+                .attr("dx", "-.8em")
+                .attr("dy", ".15em")
+                .attr("transform", function(d) {
+                    return "rotate(-45)"
+                });
+
+            svg.append("g")
+                .attr("class", "y axis")
+                .call(yAxis);
+
+            // Y Axis Label
+            svg.append("text")
+                .attr("transform", "rotate(-90)")
+                .attr("x", (-height - margin.top + margin.bottom) / 2)
+                .attr("dy", "+2em")
+                .style("text-anchor", "middle")
+                .text(I18n.t('exercises.study_group_dashboard.time_spent_in_minutes'))
+                .style('font-size', 14);
+
+            // X Axis Label
+            svg.append("text")
+                .attr("class", "x axis")
+                .attr("text-anchor", "middle")
+                .attr("x", (width + margin.left - margin.right) / 2)
+                .attr("y", height)
+                .attr("dy", '-1em')
+                .text(I18n.t('exercises.study_group_dashboard.learner'))
+                .style('font-size', 14);
+
+            let tip = d3.tip()
+                .attr('class', 'd3-tip')
+                .offset([-10, 0])
+                .html(function(d, i, a) {
+                    return "<strong>Student: </strong><span style='color:orange'>" + learners_name(i) + "</span><br/>" +
+                        "0: " + learners_time(0, i) + "<br/>" +
+                        "1: " + learners_time(1, i) + "<br/>" +
+                        "2: " + learners_time(2, i) + "<br/>" +
+                        "3: " + learners_time(3, i) + "<br/>" +
+                        "4: " + learners_time(4, i);
+                });
+
+            svg.call(tip);
+
+            rect.on('mouseenter', tip.show)
+                .on('mouseout', tip.hide);
+
+            function transitionGrouped() {
+                // Show all sub-bars next to each other
+                y.domain([0, maxYSingleBar]);
+
+                rect.transition()
+                    .duration(500)
+                    .delay((d, i) => i * 20)
+                    .attr("x", (d, i) => x(i) + x.bandwidth() / n * d[2])
+                    .attr("width", x.bandwidth() / n)
+                    .transition()
+                    .attr("y", d => y(d[1] - d[0]))
+                    .attr("height", d => y(0) - y(d[1] - d[0]));
+            }
+
+            function transitionStacked() {
+                // Show all sub-bars on top of each other
+                y.domain([0, maxYBarStacked]);
+
+                rect.transition()
+                    .duration(500)
+                    .delay((d, i) => i * 20)
+                    .attr("y", d => y(d[1]))
+                    .attr("height", d => y(d[0]) - y(d[1]))
+                    .transition()
+                    .attr("x", (d, i) => x(i))
+                    .attr("width", x.bandwidth());
+            }
+
+            $('#no_chart_data').addClass("d-none");
+            transitionStacked();
+            // transitionGrouped();
+
+            buildDictionary(additional_user_data);
+        }
+
+        function buildDictionary(users) {
+            users[users.length - 1].forEach(function(user, index) {
+                userPosition[user.type + user.id] = index;
+            });
+        }
     }
 });

app/assets/stylesheets/exercises.css.scss

@@ -84,6 +84,10 @@ div#chart_2 {
   background-color: #FAFAFA;
 }
 
+div#chart_stacked {
+  max-height: 500px;
+  background-color: #FAFAFA;
+}
 
 a.file-heading {
   color: black !important;
@@ -115,7 +119,7 @@ a.file-heading {
 .d3-tip:after {
   box-sizing: border-box;
   display: inline;
-  font-size: 10px;
+  font-size: 14px;
   width: 100%;
   line-height: 1;
   color: rgba(0, 0, 0, 0.8);
@@ -126,7 +130,7 @@ a.file-heading {
 
 /* Style northward tooltips differently */
 .d3-tip.n:after {
-  margin: -1px 0 0 0;
+  margin: -3px 0 0 0;
   top: 100%;
   left: 0;
 }

app/controllers/exercises_controller.rb

@@ -482,6 +482,8 @@ class ExercisesController < ApplicationController
             where(exercise: @exercise).includes(:submission).
             where(submissions: {study_group_id: @study_group_id}).
             order(created_at: :desc)
+
+    @graph_data = @exercise.get_working_times_for_study_group(@study_group_id)
   end
 
 end

app/helpers/action_cable_helper.rb

@@ -1,8 +1,10 @@
 module ActionCableHelper
   def trigger_rfc_action_cable
+    # Context: RfC
     if submission.study_group_id.present?
       ActionCable.server.broadcast(
         "la_exercises_#{exercise_id}_channel_study_group_#{submission.study_group_id}",
+        type: :rfc,
         id: id,
         html: (ApplicationController.render(partial: 'request_for_comments/list_entry',
                                             locals: {request_for_comment: self})))
@@ -10,6 +12,19 @@ module ActionCableHelper
   end
 
   def trigger_rfc_action_cable_from_comment
+    # Context: Comment
     RequestForComment.find_by(submission: file.context).trigger_rfc_action_cable
   end
+
+  def trigger_working_times_action_cable
+    # Context: Submission
+    if study_group_id.present?
+      ActionCable.server.broadcast(
+          "la_exercises_#{exercise_id}_channel_study_group_#{study_group_id}",
+          type: :working_times,
+          working_time_data: exercise.get_working_times_for_study_group(study_group_id, user))
+    end
+  end
 end
+
+# TODO: Check if any user is connected and prevent preparing the data otherwise

app/models/exercise.rb

@@ -90,6 +90,145 @@ class Exercise < ApplicationRecord
     "
   end
 
+  def study_group_working_time_query(exercise_id, study_group_id, additional_filter)
+    """
+    WITH working_time_between_submissions AS (
+      SELECT submissions.user_id,
+         submissions.user_type,
+         score,
+         created_at,
+         (created_at - lag(created_at) over (PARTITION BY submissions.user_id, exercise_id
+           ORDER BY created_at)) AS working_time
+  FROM submissions
+  WHERE exercise_id = #{exercise_id} AND study_group_id = #{study_group_id} #{additional_filter}),
+        working_time_with_deltas_ignored AS (
+          SELECT user_id,
+                 user_type,
+                 score,
+                 sum(CASE WHEN score IS NOT NULL THEN 1 ELSE 0 END)
+                     over (ORDER BY user_type, user_id, created_at)                 AS change_in_score,
+                 created_at,
+                 CASE WHEN working_time >= #{StatisticsHelper::WORKING_TIME_DELTA_IN_SQL_INTERVAL} THEN '0' ELSE working_time END AS working_time_filtered
+          FROM working_time_between_submissions
+        ),
+        working_times_with_score_expanded AS (
+          SELECT user_id,
+                 user_type,
+                 created_at,
+                 working_time_filtered,
+                 first_value(score)
+                             over (PARTITION BY user_type, user_id, change_in_score ORDER BY created_at ASC) AS corrected_score
+          FROM working_time_with_deltas_ignored
+        ),
+        working_times_with_duplicated_last_row_per_score AS (
+          SELECT *
+          FROM working_times_with_score_expanded
+          UNION ALL
+          -- Duplicate last row per score and make it unique by setting another created_at timestamp.
+          -- In addition, the working time is set to zero in order to prevent getting a wrong time.
+          -- This duplication is needed, as we will shift the scores and working times by one and need to ensure not to loose any information.
+          SELECT DISTINCT ON (user_type, user_id, corrected_score) user_id,
+                                                                   user_type,
+                                                                   created_at + INTERVAL '1us',
+                                                                   '00:00:00' as working_time_filtered,
+                                                                   corrected_score
+          FROM working_times_with_score_expanded
+        ),
+        working_times_with_score_not_null_and_shifted AS (
+          SELECT user_id,
+                 user_type,
+                 coalesce(lag(corrected_score) over (PARTITION BY user_type, user_id ORDER BY created_at ASC),
+                          0) AS shifted_score,
+                 created_at,
+                 working_time_filtered
+          FROM working_times_with_duplicated_last_row_per_score
+        ),
+        working_times_to_be_sorted AS (
+          SELECT user_id,
+                 user_type,
+                 shifted_score                                                          AS score,
+                 MIN(created_at)                                                        AS start_time,
+                 SUM(working_time_filtered)                                             AS working_time,
+                 SUM(SUM(working_time_filtered)) over (PARTITION BY user_type, user_id) AS total_working_time
+          FROM working_times_with_score_not_null_and_shifted
+          GROUP BY user_id, user_type, score
+        ),
+        working_times_with_index AS (
+          SELECT (dense_rank() over (ORDER BY total_working_time, user_type, user_id ASC) - 1) AS index,
+                 user_id,
+                 user_type,
+                 score,
+                 start_time,
+                 working_time,
+                 total_working_time
+          FROM working_times_to_be_sorted)
+        SELECT index,
+           user_id,
+           user_type,
+           name,
+           score,
+           start_time,
+           working_time,
+           total_working_time
+        FROM working_times_with_index
+           JOIN external_users ON user_type = 'ExternalUser' AND user_id = external_users.id
+        UNION ALL
+        SELECT index,
+           user_id,
+           user_type,
+           name,
+           score,
+           start_time,
+           working_time,
+           total_working_time
+        FROM working_times_with_index
+           JOIN internal_users ON user_type = 'InternalUser' AND user_id = internal_users.id
+        ORDER BY index, score ASC LIMIT 200;
+    """
+  end
+
+  def get_working_times_for_study_group(study_group_id, user = nil)
+    user_progress = []
+    additional_user_data = []
+    max_bucket = 4
+    maximum_score = self.maximum_score
+
+    if user.blank?
+      additional_filter = ''
+    else
+      additional_filter = "AND user_id = #{user.id} AND user_type = '#{user.class.name}'"
+    end
+
+    results = self.class.connection.execute(study_group_working_time_query(id, study_group_id, additional_filter)).each do |tuple|
+      if tuple['score'] <= maximum_score
+        bucket = tuple['score'] / maximum_score * max_bucket
+      else
+        bucket = max_bucket # maximum_score / maximum_score will always be 1
+      end
+
+      user_progress[bucket] ||= []
+      additional_user_data[bucket] ||= []
+      additional_user_data[max_bucket + 1] ||= []
+
+      user_progress[bucket][tuple['index']] = tuple["working_time"]
+      additional_user_data[bucket][tuple['index']] = {start_time: tuple["start_time"], score: tuple["score"]}
+      additional_user_data[max_bucket + 1][tuple['index']] = {id: tuple['user_id'], type: tuple['user_type'], name: tuple['name']}
+    end
+
+    if results.ntuples > 0
+      first_index = results[0]['index']
+      last_index = results[results.ntuples-1]['index']
+      buckets = last_index - first_index
+      user_progress.each do |timings_array|
+        if timings_array.present? && timings_array.length != buckets + 1
+          timings_array[buckets] = nil
+        end
+      end
+    end
+
+    {user_progress: user_progress, additional_user_data: additional_user_data}
+  end
+
   def get_quantiles(quantiles)
     quantiles_str = "[" + quantiles.join(",") + "]"
     result = self.class.connection.execute("""

app/models/submission.rb

@@ -1,6 +1,7 @@
 class Submission < ApplicationRecord
   include Context
   include Creation
+  include ActionCableHelper
 
   CAUSES = %w(assess download file render run save submit test autosave requestComments remoteAssess)
   FILENAME_URL_PLACEHOLDER = '{filename}'
@@ -20,6 +21,8 @@ class Submission < ApplicationRecord
   validates :cause, inclusion: {in: CAUSES}
   validates :exercise_id, presence: true
 
+  after_save :trigger_working_times_action_cable
+
   MAX_COMMENTS_ON_RECOMMENDED_RFC = 5
 
   def build_files_hash(files, attribute)

app/views/exercises/study_group_dashboard.html.slim

@@ -1,9 +1,25 @@
+- content_for :head do
+  // Force a full page reload, see https://github.com/turbolinks/turbolinks/issues/326.
+     Otherwise, code might not be highlighted correctly (race condition)
+  meta name='turbolinks-visit-control' content='reload'
+  = javascript_pack_tag('d3-tip', 'data-turbolinks-track': true)
+
 h1
   = t('.live_dashboard')
 
 div.teacher_dashboard data-exercise-id="#{@exercise.id}" data-study-group-id="#{@study_group_id}"
 
-h4
+h4.mt-4
+  = t('.time_spent_per_learner')
+
+.d-none#initial_graph_data data-graph_data=ActiveSupport::JSON.encode(@graph_data);
+div.w-100#chart_stacked
+
+.d-none.badge-info.container.py-2#no_chart_data
+  i class="fa fa-info" aria-hidden="true"
+  = t('.no_data_yet')
+
+h4.mt-4
   = t('.related_requests_for_comments')
 
 .table-responsive
@@ -17,5 +33,5 @@ h4
         th.col-12 = t('activerecord.attributes.request_for_comments.question')
         th = t('activerecord.attributes.request_for_comments.username')
         th.text-nowrap = t('activerecord.attributes.request_for_comments.requested_at')
-      tbody#posted_rfcs
-        = render(partial: 'request_for_comments/list_entry', collection: @request_for_comments, as: :request_for_comment)
+    tbody#posted_rfcs
+      = render(partial: 'request_for_comments/list_entry', collection: @request_for_comments, as: :request_for_comment)

config/locales/de.yml

@@ -369,7 +369,11 @@ de:
       full_score_redirect_to_own_rfc: Herzlichen Glückwunsch! Sie haben die maximale Punktzahl für diese Aufgabe an den Kurs übertragen. Ihre Frage ist damit wahrscheinlich gelöst? Falls ja, fügen Sie doch den entscheidenden Kniff als Antwort hinzu und markieren die Frage als gelöst, bevor sie das Fenster schließen.
     study_group_dashboard:
       live_dashboard: Live Dashboard
+      time_spent_per_learner: Verwendete Zeit pro Lerner
       related_requests_for_comments: Zugehörige Kommentaranfragen
+      learner: Lerner
+      time_spent_in_minutes: benötigte Zeit in Minuten
+      no_data_yet: Bisher sind keine Daten verfügbar
     external_users:
       statistics:
         no_data_available: Keine Daten verfügbar.

config/locales/en.yml

@@ -369,7 +369,11 @@ en:
       full_score_redirect_to_own_rfc: Congratulations! You achieved and submitted the highest possible score for this exercise. Your question concerning the exercise is solved? If so, please share the essential insight with your fellows and mark the question as solved, before you close this window!
     study_group_dashboard:
       live_dashboard: Live Dashboard
+      time_spent_per_learner: Time spent per Learner
       related_requests_for_comments: Related Requests for Comments
+      learner: Learner
+      time_spent_in_minutes: Time spent in Minutes
+      no_data_yet: No data available yet
     external_users:
       statistics:
         no_data_available: No data available.