Manually merge changes from webpython branch.

Gemfile

@@ -33,6 +33,8 @@ gem 'thread_safe'
 gem 'turbolinks'
 gem 'uglifier', '>= 1.3.0'
 gem 'will_paginate', '~> 3.0'
+gem 'tubesock'
+gem 'faye-websocket'
 
 group :development do
   gem 'better_errors', platform: :ruby

app/assets/javascripts/editor.js

@@ -20,6 +20,14 @@ $(function() {
   var qa_api = undefined;
   var output_mode_is_streaming = true;
 
+  var websocket,
+      turtlescreen,
+      numMessages = 0,
+      turtlecanvas = $('#turtlecanvas'),
+      prompt = $('#prompt'),
+      commands = ['input', 'write', 'turtle', 'turtlebatch'],
+      streams = ['stdin', 'stdout', 'stderr'];
+
   var flowrResultHtml = '<div class="panel panel-default"><div id="{{headingId}}" role="tab" class="panel-heading"><h4 class="panel-title"><a data-toggle="collapse" data-parent="#flowrHint" href="#{{collapseId}}" aria-expanded="true" aria-controls="{{collapseId}}"></a></h4></div><div id="{{collapseId}}" role="tabpanel" aria-labelledby="{{headingId}}" class="panel-collapse collapse"><div class="panel-body"></div></div></div>'
 
   var ajax = function(options) {
@@ -181,7 +189,13 @@ $(function() {
   };
 
   var evaluateCodeWithStreamedResponse = function(url, callback) {
-    var event_source = new EventSource(url);
+    initWebsocketConnection(url);
+
+    // TODO only init turtle when required
+    initTurtle();
+
+    // TODO reimplement via websocket messsages
+    /*var event_source = new EventSource(url);
 
     event_source.addEventListener('close', closeEventSource);
     event_source.addEventListener('error', closeEventSource);
@@ -201,7 +215,7 @@ $(function() {
 
     event_source.addEventListener('status', function(event) {
       showStatus(JSON.parse(event.data));
-    });
+    });*/
   };
 
   var handleStreamedResponseForCodePilot = function(event) {
@@ -1065,6 +1079,183 @@ $(function() {
     $('#request-for-comments').toggle(isActiveFileSubmission() && !isActiveFileBinary());
   };
 
+  var initWebsocketConnection = function(url) {
+      websocket = new WebSocket('ws://' + window.location.hostname + ':' + window.location.port + url);
+      websocket.onopen = function(evt) { onWebSocketOpen(evt) };
+      websocket.onclose = function(evt) { onWebSocketClose(evt) };
+      websocket.onmessage = function(evt) { onWebSocketMessage(evt) };
+      websocket.onerror = function(evt) { onWebSocketError(evt) };
+      websocket.flush = function() { this.send('\n'); }
+  };
+
+  var initTurtle = function() {
+      turtlescreen = new Turtle(websocket, $('#turtlecanvas'));
+  };
+
+  var initCallbacks = function() {
+      if ($('#run').isPresent()) {
+          $('#run').bind('click', function(event) {
+              hideCanvas();
+              hidePrompt();
+          });
+      }
+      if ($('#prompt').isPresent()) {
+          $('#prompt').on('keypress', handlePromptKeyPress);
+          $('#prompt-submit').on('click', submitPromptInput);
+      }
+  }
+
+  var onWebSocketOpen = function(evt) {
+      //alert("Session started");
+  };
+
+  var onWebSocketClose = function(evt) {
+      //alert("Session terminated");
+  };
+
+  var onWebSocketMessage = function(evt) {
+      numMessages++;
+      parseCanvasMessage(evt.data, true);
+  };
+
+  var onWebSocketError = function(evt) {
+      //alert("Something went wrong.")
+  };
+
+  var executeCommand = function(msg) {
+      if ($.inArray(msg.cmd, commands) == -1) {
+          console.log("Unknown command: " + msg.cmd);
+          // skipping unregistered commands is required
+          // as we may receive mirrored response due to internal behaviour
+          return;
+      }
+      switch(msg.cmd) {
+          case 'input':
+              showPrompt();
+              break;
+          case 'write':
+              printWebsocketOutput(msg);
+              break;
+          case 'turtle':
+              showCanvas();
+              handleTurtleCommand(msg);
+              break;
+          case 'turtlebatch':
+              showCanvas();
+              handleTurtlebatchCommand(msg);
+              break;
+      }
+  };
+
+  // todo reuse method from editor.js
+  var printWebsocketOutput = function(msg) {
+      var element = findOrCreateOutputElement(0);
+      console.log(element);
+      switch (msg.stream) {
+          case 'internal':
+              element.addClass('text-danger');
+              break;
+          case 'stderr':
+              element.addClass('text-warning');
+              break;
+          case 'stdout':
+          case 'stdin': // for eventual prompts
+          default:
+              element.addClass('text-muted');
+      }
+      element.append(msg.data)
+  };
+
+  var handleTurtleCommand = function(msg) {
+      if (msg.action in turtlescreen) {
+          result = turtlescreen[msg.action].apply(turtlescreen, msg.args);
+          websocket.send(JSON.stringify({cmd: 'result', 'result': result}));
+      } else {
+          websocket.send(JSON.stringify({cmd: 'exception', exception: 'AttributeError', message: msg.action}));
+      }
+      websocket.flush();
+  };
+
+  var handleTurtlebatchCommand = function(msg) {
+      for (i = 0; i < msg.batch.length; i++) {
+          cmd = msg.batch[i];
+          turtlescreen[cmd[0]].apply(turtlescreen, cmd[1]);
+      }
+  };
+
+  var handlePromptKeyPress = function(evt) {
+      if (evt.which === ENTER_KEY_CODE) {
+          submitPromptInput();
+      }
+  }
+
+  var submitPromptInput = function() {
+      var input = $('#prompt-input');
+      var message = input.val();
+      websocket.send(JSON.stringify({cmd: 'result', 'data': message}));
+      websocket.flush();
+      input.val('');
+      hidePrompt();
+  }
+
+  var parseCanvasMessage = function(message, recursive) {
+      var msg;
+      message = message.replace(/^\s+|\s+$/g, "");
+      try {
+          // todo validate json instead of catching
+          msg = JSON.parse(message);
+      } catch (e) {
+          if (!recursive) {
+              return false;
+          }
+          // why does docker sometimes send multiple commands at once?
+          message = message.replace(/^\s+|\s+$/g, "");
+          messages = message.split("\n");
+          for (var i = 0; i < messages.length; i++) {
+              if (!messages[i]) {
+                  continue;
+              }
+              parseCanvasMessage(messages[i], false);
+          }
+          return;
+      }
+      executeCommand(msg);
+  };
+
+  var showPrompt = function() {
+      if (prompt.isPresent() && prompt.hasClass('hidden')) {
+          prompt.removeClass('hidden');
+      }
+      prompt.focus();
+  }
+
+  var hidePrompt = function() {
+      if (prompt.isPresent() && !prompt.hasClass('hidden')) {
+          console.log("hiding prompt2");
+          prompt.addClass('hidden');
+      }
+  }
+
+  var showCanvas = function() {
+      if ($('#turtlediv').isPresent()
+              && turtlecanvas.hasClass('hidden')) {
+          // initialize two-column layout
+          $('#output-col1').addClass('col-lg-7 col-md-7 two-column');
+          turtlecanvas.removeClass('hidden');
+      }
+  };
+
+  var hideCanvas = function() {
+      if ($('#turtlediv').isPresent()
+              && !(turtlecanvas.hasClass('hidden'))) {
+          output = $('#output-col1');
+          if (output.hasClass('two-column')) {
+              output.removeClass('col-lg-7 col-md-7 two-column');
+          }
+          turtlecanvas.addClass('hidden');
+      }
+  };
+
   var requestComments = function(e) {
     var user_id = $('#editor').data('user-id')
     var exercise_id = $('#editor').data('exercise-id')

app/assets/javascripts/turtle.js

@@ -0,0 +1,222 @@
+var output;
+var editor;
+var pipeurl;
+var filename;
+var pendingChanges = -1;
+
+function Turtle(pipe, canvas) {
+    var dx, dy, xpos, ypos;
+    this.canvas = canvas; // jQuery object
+    this.items = [];
+    this.canvas.off('click');
+    this.canvas.click(function (e) {
+        if (e.eventPhase !== 2) {
+            return;
+        }
+        e.stopPropagation();
+        dx = this.width / 2;
+        dy = this.height / 2;
+        if(e.offsetX==undefined)
+        {
+            var offset = canvas.offset();
+            xpos = e.pageX-offset.left;
+            ypos = e.pageY-offset.top;
+        }
+        else
+        {
+            xpos = e.offsetX;
+            ypos = e.offsetY;
+        }
+        pipe.send(JSON.stringify({
+            'cmd': 'canvasevent',
+            'type': '<Button-1>',
+            'x': xpos - dx,
+            'y': ypos - dy
+        }));
+    });
+}
+
+Turtle.prototype.update = function () {
+    var i, k, canvas, ctx, dx, dy, item, c, length;
+    canvas = this.canvas[0];
+    ctx = canvas.getContext('2d');
+    ctx.fillStyle = '#fff';
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+    length = this.items.length;
+    dx = canvas.width / 2;
+    dy = canvas.height / 2;
+    for (i = 0; i < length; i += 1) {
+        item = this.items[i];
+        c = item.coords;
+        switch (item.type) {
+        case 'line':
+            ctx.beginPath();
+            ctx.moveTo(c[0] + dx, c[1] + dy);
+            for (k = 2; k < c.length; k += 2) {
+                ctx.lineTo(c[k] + dx, c[k + 1] + dy);
+            }
+            if (this.fill) {
+                ctx.strokeStyle = this.fill;
+            }
+
+            ctx.stroke();
+            break;
+        case 'polygon':
+            ctx.beginPath();
+            ctx.moveTo(c[0] + dx, c[1] + dy);
+            for (k = 2; k < c.length; k += 2) {
+                ctx.lineTo(c[k] + dx, c[k + 1] + dy);
+            }
+            ctx.closePath();
+            if (item.fill !== "") {
+                ctx.fillStyle = item.fill;
+                ctx.strokeStyle = item.fill;
+                ctx.fill();
+            }
+            ctx.stroke();
+            break;
+        case 'image':
+            break;
+        }
+    }
+}
+
+Turtle.prototype.get_width = function () {
+    return this.canvas[0].width;
+}
+
+Turtle.prototype.get_height = function () {
+    return this.canvas[0].height;
+}
+
+Turtle.prototype.delete = function (item) {
+    if (item == 'all') {
+        this.items = [];
+    } else {
+        delete this.items[item];
+    }
+}
+
+Turtle.prototype.create_image = function (image) {
+    this.items.push({type:'image',image:image});
+    return this.items.length - 1;
+}
+
+Turtle.prototype.create_line = function () {
+    this.items.push({type:'line',
+                     fill: '',
+                     coords:[0,0,0,0],
+                     width:2,
+                     capstyle:'round'});
+    return this.items.length - 1;
+}
+
+Turtle.prototype.create_polygon = function () {
+    this.items.push({type:'polygon',
+                     // fill: "" XXX
+                     // outline: "" XXX
+                     coords:[0,0,0,0,0,0]
+                    });
+    return this.items.length - 1;
+}
+
+// XXX might make this varargs as in Tkinter
+Turtle.prototype.coords = function (item, coords) {
+    if (coords === undefined) {
+        return this.items[item].coords;
+    }
+    this.items[item].coords = coords;
+}
+
+Turtle.prototype.itemconfigure = function (item, key, value) {
+    this.items[item][key] = value;
+}
+
+// value might be undefined
+Turtle.prototype.css = function (key, value) {
+    if (value === undefined) {
+        return this.canvas.css(key);
+    } else {
+        // jQuery return value is confusing when the css is set
+        this.canvas.css(key, value);
+    }
+}
+
+function run(launchmsg) {
+    var i, turtlescreen, msg, result, cmd;
+    $('#assess').empty();
+
+    turtlescreen = new Turtle();
+
+    output = $('#output');
+    output.empty();
+    if (typeof pipeurl === 'undefined') {
+        if (wp_port == '443') {
+            pipeurl = 'wss://'+wp_hostname+'/pipe';
+        } else {
+            pipeurl = 'ws://'+wp_hostname+':'+wp_port+'/pipe';
+        }
+    }
+    saveFile();
+    output.pipe = new WebSocket(pipeurl);
+    output.pipe.onopen = function () {
+        output.pipe.send(JSON.stringify(launchmsg));
+    };
+    output.pipe.onmessage = function (response) {
+        msg = JSON.parse(response.data);
+        if (msg.cmd == 'input') {
+            output.inputelem = $('<input>',{'size':40});
+            submit = $('<input>',{'type':'submit'});
+            submit.click(function (){
+                text = output.inputelem.val();
+                output.input.replaceWith($('<code>', {text:text+'\n'}));
+                output.pipe.send(JSON.stringify({'cmd':'inputresult',
+                                                 'data':text}));
+            });
+        output.inputelem.keydown(function(event){
+                if(event.keyCode == 13){
+                    submit.click();
+                }
+        });
+            output.append($('<code>', {text:msg.data}));
+            output.input = $('<span>').append(output.inputelem).append(submit);
+            output.append(output.input);
+        output.inputelem.focus();
+        } else if (msg.cmd == 'stop') {
+            if (launchmsg.cmd == 'runscript') {
+                if (msg.timedout) {
+                    output.append('<hr>Dein Programm hat zu lange gerechnet und wurde beendet.');
+                } else {
+                    output.append('<hr>Dein Progamm wurde beendet');
+                }
+            }
+            output.pipe.close();
+        } else if (msg.cmd == 'passed') {
+            $('#assess').html("Herzlich Glückwunsch! Dein Programm funktioniert korrekt.");
+        } else if (msg.cmd == 'failed') {
+            $('#assess').html(msg.data);
+        } else if (msg.cmd == 'turtle') {
+            if (msg.action in turtlescreen) {
+                result = turtlescreen[msg.action].apply(turtlescreen, msg.args);
+                output.pipe.send(JSON.stringify({cmd:'result', 'result':result}));
+            } else {
+                output.pipe.send(JSON.stringify({cmd:'exception', exception:'AttributeError',
+                                                 message:msg.action}));
+            }
+        } else if (msg.cmd == 'turtlebatch') {
+            for (i=0; i < msg.batch.length; i += 1) {
+                cmd = msg.batch[i];
+                turtlescreen[cmd[0]].apply(turtlescreen, cmd[1]);
+            }
+        } else {
+            if(msg.stream == 'internal') {
+                output.append('<hr><em>Interner Fehler (bitte melden):</em>\n');
+            }
+        else if (msg.stream == 'stderr') {
+        showConsole();
+        $('#consoleradio').prop('checked', 'checked');
+        }
+            output.append($('<code>',{text:msg.data, 'class':msg.stream}));
+        }
+    };
+}

app/controllers/submissions_controller.rb

@@ -4,6 +4,7 @@ class SubmissionsController < ApplicationController
   include Lti
   include SubmissionParameters
   include SubmissionScoring
+  include Tubesock::Hijack
 
   before_action :set_submission, only: [:download_file, :render_file, :run, :score, :show, :statistics, :stop, :test]
   before_action :set_docker_client, only: [:run, :test]
@@ -70,20 +71,58 @@ class SubmissionsController < ApplicationController
   end
 
   def run
-    with_server_sent_events do |server_sent_event|
+    # with_server_sent_events do |server_sent_event|
-      output = @docker_client.execute_run_command(@submission, params[:filename])
+    #   output = @docker_client.execute_run_command(@submission, params[:filename])
 
-      server_sent_event.write({stdout: output[:stdout]}, event: 'output') if output[:stdout]
+    #   server_sent_event.write({stdout: output[:stdout]}, event: 'output') if output[:stdout]
-      server_sent_event.write({stderr: output[:stderr]}, event: 'output') if output[:stderr]
+    #   server_sent_event.write({stderr: output[:stderr]}, event: 'output') if output[:stderr]
 
-      server_sent_event.write({status: output[:status]}, event: 'status')
+    #   server_sent_event.write({status: output[:status]}, event: 'status')
 
-      unless output[:stderr].nil?
+    #   unless output[:stderr].nil?
-        if hint = Whistleblower.new(execution_environment: @submission.execution_environment).generate_hint(output[:stderr])
+    #     if hint = Whistleblower.new(execution_environment: @submission.execution_environment).generate_hint(output[:stderr])
-          server_sent_event.write(hint, event: 'hint')
+    #       server_sent_event.write(hint, event: 'hint')
-        else
+    #     else
-          store_error(output[:stderr])
+    #       store_error(output[:stderr])
+    #     end
+    #   end
+    # end
+
+    hijack do |tubesock|
+      Thread.new { EventMachine.run } unless EventMachine.reactor_running? && EventMachine.reactor_thread.alive?
+
+      result = @docker_client.execute_run_command(@submission, params[:filename])
+      socket = result[:socket]
+
+      socket.on :message do |event|
+        puts "Docker sending: " + event.data
+        parse_message(event.data, 'stdout', tubesock)
+      end
+
+      tubesock.onmessage do |data|
+          puts "Client sending: " + data
+          res = socket.send data
+          if res == false
+              puts "Something is wrong."
+          end
+      end
+    end
+  end
+
+  def parse_message(message, output_stream, socket, recursive = true)
+    begin
+      parsed = JSON.parse(message)
+      socket.send_data message
+    rescue JSON::ParserError => e
+      print "1\n"
+      if ((recursive == true) && (message.include? "\n"))
+        print "3\n"
+        for part in message.split("\n")
+          self.parse_message(part,output_stream,socket,false)
         end
+      else
+        parsed = {'cmd'=>'write','stream'=>output_stream,'data'=>message}
+        socket.send_data JSON.dump(parsed)
       end
     end
   end

app/views/exercises/implement.html.slim

@@ -45,12 +45,24 @@
             .panel.panel-warning
               .panel-heading = t('.hint')
               .panel-body
-          #output
+          .row
-            pre = t('.no_output_yet')
+            #output-col1
-          - if CodeOcean::Config.new(:code_ocean).read[:flowr][:enabled]
+              // todo set to full width if turtle isnt used
-            #flowrHint.panel.panel-info data-url=CodeOcean::Config.new(:code_ocean).read[:flowr][:url] role='tab'
+              #prompt.input-group.hidden
-              .panel-heading = 'Gain more insights here'
+                span.input-group-addon = 'Your input'
-              .panel-body
+                input#prompt-input.form-control type='text'
+                span.input-group-btn
+                  button#prompt-submit.btn.btn-primary type="button" = 'Send'
+              #output
+                pre = t('.no_output_yet')
+              - if CodeOcean::Config.new(:code_ocean).read[:flowr][:enabled]
+                #flowrHint.panel.panel-info data-url=CodeOcean::Config.new(:code_ocean).read[:flowr][:url] role='tab'
+                  .panel-heading = 'Gain more insights here'
+                  .panel-body
+            #output-col2.col-lg-5.col-md-5
+              #turtlediv
+                // todo what should the canvas default size be?
+                canvas#turtlecanvas.hidden style='border-style:solid;border-width:thin'
         #progress.tab-pane
           #results
             h2 = t('.results')

lib/docker_client.rb

@@ -11,6 +11,7 @@ class DockerClient
   RETRY_COUNT = 2
 
   attr_reader :container
+  attr_reader :socket
 
   def self.check_availability!
     Timeout.timeout(config[:connection_timeout]) { Docker.version }
@@ -41,7 +42,12 @@ class DockerClient
       'Memory' => execution_environment.memory_limit.megabytes,
       'NetworkDisabled' => !execution_environment.network_enabled?,
       'OpenStdin' => true,
-      'StdinOnce' => true
+      'StdinOnce' => true,
+      # required to expose standard streams over websocket
+      'AttachStdout' => true,
+      'AttachStdin' => true,
+      'AttachStderr' => true,
+      'Tty' => true
     }
   end
 
@@ -52,6 +58,29 @@ class DockerClient
     }
   end
 
+  def create_socket(container, stderr=false)
+    # todo factor out query params
+    # todo separate stderr
+    query_params = 'logs=1&stream=1&' + (stderr ? 'stderr=1' : 'stdout=1&stdin=1')
+
+    # Headers are required by Docker
+    headers = {'Origin' => 'http://localhost'}
+
+    socket = Faye::WebSocket::Client.new(DockerClient.config['ws_host'] + '/containers/' + @container.id + '/attach/ws?' + query_params, [], :headers => headers)
+
+    socket.on :error do |event|
+      Rails.logger.info "Websocket error: " + event.message
+    end
+    socket.on :close do |event|
+      Rails.logger.info "Websocket closed."
+    end
+    socket.on :open do |event|
+      Rails.logger.info "Websocket created."
+      kill_after_timeout(container)
+    end
+    socket
+  end
+
   def copy_file_to_workspace(options = {})
     FileUtils.cp(options[:file].native_file.path, local_file_path(options))
   end
@@ -118,14 +147,66 @@ class DockerClient
     #(tries += 1) <= RETRY_COUNT ? retry : raise(error)
   end
 
-  [:run, :test].each do |cause|
+  def execute_websocket_command(command, before_execution_block, output_consuming_block)
-    define_method("execute_#{cause}_command") do |submission, filename, &block|
+    @container = DockerContainerPool.get_container(@execution_environment)
-      command = submission.execution_environment.send(:"#{cause}_command") % command_substitutions(filename)
+    if @container
-      create_workspace_files = proc { create_workspace_files(container, submission) }
+      before_execution_block.try(:call)
-      execute_command(command, create_workspace_files, block)
+      # todo catch exception if socket could not be created
+      @socket ||= create_socket(@container)
+      # Newline required to flush
+      @socket.send command + "\n"
+      {status: :container_running, socket: @socket}
+    else
+      {status: :container_depleted}
     end
   end
 
+  def kill_after_timeout(container)
+    """
+    We need to start a second thread to kill the websocket connection,
+    as it is impossible to determine when no more input is requested.
+    """
+    Thread.new do
+      timeout = @execution_environment.permitted_execution_time.to_i # seconds
+      sleep(timeout)
+      Rails.logger.info("Killing container after timeout of " + timeout.to_s + " seconds.")
+      # if we use pooling and recylce the containers, put it back. otherwise, destroy it.
+      # (DockerContainerPool.config[:active] && RECYCLE_CONTAINERS) ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container)
+
+      # todo won't this always create a new container?
+      # remove container from pool, then destroy it
+      (DockerContainerPool.config[:active]) ? DockerContainerPool.remove_from_all_containers(container, @execution_environment) :
+
+      # destroy container
+      self.class.destroy_container(container)
+
+      # if we recylce containers, we start a fresh one
+      if(DockerContainerPool.config[:active] && RECYCLE_CONTAINERS)
+        # create new container and add it to @all_containers and @containers.
+        container = self.class.create_container(@execution_environment)
+        DockerContainerPool.add_to_all_containers(container, @execution_environment)
+      end
+    end
+  end
+
+  def execute_run_command(submission, filename, &block)
+    """
+    Run commands by attaching a websocket to Docker.
+    """
+    command = submission.execution_environment.send(:"run_command") % command_substitutions(filename)
+    create_workspace_files = proc { create_workspace_files(container, submission) }
+    execute_websocket_command(command, create_workspace_files, block)
+  end
+
+  def execute_test_command(subbmission, filename, &block)
+    """
+    Stick to existing Docker API with exec command.
+    """
+    command = submission.execution_environment.send(:"test_command") % command_substitutions(filename)
+    create_workspace_files = proc { create_workspace_files(container, submission) }
+    execute_command(command, create_workspace_files, block)
+  end
+
   def self.find_image_by_tag(tag)
     Docker::Image.all.detect { |image| image.info['RepoTags'].flatten.include?(tag) }
   end