Shell: Add file system browser to retrieve arbitrary files

app/assets/javascripts/shell.js

@@ -2,17 +2,17 @@ $(document).on('turbolinks:load', function () {
     const ENTER_KEY_CODE = 13;
 
     const clearOutput = function () {
-        $('#output').html('');
+        log.html('');
     };
 
     const executeCommand = function (command) {
         $.ajax({
             data: {
                 command: command,
-                sudo: $('#sudo').is(':checked')
+                sudo: sudo.is(':checked')
             },
             method: 'POST',
-            url: $('#shell').data('url')
+            url: Routes.execute_command_execution_environment_path(id)
         }).done(handleResponse);
     };
 
@@ -46,7 +46,7 @@ $(document).on('turbolinks:load', function () {
         em.text(command);
         const p = $('<p>');
         p.append(em)
-        $('#output').append(p);
+        log.append(p);
     };
 
     const printOutput = function (output) {
@@ -55,22 +55,21 @@ $(document).on('turbolinks:load', function () {
                 const element = $('<p>');
                 element.addClass('text-success');
                 element.text(output.stdout);
-                $('#output').append(element);
+                log.append(element);
             }
 
             if (output.stderr) {
                 const element = $('<p>');
                 element.addClass('text-warning');
                 element.text(output.stderr);
-                $('#output').append(element);
+                log.append(element);
             }
 
             if (!output.stdout && !output.stderr) {
                 const element = $('<p>');
                 element.addClass('text-muted');
-                const output = $('#output');
+                element.text(log.data('message-no-output'));
-                element.text(output.data('message-no-output'));
+                log.append(element);
-                output.append(element);
             }
         }
     };
@@ -79,26 +78,99 @@ $(document).on('turbolinks:load', function () {
         const element = $('<p>');
         element.addClass('text-danger');
         element.text($('#shell').data('message-timeout'));
-        $('#output').append(element);
+        log.append(element);
     };
 
     const printOutOfMemory = function (output) {
         const element = $('<p>');
         element.addClass('text-danger');
         element.text($('#shell').data('message-out-of-memory'));
-        $('#output').append(element);
+        log.append(element);
     };
 
-    if ($('#shell').isPresent()) {
+    const retrieveFiles = function () {
-        const command = $('#command')
+        let fileTree = $('#download-file-tree');
-        command.focus();
-        command.on('keypress', handleKeyPress);
 
-        const sudo = $('#sudo');
+        // Get current instance of the jstree if available and refresh the existing one.
-        sudo.on('change', function () {
+        // Otherwise, initialize a new one.
-            sudo.parent().toggleClass('text-muted')
+        if (fileTree.jstree(true)) {
-            command.focus();
+            return fileTree.jstree('refresh');
-        });
+        } else {
+            fileTree.removeClass('my-3 justify-content-center');
+            fileTree.jstree({
+                'core': {
+                    'data': {
+                        'url': function (node) {
+                            const params = {sudo: sudo.is(':checked')};
+                            return Routes.list_files_in_execution_environment_path(id, params);
+                        },
+                        'data': function (node) {
+                            return {'path': getPath(fileTree.jstree(), node)|| '/'};
+                        }
+                    }
+                }
+            });
+            fileTree.on('select_node.jstree', function (node, selected, _event) {
+                // We never want a node to be selected permanently, so we deselect it immediately.
+                selected.instance.deselect_all();
+
+                const path = getPath(selected.instance, selected.node)
+                const params = {sudo: sudo.is(':checked')};
+                const downloadPath = Routes.download_file_from_execution_environment_path(id, path, params);
+
+                // Now we download the file if allowed.
+                if (selected.node.original.icon.split(" ").some(function (icon) {
+                    return ['fa-lock', 'fa-folder'].includes(icon);
+                })) {
+                    $.flash.danger({
+                        icon: ['fa-solid', 'fa-shield-halved'],
+                        text: I18n.t('execution_environments.shell.file_tree.permission_denied')
+                    });
+                } else {
+                    window.location = downloadPath;
+                }
+            }.bind(this));
+        }
     }
+
+    const getPath = function (jstree, node) {
+        if (node.id === '#') {
+            // Root node
+            return '/'
+        }
+
+        // We build the path to the file by concatenating the paths of all parent nodes.
+        let file_path = node.parents.reverse().map(function (id) {
+            return jstree.get_text(id);
+        }).filter(function (text) {
+            return text !== false;
+        }).join('/');
+
+        return `${node.parent !== '#' ? '/' : ''}${file_path}${node.original.path}`;
+    }
+
+    const shell = $('#shell');
+
+    if (!shell.isPresent()) {
+        return;
+    }
+
+    const command = $('#command')
+    command.focus();
+    command.on('keypress', handleKeyPress);
+
+    const id = shell.data('id');
+    const log = $('#output');
+
+    const sudo = $('#sudo');
+    sudo.on('change', function () {
+        sudo.parent().toggleClass('text-muted')
+        command.focus();
+    });
+    $('#reload-files').on('click', function () {
+        new bootstrap.Collapse('#collapse_files', 'show');
+        retrieveFiles();
+    });
+    $('#reload-now-link').on('click', retrieveFiles);
 })
 ;

app/assets/stylesheets/exercises.css.scss

@@ -40,11 +40,11 @@ input[type='file'] {
   }
 }
 
-[data-bs-toggle="collapse"] .fa-solid:before {
+[data-bs-toggle="collapse"] .fa-solid:first-child:before {
   content: "\f139";
 }
 
-[data-bs-toggle="collapse"].collapsed .fa-solid:before {
+[data-bs-toggle="collapse"].collapsed .fa-solid:first-child:before {
   content: "\f13a";
 }
 

app/controllers/execution_environments_controller.rb

@@ -2,9 +2,10 @@
 
 class ExecutionEnvironmentsController < ApplicationController
   include CommonBehavior
+  include FileConversion
 
   before_action :set_docker_images, only: %i[create edit new update]
-  before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell statistics sync_to_runner_management]
+  before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell list_files statistics sync_to_runner_management]
   before_action :set_testing_framework_adapters, only: %i[create edit new update]
 
   def authorize!
@@ -30,11 +31,24 @@ class ExecutionEnvironmentsController < ApplicationController
 
   def execute_command
     runner = Runner.for(current_user, @execution_environment)
-    sudo = ActiveModel::Type::Boolean.new.cast(params[:sudo])
+    @privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
-    output = runner.execute_command(params[:command], privileged_execution: sudo, raise_exception: false)
+    output = runner.execute_command(params[:command], privileged_execution: @privileged_execution, raise_exception: false)
     render json: output.except(:messages)
   end
 
+  def list_files
+    runner = Runner.for(current_user, @execution_environment)
+    @privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
+    begin
+      files = runner.retrieve_files(path: params[:path], recursive: false, privileged_execution: @privileged_execution)
+      downloadable_files, additional_directories = convert_files_json_to_files files
+      js_tree = FileTree.new(downloadable_files, additional_directories, force_closed: true).to_js_tree
+      render json: js_tree[:core][:data]
+    rescue Runner::Error::WorkspaceError
+      render json: []
+    end
+  end
+
   def working_time_query
     "
       SELECT exercise_id, avg(working_time) as average_time, stddev_samp(extract('epoch' from working_time)) * interval '1 second' as stddev_time
@@ -225,4 +239,14 @@ class ExecutionEnvironmentsController < ApplicationController
       redirect_to ExecutionEnvironment, alert: t('execution_environments.index.synchronize_all.failure')
     end
   end
+
+  def augment_files_for_download(files)
+    files.map do |file|
+      # Downloadable files get an indicator whether we performed a privileged execution.
+      # The download path is added dynamically in the frontend.
+      file.privileged_execution = @privileged_execution
+      file
+    end
+  end
+  private :augment_files_for_download
 end

app/controllers/live_streams_controller.rb

@@ -21,12 +21,21 @@ class LiveStreamsController < ApplicationController
     send_runner_file(runner, desired_file, fallback_location)
   end
 
+  def download_arbitrary_file
+    @execution_environment = authorize ExecutionEnvironment.find(params[:id])
+    desired_file = params[:filename].to_s
+    runner = Runner.for(current_user, @execution_environment)
+    fallback_location = shell_execution_environment_path(@execution_environment)
+    privileged = params[:sudo] || @execution_environment.privileged_execution?
+    send_runner_file(runner, desired_file, fallback_location, privileged: privileged)
+  end
+
   private
 
-  def send_runner_file(runner, desired_file, redirect_fallback = root_path)
+  def send_runner_file(runner, desired_file, redirect_fallback = root_path, privileged: false)
     filename = File.basename(desired_file)
     send_stream(filename: filename, disposition: 'attachment') do |stream|
-      runner.download_file desired_file do |chunk, overall_size, content_type|
+      runner.download_file desired_file, privileged_execution: privileged do |chunk, overall_size, content_type|
         unless response.committed?
           # Disable Rack::ETag, which would otherwise cause the response to be cached
           # See https://github.com/rack/rack/issues/1619#issuecomment-848460528

app/models/code_ocean/file.rb

@@ -11,6 +11,8 @@ module CodeOcean
     ROLES = %w[regular_file main_file reference_implementation executable_file teacher_defined_test user_defined_file
                user_defined_test teacher_defined_linter].freeze
     TEACHER_DEFINED_ROLES = ROLES - %w[user_defined_file]
+    OWNER_READ_PERMISSION = 0o400
+    OTHER_READ_PERMISSION = 0o004
 
     after_initialize :set_default_values
     before_validation :clear_weight, unless: :teacher_defined_assessment?
@@ -19,7 +21,8 @@ module CodeOcean
 
     attr_writer :size
     # These attributes are mainly used when retrieving files from a runner
-    attr_accessor :download_path
+    attr_accessor :download_path, :owner, :group, :privileged_execution
+    attr_reader :permissions
 
     belongs_to :context, polymorphic: true
     belongs_to :file, class_name: 'CodeOcean::File', optional: true # This is only required for submissions and is validated below
@@ -152,5 +155,31 @@ module CodeOcean
                   content.size
                 end
     end
+
+    def permissions=(permission_string)
+      # We iterate through the permission string (e.g., `rwxrw-r--`) as received through Linux
+      # For each character in the string, we check for a corresponding permission (which is available if the character is not `-`)
+      # Then, we use a bit shift to move a `1` to the position of the given permission.
+      # First, it is moved within a group (e.g., `r` in `rwx` is moved twice to the left, `w` once, `x` not at all)
+      # Second, the bit is moved in accordance with the group (e.g., the `owner` is moved twice, the `group` once, the `other` group not at all)
+      # Finally, a sum is created, which technically could be an OR operation as well.
+      @permissions = permission_string.chars.map.with_index do |permission, index|
+        next 0 if permission == '-' # No permission
+
+        bit = 0b1 << ((2 - index) % 3) # Align bit in respective group
+        bit << ((2 - (index / 3)) * 3) # Align bit in bytes (for the group)
+      end.sum
+    end
+
+    def missing_read_permissions?
+      return false if permissions.blank?
+
+      # We use a bitwise AND with the permission bits and compare that to zero
+      if privileged_execution.present?
+        (permissions & OWNER_READ_PERMISSION).zero?
+      else
+        (permissions & OTHER_READ_PERMISSION).zero?
+      end
+    end
   end
 end

app/policies/execution_environment_policy.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 class ExecutionEnvironmentPolicy < AdminOnlyPolicy
-  %i[execute_command? shell? statistics? show? sync_to_runner_management?].each do |action|
+  # download_arbitrary_file? is used in the live_streams_controller.rb
+  %i[execute_command? shell? list_files? statistics? show? sync_to_runner_management? download_arbitrary_file?].each do |action|
     define_method(action) { admin? || author? }
   end
 

app/views/execution_environments/shell.html.slim

@@ -1,11 +1,29 @@
 h1 = @execution_environment
 
-#shell data-message-timeout=t('exercises.editor.timeout', permitted_execution_time: @execution_environment.permitted_execution_time) data-message-out-of-memory=t('exercises.editor.out_of_memory',  memory_limit: @execution_environment.memory_limit) data-url=execute_command_execution_environment_path(@execution_environment)
+#shell data-message-timeout=t('exercises.editor.timeout', permitted_execution_time: @execution_environment.permitted_execution_time) data-message-out-of-memory=t('exercises.editor.out_of_memory',  memory_limit: @execution_environment.memory_limit) data-id=@execution_environment.id
   label.form-label for='command' = t('execution_environments.shell.command')
   .input-group.mb-3
     .input-group-text.form-switch.ps-5.text-muted
       input#sudo.form-check-input.mt-0 type='checkbox'
       label.ms-2 for='sudo' = 'sudo'
     input#command.form-control type='text'
+
+  .card.mb-3
+    .card-header#download-files role="tab"
+      a.file-heading.collapsed.d-flex.justify-content-between.align-items-center data-bs-toggle="collapse" href="#collapse_files" aria-expanded="false"
+        div.clearfix role="button"
+          i.fa-solid aria-hidden="true"
+          span = t('execution_environments.shell.file_tree.headline')
+        div
+          = render('exercises/editor_button', classes: 'btn-default btn-sm', data: {:'data-bs-toggle' => 'tooltip', :'data-url' => list_files_in_execution_environment_path(@execution_environment)}, icon: 'fa-solid fa-arrows-rotate', id: 'reload-files', label: t('execution_environments.shell.file_tree.reload'), title: t('execution_environments.shell.file_tree.reload_tooltip'))
+    .card-collapse.collapse id="collapse_files" role="tabpanel"
+      .card-body.pt-0.pe-0.ps-1.pb-1
+        #download-file-tree.justify-content-center.d-flex.my-3
+          span.mx-1 = t('execution_environments.shell.file_tree.empty')
+          button#reload-now-link.btn.btn-link.p-0.m-0.border-0 = t('execution_environments.shell.file_tree.list_now')
+      .card-footer.justify-content-center.align-items-center.d-flex.text-muted
+        i.fa-solid.fa-info
+        span.ms-2 = t('execution_environments.shell.file_tree.root_notice')
+
   pre#output data-message-no-output=t('exercises.implement.no_output', timestamp: l(Time.now, format: :short))
     p = t('exercises.implement.no_output_yet')

config/locales/de.yml

@@ -326,6 +326,14 @@ de:
     shell:
       command: Befehl
       headline: Shell
+      file_tree:
+        reload: Aktualisieren
+        reload_tooltip: Aktualisieren Sie die Liste der verfügbaren Dateien im Runners. Diese Aktion wird einige Sekunden in Anspruch nehmen.
+        list_now: Jetzt laden.
+        headline: Dateisystem
+        empty: Das Dateisystem wurde bisher noch nicht aufgelistet.
+        root_notice: Dateien werden standardmäßig mit einem nicht-priviligerten Nutzer abgerufen. Um Dateien als "root" abzurufen, müssen Sie den "sudo" Schalter neben der Befehlszeile aktivieren und anschließend das Dateisystem vor dem Herunterladen einer Datei aktualisieren.
+        permission_denied: Der Zugriff auf die angeforderte Datei wurde verweigert. Bitte überprüfen Sie, dass die Datei existiert, der aktuelle Benutzer Leseberechtigungen besitzt und versuchen Sie ggf. die Datei mit "root"-Rechten anzufordern. Dazu müssen Sie den "sudo"-Schalter neben der Befehlszeile aktivieren und anschließend das Dateisystem vor dem Herunterladen einer Datei aktualisieren.
     statistics:
       exercise: Übung
       users: Anzahl (externer) Nutzer

config/locales/en.yml

@@ -326,6 +326,14 @@ en:
     shell:
       command: Command
       headline: Shell
+      file_tree:
+        reload: Reload
+        reload_tooltip: Reload the file system of the runner. This action might take a few seconds.
+        list_now: List now.
+        headline: File System
+        empty: The file system has not been queried yet.
+        root_notice: Files are retrieved with a non-privileged user by default. To retrieve files as "root", you must enable the "sudo" switch shown next to the command input and then reload the file system before accessing any file.
+        permission_denied: Access to the requested file has been denied. Please verify that the file exists, the current user has read permissions, and try requesting the file with "root" privileges if necessary.  To retrieve files as "root", you must enable the "sudo" switch shown next to the command input and then reload the file system before accessing any file.
     statistics:
       exercise: Exercise
       users: (External) Users Count

config/routes.rb

@@ -66,6 +66,8 @@ Rails.application.routes.draw do
     member do
       get :shell
       post 'shell', as: :execute_command, action: :execute_command
+      get :list_files, as: :list_files_in
+      get 'download/:filename', as: :download_file_from, constraints: {filename: FILENAME_REGEXP}, action: :download_arbitrary_file, controller: 'live_streams'
       get :statistics
       post :sync_to_runner_management
     end

lib/file_tree.rb

@@ -2,6 +2,8 @@
 
 class FileTree
   def file_icon(file)
+    return 'fa-solid fa-lock' if file.missing_read_permissions?
+
     if file.file_type.audio?
       'fa-regular fa-file-audio'
     elsif file.file_type.compressed?
@@ -35,9 +37,13 @@ class FileTree
   end
   private :folder_icon
 
-  def initialize(files = [])
+  # @param [CodeOcean::File] files The files to be displayed in the tree.
+  # @param [String] directories Additional directories to be displayed in the tree
+  # @param [Boolean] force_closed Specify whether the tree should be closed by default
+  def initialize(files = [], directories = [], force_closed: false)
     # Our tree needs a root node, but we won't display it.
     @root = Tree::TreeNode.new('ROOT')
+    @force_closed = force_closed
 
     files.uniq(&:filepath).each do |file|
       parent = @root
@@ -47,25 +53,34 @@ class FileTree
       end
       parent.add(Tree::TreeNode.new(file.name_with_extension, file))
     end
+
+    directories.uniq.each do |directory|
+      parent = @root
+      (directory || '').split('/').each do |segment|
+        node = parent.children.detect {|child| child.name == segment } || parent.add(Tree::TreeNode.new(segment))
+        parent = node
+      end
+    end
   end
 
   def map_to_js_tree(node)
     {
-      children: node.children.map {|child| map_to_js_tree(child) },
+      children: children(node),
       icon: node_icon(node),
       id: node.content.try(:ancestor_id),
       state: {
-        disabled: !node.leaf?,
+        disabled: !(node.leaf? && node.content.is_a?(CodeOcean::File)),
-        opened: !node.leaf?,
+        opened: !(node.leaf? || @force_closed),
       },
       text: name(node),
       download_path: node.content.try(:download_path),
+      path: node.content.try(:download_path) ? nil : path(node),
     }
   end
   private :map_to_js_tree
 
   def node_icon(node)
-    if node.leaf? && !node.root?
+    if node.leaf? && !node.root? && node.content.is_a?(CodeOcean::File)
       file_icon(node.content)
     else
       folder_icon
@@ -73,6 +88,17 @@ class FileTree
   end
   private :node_icon
 
+  def children(node)
+    if node.children.present? || node.content.is_a?(CodeOcean::File)
+      node.children.sort_by {|n| n.name.downcase }.map {|child| map_to_js_tree(child) }
+    else
+      # Folders added manually should always be expandable and therefore might have children.
+      # This allows users to open the folder and get a refreshed view, even if it might be empty.
+      true
+    end
+  end
+  private :children
+
   def name(node)
     # We just need any information that is only present in files retrieved from the runner's file system.
     # In our case, that is the presence of the `privileged_execution` attribute.
@@ -84,10 +110,15 @@ class FileTree
   end
   private :name
 
+  def path(node)
+    "#{node.parentage&.reverse&.drop(1)&.map(&:name)&.join('/')}/#{node.name}"
+  end
+  private :path
+
   def to_js_tree
     {
       core: {
-        data: @root.children.map {|child| map_to_js_tree(child) },
+        data: @root.children.sort_by {|node| node.name.downcase }.map {|child| map_to_js_tree(child) },
       },
     }
   end

spec/lib/file_tree_spec.rb

@@ -161,7 +161,7 @@ describe FileTree do
     end
 
     context 'with leaf nodes' do
-      let(:node) { root.add(Tree::TreeNode.new('')) }
+      let(:node) { root.add(Tree::TreeNode.new('', CodeOcean::File.new)) }
 
       it 'is a file icon' do
         expect(file_tree).to receive(:file_icon)

spec/views/execution_environments/shell.html.slim_spec.rb

@@ -12,6 +12,6 @@ describe 'execution_environments/shell.html.slim' do
 
   it 'contains the required data attributes' do
     expect(rendered).to have_css('#shell[data-message-timeout]')
-    expect(rendered).to have_css("#shell[data-url='#{execute_command_execution_environment_path(execution_environment)}']")
+    expect(rendered).to have_css("#shell[data-id='#{execution_environment.id}']")
   end
 end