htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shell: Add file system browser to retrieve arbitrary files

Browse Source
This commit is contained in:
Sebastian Serth
2022-10-04 15:21:06 +02:00
committed by Sebastian Serth
parent 60078701f5
commit 58548555a5
13 changed files with 240 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
app/assets/javascripts/shell.js
View File

@ -2,17 +2,17 @@ $(document).on('turbolinks:load', function () {
const ENTER_KEY_CODE = 13;
const clearOutput = function () {
$('#output').html('');
log.html('');
};
const executeCommand = function (command) {
$.ajax({
data: {
command: command,
sudo: $('#sudo').is(':checked')
sudo: sudo.is(':checked')
},
method: 'POST',
url: $('#shell').data('url')
url: Routes.execute_command_execution_environment_path(id)
}).done(handleResponse);
};
@ -46,7 +46,7 @@ $(document).on('turbolinks:load', function () {
em.text(command);
const p = $('<p>');
p.append(em)
$('#output').append(p);
log.append(p);
};
const printOutput = function (output) {
@ -55,22 +55,21 @@ $(document).on('turbolinks:load', function () {
const element = $('<p>');
element.addClass('text-success');
element.text(output.stdout);
$('#output').append(element);
log.append(element);
}
if (output.stderr) {
const element = $('<p>');
element.addClass('text-warning');
element.text(output.stderr);
$('#output').append(element);
log.append(element);
}
if (!output.stdout && !output.stderr) {
const element = $('<p>');
element.addClass('text-muted');
const output = $('#output');
element.text(output.data('message-no-output'));
output.append(element);
element.text(log.data('message-no-output'));
log.append(element);
}
}
};
@ -79,26 +78,99 @@ $(document).on('turbolinks:load', function () {
const element = $('<p>');
element.addClass('text-danger');
element.text($('#shell').data('message-timeout'));
$('#output').append(element);
log.append(element);
};
const printOutOfMemory = function (output) {
const element = $('<p>');
element.addClass('text-danger');
element.text($('#shell').data('message-out-of-memory'));
$('#output').append(element);
log.append(element);
};
if ($('#shell').isPresent()) {
const command = $('#command')
command.focus();
command.on('keypress', handleKeyPress);
const retrieveFiles = function () {
let fileTree = $('#download-file-tree');
const sudo = $('#sudo');
sudo.on('change', function () {
sudo.parent().toggleClass('text-muted')
command.focus();
});
// Get current instance of the jstree if available and refresh the existing one.
// Otherwise, initialize a new one.
if (fileTree.jstree(true)) {
return fileTree.jstree('refresh');
} else {
fileTree.removeClass('my-3 justify-content-center');
fileTree.jstree({
'core': {
'data': {
'url': function (node) {
const params = {sudo: sudo.is(':checked')};
return Routes.list_files_in_execution_environment_path(id, params);
},
'data': function (node) {
return {'path': getPath(fileTree.jstree(), node)|| '/'};
}
}
}
});
fileTree.on('select_node.jstree', function (node, selected, _event) {
// We never want a node to be selected permanently, so we deselect it immediately.
selected.instance.deselect_all();
const path = getPath(selected.instance, selected.node)
const params = {sudo: sudo.is(':checked')};
const downloadPath = Routes.download_file_from_execution_environment_path(id, path, params);
// Now we download the file if allowed.
if (selected.node.original.icon.split(" ").some(function (icon) {
return ['fa-lock', 'fa-folder'].includes(icon);
})) {
$.flash.danger({
icon: ['fa-solid', 'fa-shield-halved'],
text: I18n.t('execution_environments.shell.file_tree.permission_denied')
});
} else {
window.location = downloadPath;
}
}.bind(this));
}
}
const getPath = function (jstree, node) {
if (node.id === '#') {
// Root node
return '/'
}
// We build the path to the file by concatenating the paths of all parent nodes.
let file_path = node.parents.reverse().map(function (id) {
return jstree.get_text(id);
}).filter(function (text) {
return text !== false;
}).join('/');
return `${node.parent !== '#' ? '/' : ''}${file_path}${node.original.path}`;
}
const shell = $('#shell');
if (!shell.isPresent()) {
return;
}
const command = $('#command')
command.focus();
command.on('keypress', handleKeyPress);
const id = shell.data('id');
const log = $('#output');
const sudo = $('#sudo');
sudo.on('change', function () {
sudo.parent().toggleClass('text-muted')
command.focus();
});
$('#reload-files').on('click', function () {
new bootstrap.Collapse('#collapse_files', 'show');
retrieveFiles();
});
$('#reload-now-link').on('click', retrieveFiles);
})
;

4
app/assets/stylesheets/exercises.css.scss
View File

@ -40,11 +40,11 @@ input[type='file'] {
}
}
[data-bs-toggle="collapse"] .fa-solid:before {
[data-bs-toggle="collapse"] .fa-solid:first-child:before {
content: "\f139";
}
[data-bs-toggle="collapse"].collapsed .fa-solid:before {
[data-bs-toggle="collapse"].collapsed .fa-solid:first-child:before {
content: "\f13a";
}

30
app/controllers/execution_environments_controller.rb
View File

@ -2,9 +2,10 @@
class ExecutionEnvironmentsController < ApplicationController
include CommonBehavior
include FileConversion
before_action :set_docker_images, only: %i[create edit new update]
before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell statistics sync_to_runner_management]
before_action :set_execution_environment, only: MEMBER_ACTIONS + %i[execute_command shell list_files statistics sync_to_runner_management]
before_action :set_testing_framework_adapters, only: %i[create edit new update]
def authorize!
@ -30,11 +31,24 @@ class ExecutionEnvironmentsController < ApplicationController
def execute_command
runner = Runner.for(current_user, @execution_environment)
sudo = ActiveModel::Type::Boolean.new.cast(params[:sudo])
output = runner.execute_command(params[:command], privileged_execution: sudo, raise_exception: false)
@privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
output = runner.execute_command(params[:command], privileged_execution: @privileged_execution, raise_exception: false)
render json: output.except(:messages)
end
def list_files
runner = Runner.for(current_user, @execution_environment)
@privileged_execution = ActiveModel::Type::Boolean.new.cast(params[:sudo]) || @execution_environment.privileged_execution
begin
files = runner.retrieve_files(path: params[:path], recursive: false, privileged_execution: @privileged_execution)
downloadable_files, additional_directories = convert_files_json_to_files files
js_tree = FileTree.new(downloadable_files, additional_directories, force_closed: true).to_js_tree
render json: js_tree[:core][:data]
rescue Runner::Error::WorkspaceError
render json: []
end
end
def working_time_query
"
SELECT exercise_id, avg(working_time) as average_time, stddev_samp(extract('epoch' from working_time)) * interval '1 second' as stddev_time
@ -225,4 +239,14 @@ class ExecutionEnvironmentsController < ApplicationController
redirect_to ExecutionEnvironment, alert: t('execution_environments.index.synchronize_all.failure')
end
end
def augment_files_for_download(files)
files.map do |file|
# Downloadable files get an indicator whether we performed a privileged execution.
# The download path is added dynamically in the frontend.
file.privileged_execution = @privileged_execution
file
end
end
private :augment_files_for_download
end

13
app/controllers/live_streams_controller.rb
View File

@ -21,12 +21,21 @@ class LiveStreamsController < ApplicationController
send_runner_file(runner, desired_file, fallback_location)
end
def download_arbitrary_file
@execution_environment = authorize ExecutionEnvironment.find(params[:id])
desired_file = params[:filename].to_s
runner = Runner.for(current_user, @execution_environment)
fallback_location = shell_execution_environment_path(@execution_environment)
privileged = params[:sudo] || @execution_environment.privileged_execution?
send_runner_file(runner, desired_file, fallback_location, privileged: privileged)
end
private
def send_runner_file(runner, desired_file, redirect_fallback = root_path)
def send_runner_file(runner, desired_file, redirect_fallback = root_path, privileged: false)
filename = File.basename(desired_file)
send_stream(filename: filename, disposition: 'attachment') do |stream|
runner.download_file desired_file do |chunk, overall_size, content_type|
runner.download_file desired_file, privileged_execution: privileged do |chunk, overall_size, content_type|
unless response.committed?
# Disable Rack::ETag, which would otherwise cause the response to be cached
# See https://github.com/rack/rack/issues/1619#issuecomment-848460528

31
app/models/code_ocean/file.rb
View File

@ -11,6 +11,8 @@ module CodeOcean
ROLES = %w[regular_file main_file reference_implementation executable_file teacher_defined_test user_defined_file
user_defined_test teacher_defined_linter].freeze
TEACHER_DEFINED_ROLES = ROLES - %w[user_defined_file]
OWNER_READ_PERMISSION = 0o400
OTHER_READ_PERMISSION = 0o004
after_initialize :set_default_values
before_validation :clear_weight, unless: :teacher_defined_assessment?
@ -19,7 +21,8 @@ module CodeOcean
attr_writer :size
# These attributes are mainly used when retrieving files from a runner
attr_accessor :download_path
attr_accessor :download_path, :owner, :group, :privileged_execution
attr_reader :permissions
belongs_to :context, polymorphic: true
belongs_to :file, class_name: 'CodeOcean::File', optional: true # This is only required for submissions and is validated below
@ -152,5 +155,31 @@ module CodeOcean
content.size
end
end
def permissions=(permission_string)
# We iterate through the permission string (e.g., `rwxrw-r--`) as received through Linux
# For each character in the string, we check for a corresponding permission (which is available if the character is not `-`)
# Then, we use a bit shift to move a `1` to the position of the given permission.
# First, it is moved within a group (e.g., `r` in `rwx` is moved twice to the left, `w` once, `x` not at all)
# Second, the bit is moved in accordance with the group (e.g., the `owner` is moved twice, the `group` once, the `other` group not at all)
# Finally, a sum is created, which technically could be an OR operation as well.
@permissions = permission_string.chars.map.with_index do |permission, index|
next 0 if permission == '-' # No permission
bit = 0b1 << ((2 - index) % 3) # Align bit in respective group
bit << ((2 - (index / 3)) * 3) # Align bit in bytes (for the group)
end.sum
end
def missing_read_permissions?
return false if permissions.blank?
# We use a bitwise AND with the permission bits and compare that to zero
if privileged_execution.present?
(permissions & OWNER_READ_PERMISSION).zero?
else
(permissions & OTHER_READ_PERMISSION).zero?
end
end
end
end

3
app/policies/execution_environment_policy.rb
View File

@ -1,7 +1,8 @@
# frozen_string_literal: true
class ExecutionEnvironmentPolicy < AdminOnlyPolicy
%i[execute_command? shell? statistics? show? sync_to_runner_management?].each do |action|
# download_arbitrary_file? is used in the live_streams_controller.rb
%i[execute_command? shell? list_files? statistics? show? sync_to_runner_management? download_arbitrary_file?].each do |action|
define_method(action) { admin? || author? }
end

20
app/views/execution_environments/shell.html.slim
View File

@ -1,11 +1,29 @@
h1 = @execution_environment
#shell data-message-timeout=t('exercises.editor.timeout', permitted_execution_time: @execution_environment.permitted_execution_time) data-message-out-of-memory=t('exercises.editor.out_of_memory', memory_limit: @execution_environment.memory_limit) data-url=execute_command_execution_environment_path(@execution_environment)
#shell data-message-timeout=t('exercises.editor.timeout', permitted_execution_time: @execution_environment.permitted_execution_time) data-message-out-of-memory=t('exercises.editor.out_of_memory', memory_limit: @execution_environment.memory_limit) data-id=@execution_environment.id
label.form-label for='command' = t('execution_environments.shell.command')
.input-group.mb-3
.input-group-text.form-switch.ps-5.text-muted
input#sudo.form-check-input.mt-0 type='checkbox'
label.ms-2 for='sudo' = 'sudo'
input#command.form-control type='text'
.card.mb-3
.card-header#download-files role="tab"
a.file-heading.collapsed.d-flex.justify-content-between.align-items-center data-bs-toggle="collapse" href="#collapse_files" aria-expanded="false"
div.clearfix role="button"
i.fa-solid aria-hidden="true"
span = t('execution_environments.shell.file_tree.headline')
div
= render('exercises/editor_button', classes: 'btn-default btn-sm', data: {:'data-bs-toggle' => 'tooltip', :'data-url' => list_files_in_execution_environment_path(@execution_environment)}, icon: 'fa-solid fa-arrows-rotate', id: 'reload-files', label: t('execution_environments.shell.file_tree.reload'), title: t('execution_environments.shell.file_tree.reload_tooltip'))
.card-collapse.collapse id="collapse_files" role="tabpanel"
.card-body.pt-0.pe-0.ps-1.pb-1
#download-file-tree.justify-content-center.d-flex.my-3
span.mx-1 = t('execution_environments.shell.file_tree.empty')
button#reload-now-link.btn.btn-link.p-0.m-0.border-0 = t('execution_environments.shell.file_tree.list_now')
.card-footer.justify-content-center.align-items-center.d-flex.text-muted
i.fa-solid.fa-info
span.ms-2 = t('execution_environments.shell.file_tree.root_notice')
pre#output data-message-no-output=t('exercises.implement.no_output', timestamp: l(Time.now, format: :short))
p = t('exercises.implement.no_output_yet')

8
config/locales/de.yml
View File

@ -326,6 +326,14 @@ de:
shell:
command: Befehl
headline: Shell
file_tree:
reload: Aktualisieren
reload_tooltip: Aktualisieren Sie die Liste der verfügbaren Dateien im Runners. Diese Aktion wird einige Sekunden in Anspruch nehmen.
list_now: Jetzt laden.
headline: Dateisystem
empty: Das Dateisystem wurde bisher noch nicht aufgelistet.
root_notice: Dateien werden standardmäßig mit einem nicht-priviligerten Nutzer abgerufen. Um Dateien als "root" abzurufen, müssen Sie den "sudo" Schalter neben der Befehlszeile aktivieren und anschließend das Dateisystem vor dem Herunterladen einer Datei aktualisieren.
permission_denied: Der Zugriff auf die angeforderte Datei wurde verweigert. Bitte überprüfen Sie, dass die Datei existiert, der aktuelle Benutzer Leseberechtigungen besitzt und versuchen Sie ggf. die Datei mit "root"-Rechten anzufordern. Dazu müssen Sie den "sudo"-Schalter neben der Befehlszeile aktivieren und anschließend das Dateisystem vor dem Herunterladen einer Datei aktualisieren.
statistics:
exercise: Übung
users: Anzahl (externer) Nutzer

8
config/locales/en.yml
View File

@ -326,6 +326,14 @@ en:
shell:
command: Command
headline: Shell
file_tree:
reload: Reload
reload_tooltip: Reload the file system of the runner. This action might take a few seconds.
list_now: List now.
headline: File System
empty: The file system has not been queried yet.
root_notice: Files are retrieved with a non-privileged user by default. To retrieve files as "root", you must enable the "sudo" switch shown next to the command input and then reload the file system before accessing any file.
permission_denied: Access to the requested file has been denied. Please verify that the file exists, the current user has read permissions, and try requesting the file with "root" privileges if necessary. To retrieve files as "root", you must enable the "sudo" switch shown next to the command input and then reload the file system before accessing any file.
statistics:
exercise: Exercise
users: (External) Users Count

2
config/routes.rb
View File

@ -66,6 +66,8 @@ Rails.application.routes.draw do
member do
get :shell
post 'shell', as: :execute_command, action: :execute_command
get :list_files, as: :list_files_in
get 'download/:filename', as: :download_file_from, constraints: {filename: FILENAME_REGEXP}, action: :download_arbitrary_file, controller: 'live_streams'
get :statistics
post :sync_to_runner_management
end

43
lib/file_tree.rb
View File

@ -2,6 +2,8 @@
class FileTree
def file_icon(file)
return 'fa-solid fa-lock' if file.missing_read_permissions?
if file.file_type.audio?
'fa-regular fa-file-audio'
elsif file.file_type.compressed?
@ -35,9 +37,13 @@ class FileTree
end
private :folder_icon
def initialize(files = [])
# @param [CodeOcean::File] files The files to be displayed in the tree.
# @param [String] directories Additional directories to be displayed in the tree
# @param [Boolean] force_closed Specify whether the tree should be closed by default
def initialize(files = [], directories = [], force_closed: false)
# Our tree needs a root node, but we won't display it.
@root = Tree::TreeNode.new('ROOT')
@force_closed = force_closed
files.uniq(&:filepath).each do |file|
parent = @root
@ -47,25 +53,34 @@ class FileTree
end
parent.add(Tree::TreeNode.new(file.name_with_extension, file))
end
directories.uniq.each do |directory|
parent = @root
(directory || '').split('/').each do |segment|
node = parent.children.detect {|child| child.name == segment } || parent.add(Tree::TreeNode.new(segment))
parent = node
end
end
end
def map_to_js_tree(node)
{
children: node.children.map {|child| map_to_js_tree(child) },
children: children(node),
icon: node_icon(node),
id: node.content.try(:ancestor_id),
state: {
disabled: !node.leaf?,
opened: !node.leaf?,
disabled: !(node.leaf? && node.content.is_a?(CodeOcean::File)),
opened: !(node.leaf? || @force_closed),
},
text: name(node),
download_path: node.content.try(:download_path),
path: node.content.try(:download_path) ? nil : path(node),
}
end
private :map_to_js_tree
def node_icon(node)
if node.leaf? && !node.root?
if node.leaf? && !node.root? && node.content.is_a?(CodeOcean::File)
file_icon(node.content)
else
folder_icon
@ -73,6 +88,17 @@ class FileTree
end
private :node_icon
def children(node)
if node.children.present? || node.content.is_a?(CodeOcean::File)
node.children.sort_by {|n| n.name.downcase }.map {|child| map_to_js_tree(child) }
else
# Folders added manually should always be expandable and therefore might have children.
# This allows users to open the folder and get a refreshed view, even if it might be empty.
true
end
end
private :children
def name(node)
# We just need any information that is only present in files retrieved from the runner's file system.
# In our case, that is the presence of the `privileged_execution` attribute.
@ -84,10 +110,15 @@ class FileTree
end
private :name
def path(node)
"#{node.parentage&.reverse&.drop(1)&.map(&:name)&.join('/')}/#{node.name}"
end
private :path
def to_js_tree
{
core: {
data: @root.children.map {|child| map_to_js_tree(child) },
data: @root.children.sort_by {|node| node.name.downcase }.map {|child| map_to_js_tree(child) },
},
}
end

2
spec/lib/file_tree_spec.rb
View File

@ -161,7 +161,7 @@ describe FileTree do
end
context 'with leaf nodes' do
let(:node) { root.add(Tree::TreeNode.new('')) }
let(:node) { root.add(Tree::TreeNode.new('', CodeOcean::File.new)) }
it 'is a file icon' do
expect(file_tree).to receive(:file_icon)

2
spec/views/execution_environments/shell.html.slim_spec.rb
View File

@ -12,6 +12,6 @@ describe 'execution_environments/shell.html.slim' do
it 'contains the required data attributes' do
expect(rendered).to have_css('#shell[data-message-timeout]')
expect(rendered).to have_css("#shell[data-url='#{execute_command_execution_environment_path(execution_environment)}']")
expect(rendered).to have_css("#shell[data-id='#{execution_environment.id}']")
end
end