htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rescue RuntimeError (FayeWebsocket) and apply style

Browse Source
This commit is contained in:
Sebastian Serth
2020-05-05 22:46:28 +02:00
parent 278d48ca6c
commit 99979eeb4f
1 changed files with 126 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
lib/docker_client.rb
View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
require 'concurrent' require 'concurrent'
require 'pathname' require 'pathname'
@ -31,9 +33,9 @@ class DockerClient
if local_workspace_path && Pathname.new(local_workspace_path).exist? if local_workspace_path && Pathname.new(local_workspace_path).exist?
Pathname.new(local_workspace_path).children.each do |p| Pathname.new(local_workspace_path).children.each do |p|
p.rmtree p.rmtree
rescue Errno::ENOENT, Errno::EACCES => error rescue Errno::ENOENT, Errno::EACCES => e
Raven.capture_exception(error) Raven.capture_exception(e)
Rails.logger.error("clean_container_workspace: Got #{error.class.to_s}: #{error.to_s}") Rails.logger.error("clean_container_workspace: Got #{e.class}: #{e}")
end end
# FileUtils.rmdir(Pathname.new(local_workspace_path)) # FileUtils.rmdir(Pathname.new(local_workspace_path))
end end
@ -78,7 +80,7 @@ class DockerClient
end end
def create_socket(container, stderr = false) def create_socket(container, stderr = false)
# todo factor out query params # TODO: factor out query params
# todo separate stderr # todo separate stderr
query_params = 'logs=0&stream=1&' + (stderr ? 'stderr=1' : 'stdout=1&stdin=1') query_params = 'logs=0&stream=1&' + (stderr ? 'stderr=1' : 'stdout=1&stdin=1')
@ -86,18 +88,18 @@ class DockerClient
headers = {'Origin' => 'http://localhost'} headers = {'Origin' => 'http://localhost'}
socket_url = DockerClient.config['ws_host'] + '/v1.27/containers/' + @container.id + '/attach/ws?' + query_params socket_url = DockerClient.config['ws_host'] + '/v1.27/containers/' + @container.id + '/attach/ws?' + query_params
socket = Faye::WebSocket::Client.new(socket_url, [], :headers => headers) socket = Faye::WebSocket::Client.new(socket_url, [], headers: headers)
Rails.logger.debug "Opening Websocket on URL " + socket_url Rails.logger.debug 'Opening Websocket on URL ' + socket_url
socket.on :error do |event| socket.on :error do |event|
Rails.logger.info "Websocket error: " + event.message.to_s Rails.logger.info 'Websocket error: ' + event.message.to_s
end end
socket.on :close do |event| socket.on :close do |_event|
Rails.logger.info "Websocket closed." Rails.logger.info 'Websocket closed.'
end end
socket.on :open do |event| socket.on :open do |_event|
Rails.logger.info "Websocket created." Rails.logger.info 'Websocket created.'
kill_after_timeout(container) kill_after_timeout(container)
end end
socket socket
@ -145,8 +147,8 @@ class DockerClient
end end
container container
rescue Docker::Error::NotFoundError => error rescue Docker::Error::NotFoundError => e
Rails.logger.error('create_container: Got Docker::Error::NotFoundError: ' + error.to_s) Rails.logger.error('create_container: Got Docker::Error::NotFoundError: ' + e.to_s)
destroy_container(container) destroy_container(container)
# (tries += 1) <= RETRY_COUNT ? retry : raise(error) # (tries += 1) <= RETRY_COUNT ? retry : raise(error)
end end
@ -163,8 +165,8 @@ class DockerClient
end end
end end
FileUtils.chmod_R('+rwX', self.class.local_workspace_path(container)) FileUtils.chmod_R('+rwX', self.class.local_workspace_path(container))
rescue Docker::Error::NotFoundError => error rescue Docker::Error::NotFoundError => e
Rails.logger.info('create_workspace_files: Rescued from Docker::Error::NotFoundError: ' + error.to_s) Rails.logger.info('create_workspace_files: Rescued from Docker::Error::NotFoundError: ' + e.to_s)
end end
private :create_workspace_files private :create_workspace_files
@ -179,23 +181,21 @@ class DockerClient
private :create_workspace_file private :create_workspace_file
def create_workspace_files_transmit(container, submission) def create_workspace_files_transmit(container, submission)
begin
# create a temporary dir, put all files in it, and put it into the container. the dir is automatically removed when leaving the block. # create a temporary dir, put all files in it, and put it into the container. the dir is automatically removed when leaving the block.
Dir.mktmpdir { |dir| Dir.mktmpdir do |dir|
submission.collect_files.each do |file| submission.collect_files.each do |file|
disk_file = File.new(dir + '/' + (file.path || '') + file.name_with_extension, 'w') disk_file = File.new(dir + '/' + (file.path || '') + file.name_with_extension, 'w')
disk_file.write(file.content) disk_file.write(file.content)
disk_file.close disk_file.close
end end
begin begin
# create target folder, TODO re-active this when we remove shared folder bindings # create target folder, TODO re-active this when we remove shared folder bindings
# container.exec(['bash', '-c', 'mkdir ' + CONTAINER_WORKSPACE_PATH]) # container.exec(['bash', '-c', 'mkdir ' + CONTAINER_WORKSPACE_PATH])
# container.exec(['bash', '-c', 'chown -R python ' + CONTAINER_WORKSPACE_PATH]) # container.exec(['bash', '-c', 'chown -R python ' + CONTAINER_WORKSPACE_PATH])
# container.exec(['bash', '-c', 'chgrp -G python ' + CONTAINER_WORKSPACE_PATH]) # container.exec(['bash', '-c', 'chgrp -G python ' + CONTAINER_WORKSPACE_PATH])
rescue StandardError => error rescue StandardError => e
Rails.logger.error('create workspace folder: Rescued from StandardError: ' + error.to_s) Rails.logger.error('create workspace folder: Rescued from StandardError: ' + e.to_s)
end end
# sleep 1000 # sleep 1000
@ -203,9 +203,8 @@ class DockerClient
begin begin
# tar the files in dir and put the tar to CONTAINER_WORKSPACE_PATH in the container # tar the files in dir and put the tar to CONTAINER_WORKSPACE_PATH in the container
container.archive_in(dir, CONTAINER_WORKSPACE_PATH, overwrite: false) container.archive_in(dir, CONTAINER_WORKSPACE_PATH, overwrite: false)
rescue StandardError => e
rescue StandardError => error Rails.logger.error('insert tar: Rescued from StandardError: ' + e.to_s)
Rails.logger.error('insert tar: Rescued from StandardError: ' + error.to_s)
end end
# Rails.logger.info('command: tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH) # Rails.logger.info('command: tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH)
@ -213,23 +212,18 @@ class DockerClient
begin begin
# untar the tar file placed in the CONTAINER_WORKSPACE_PATH # untar the tar file placed in the CONTAINER_WORKSPACE_PATH
container.exec(['bash', '-c', 'tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH]) container.exec(['bash', '-c', 'tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH])
rescue StandardError => error rescue StandardError => e
Rails.logger.error('untar: Rescued from StandardError: ' + error.to_s) Rails.logger.error('untar: Rescued from StandardError: ' + e.to_s)
end end
# sleep 1000 # sleep 1000
}
rescue StandardError => error
Rails.logger.error('create_workspace_files_transmit: Rescued from StandardError: ' + error.to_s)
end end
rescue StandardError => e
Rails.logger.error('create_workspace_files_transmit: Rescued from StandardError: ' + e.to_s)
end end
def self.destroy_container(container) def self.destroy_container(container)
if @socket @socket&.close
@socket.close
end
Rails.logger.info('destroying container ' + container.to_s) Rails.logger.info('destroying container ' + container.to_s)
# Checks only if container assignment is not nil and not whether the container itself is still present. # Checks only if container assignment is not nil and not whether the container itself is still present.
@ -241,11 +235,11 @@ class DockerClient
elsif container elsif container
DockerContainerPool.destroy_container(container) DockerContainerPool.destroy_container(container)
end end
rescue Docker::Error::NotFoundError => error rescue Docker::Error::NotFoundError => e
Rails.logger.error('destroy_container: Rescued from Docker::Error::NotFoundError: ' + error.to_s) Rails.logger.error('destroy_container: Rescued from Docker::Error::NotFoundError: ' + e.to_s)
Rails.logger.error('No further actions are done concerning that.') Rails.logger.error('No further actions are done concerning that.')
rescue Docker::Error::ConflictError => error rescue Docker::Error::ConflictError => e
Rails.logger.error('destroy_container: Rescued from Docker::Error::ConflictError: ' + error.to_s) Rails.logger.error('destroy_container: Rescued from Docker::Error::ConflictError: ' + e.to_s)
Rails.logger.error('No further actions are done concerning that.') Rails.logger.error('No further actions are done concerning that.')
end end
@ -274,14 +268,14 @@ class DockerClient
else else
{status: :container_depleted, waiting_for_container_time: waiting_for_container_time, container_execution_time: nil} {status: :container_depleted, waiting_for_container_time: waiting_for_container_time, container_execution_time: nil}
end end
rescue Excon::Errors::SocketError => error rescue Excon::Errors::SocketError => e
# socket errors seems to be normal when using exec # socket errors seems to be normal when using exec
# so lets ignore them for now # so lets ignore them for now
# (tries += 1) <= RETRY_COUNT ? retry : raise(error) # (tries += 1) <= RETRY_COUNT ? retry : raise(error)
end end
# called when the user clicks the "Run" button # called when the user clicks the "Run" button
def open_websocket_connection(command, before_execution_block, output_consuming_block) def open_websocket_connection(command, before_execution_block, _output_consuming_block)
@container = DockerContainerPool.get_container(@execution_environment) @container = DockerContainerPool.get_container(@execution_environment)
if @container if @container
@container.status = :executing @container.status = :executing
@ -289,8 +283,8 @@ class DockerClient
# before_execution_block.try(:call) # before_execution_block.try(:call)
begin begin
before_execution_block.call before_execution_block.call
rescue StandardError => error rescue StandardError => e
Rails.logger.error('execute_websocket_command: Rescued from StandardError caused by before_execution_block.call: ' + error.to_s) Rails.logger.error('execute_websocket_command: Rescued from StandardError caused by before_execution_block.call: ' + e.to_s)
end end
# TODO: catch exception if socket could not be created # TODO: catch exception if socket could not be created
@socket ||= create_socket(@container) @socket ||= create_socket(@container)
@ -312,14 +306,16 @@ class DockerClient
Rails.logger.info('Killing container after timeout of ' + timeout.to_s + ' seconds.') Rails.logger.info('Killing container after timeout of ' + timeout.to_s + ' seconds.')
# send timeout to the tubesock socket # send timeout to the tubesock socket
# FIXME: 2nd thread to notify user. # FIXME: 2nd thread to notify user.
if @tubesock @tubesock&.send_data JSON.dump({'cmd' => 'timeout'})
@tubesock.send_data JSON.dump({'cmd' => 'timeout'})
end
if @socket if @socket
begin
@socket.send('#timeout') @socket.send('#timeout')
# sleep one more second to ensure that the message reaches the submissions_controller. # sleep one more second to ensure that the message reaches the submissions_controller.
sleep(1) sleep(1)
@socket.close @socket.close
rescue RuntimeError => e
Rails.logger.error(e)
end
end end
Thread.new do Thread.new do
kill_container(container) kill_container(container)
@ -334,9 +330,7 @@ class DockerClient
end end
def exit_thread_if_alive def exit_thread_if_alive
if (@thread && @thread.alive?) @thread.exit if @thread&.alive?
@thread.exit
end
end end
def exit_container(container) def exit_container(container)
@ -345,10 +339,10 @@ class DockerClient
exit_thread_if_alive exit_thread_if_alive
@socket.close @socket.close
# if we use pooling and recylce the containers, put it back. otherwise, destroy it. # if we use pooling and recylce the containers, put it back. otherwise, destroy it.
(DockerContainerPool.config[:active] && RECYCLE_CONTAINERS) ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container) DockerContainerPool.config[:active] && RECYCLE_CONTAINERS ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container)
end end
def kill_container(container, create_new = true) def kill_container(container, _create_new = true)
exit_thread_if_alive exit_thread_if_alive
Rails.logger.info('killing container ' + container.to_s) Rails.logger.info('killing container ' + container.to_s)
self.class.destroy_container(container) self.class.destroy_container(container)
@ -376,16 +370,14 @@ class DockerClient
end end
def self.find_image_by_tag(tag) def self.find_image_by_tag(tag)
# todo: cache this. # TODO: cache this.
Docker::Image.all.detect do |image| Docker::Image.all.detect do |image|
begin
image.info['RepoTags'].flatten.include?(tag) image.info['RepoTags'].flatten.include?(tag)
rescue rescue StandardError
# Skip image if it is not tagged # Skip image if it is not tagged
next next
end end
end end
end
def self.generate_local_workspace_path def self.generate_local_workspace_path
File.join(LOCAL_WORKSPACE_ROOT, SecureRandom.uuid) File.join(LOCAL_WORKSPACE_ROOT, SecureRandom.uuid)
@ -397,7 +389,7 @@ class DockerClient
def initialize(options = {}) def initialize(options = {})
@execution_environment = options[:execution_environment] @execution_environment = options[:execution_environment]
# todo: eventually re-enable this if it is cached. But in the end, we do not need this. # TODO: eventually re-enable this if it is cached. But in the end, we do not need this.
# docker daemon got much too much load. all not 100% necessary calls to the daemon were removed. # docker daemon got much too much load. all not 100% necessary calls to the daemon were removed.
# @image = self.class.find_image_by_tag(@execution_environment.docker_image) # @image = self.class.find_image_by_tag(@execution_environment.docker_image)
# fail(Error, "Cannot find image #{@execution_environment.docker_image}!") unless @image # fail(Error, "Cannot find image #{@execution_environment.docker_image}!") unless @image
@ -405,11 +397,10 @@ class DockerClient
def self.initialize_environment def self.initialize_environment
# TODO: Move to DockerContainerPool # TODO: Move to DockerContainerPool
unless config[:connection_timeout] && config[:workspace_root] raise(Error, 'Docker configuration missing!') unless config[:connection_timeout] && config[:workspace_root]
fail(Error, 'Docker configuration missing!')
end
Docker.url = config[:host] if config[:host] Docker.url = config[:host] if config[:host]
# todo: availability check disabled for performance reasons. Reconsider if this is necessary. # TODO: availability check disabled for performance reasons. Reconsider if this is necessary.
# docker daemon got much too much load. all not 100% necessary calls to the daemon were removed. # docker daemon got much too much load. all not 100% necessary calls to the daemon were removed.
# check_availability! # check_availability!
FileUtils.mkdir_p(LOCAL_WORKSPACE_ROOT) FileUtils.mkdir_p(LOCAL_WORKSPACE_ROOT)
@ -445,9 +436,9 @@ class DockerClient
Rails.logger.debug('returning container ' + container.to_s) Rails.logger.debug('returning container ' + container.to_s)
begin begin
clean_container_workspace(container) clean_container_workspace(container)
rescue Docker::Error::NotFoundError => error rescue Docker::Error::NotFoundError => e
# FIXME: Create new container? # FIXME: Create new container?
Rails.logger.info('return_container: Rescued from Docker::Error::NotFoundError: ' + error.to_s) Rails.logger.info('return_container: Rescued from Docker::Error::NotFoundError: ' + e.to_s)
Rails.logger.info('Nothing is done here additionally. The container will be exchanged upon its next retrieval.') Rails.logger.info('Nothing is done here additionally. The container will be exchanged upon its next retrieval.')
end end
DockerContainerPool.return_container(container, execution_environment) DockerContainerPool.return_container(container, execution_environment)
@ -456,23 +447,23 @@ class DockerClient
# private :return_container # private :return_container
def send_command(command, container, &block) def send_command(command, container)
result = {status: :failed, stdout: '', stderr: ''} result = {status: :failed, stdout: '', stderr: ''}
output = nil output = nil
Timeout.timeout(@execution_environment.permitted_execution_time.to_i) do Timeout.timeout(@execution_environment.permitted_execution_time.to_i) do
# TODO: check phusion doku again if we need -i -t options here # TODO: check phusion doku again if we need -i -t options here
output = container.exec(['bash', '-c', command]) output = container.exec(['bash', '-c', command])
end end
Rails.logger.debug "output from container.exec" Rails.logger.debug 'output from container.exec'
Rails.logger.debug output Rails.logger.debug output
if output == nil if output.nil?
kill_container(container) kill_container(container)
else else
result = {status: output[2] == 0 ? :ok : :failed, stdout: output[0].join.force_encoding('utf-8'), stderr: output[1].join.force_encoding('utf-8')} result = {status: output[2] == 0 ? :ok : :failed, stdout: output[0].join.force_encoding('utf-8'), stderr: output[1].join.force_encoding('utf-8')}
end end
# if we use pooling and recylce the containers, put it back. otherwise, destroy it. # if we use pooling and recylce the containers, put it back. otherwise, destroy it.
(DockerContainerPool.config[:active] && RECYCLE_CONTAINERS) ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container) DockerContainerPool.config[:active] && RECYCLE_CONTAINERS ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container)
result result
rescue Timeout::Error rescue Timeout::Error
Rails.logger.info('got timeout error for container ' + container.to_s) Rails.logger.info('got timeout error for container ' + container.to_s)
@ -482,6 +473,5 @@ class DockerClient
private :send_command private :send_command
class Error < RuntimeError; class Error < RuntimeError; end
end
end end