htwkmooc/codeocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply automatic rubocop fixes

Browse Source
This commit is contained in:
Sebastian Serth
2021-05-14 10:51:44 +02:00
parent fe4000916c
commit 6cbecb5b39
440 changed files with 2705 additions and 1853 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
lib/docker_client.rb
View File

@ -11,7 +11,7 @@ class DockerClient
CONTAINER_WORKSPACE_PATH = '/workspace' # '/home/python/workspace' #'/tmp/workspace'
DEFAULT_MEMORY_LIMIT = 256
# Ralf: I suggest to replace this with the environment variable. Ask Hauke why this is not the case!
LOCAL_WORKSPACE_ROOT = File.expand_path(self.config[:workspace_root])
LOCAL_WORKSPACE_ROOT = File.expand_path(config[:workspace_root])
MINIMUM_MEMORY_LIMIT = 4
RECYCLE_CONTAINERS = false
RETRY_COUNT = 2
@ -19,14 +19,13 @@ class DockerClient
MAXIMUM_CONTAINER_LIFETIME = 20.minutes
SELF_DESTROY_GRACE_PERIOD = 2.minutes
attr_reader :container
attr_reader :socket
attr_reader :container, :socket
attr_accessor :tubesock
def self.check_availability!
Timeout.timeout(config[:connection_timeout]) { Docker.version }
rescue Excon::Errors::SocketError, Timeout::Error
raise(Error, "The Docker host at #{Docker.url} is not reachable!")
raise Error.new("The Docker host at #{Docker.url} is not reachable!")
end
def self.clean_container_workspace(container)
@ -37,9 +36,9 @@ class DockerClient
if local_workspace_path && Pathname.new(local_workspace_path).exist?
Pathname.new(local_workspace_path).children.each do |p|
p.rmtree
rescue Errno::ENOENT, Errno::EACCES => error
Sentry.capture_exception(error)
Rails.logger.error("clean_container_workspace: Got #{error.class.to_s}: #{error.to_s}")
rescue Errno::ENOENT, Errno::EACCES => e
Sentry.capture_exception(e)
Rails.logger.error("clean_container_workspace: Got #{e.class}: #{e}")
end
# FileUtils.rmdir(Pathname.new(local_workspace_path))
end
@ -49,7 +48,7 @@ class DockerClient
{
class_name: File.basename(filename, File.extname(filename)).camelize,
filename: filename,
module_name: File.basename(filename, File.extname(filename)).underscore
module_name: File.basename(filename, File.extname(filename)).underscore,
}
end
@ -70,32 +69,32 @@ class DockerClient
'Binds' => mapped_directories(local_workspace_path),
'PortBindings' => mapped_ports(execution_environment),
# Resource limitations.
'NanoCPUs' => 4 * 1000000000, # CPU quota in units of 10^-9 CPUs.
'NanoCPUs' => 4 * 1_000_000_000, # CPU quota in units of 10^-9 CPUs.
'PidsLimit' => 100,
'KernelMemory' => execution_environment.memory_limit.megabytes, # if below Memory, the Docker host (!) might experience an OOM
'Memory' => execution_environment.memory_limit.megabytes,
'MemorySwap' => execution_environment.memory_limit.megabytes, # same value as Memory to disable Swap
'OomScoreAdj' => 500
'OomScoreAdj' => 500,
}
end
def create_socket(container, stderr = false)
# TODO: factor out query params
# todo separate stderr
query_params = 'logs=0&stream=1&' + (stderr ? 'stderr=1' : 'stdout=1&stdin=1')
query_params = "logs=0&stream=1&#{stderr ? 'stderr=1' : 'stdout=1&stdin=1'}"
# Headers are required by Docker
headers = {'Origin' => 'http://localhost'}
socket_url = DockerClient.config['ws_host'] + '/v1.27/containers/' + @container.id + '/attach/ws?' + query_params
socket_url = "#{DockerClient.config['ws_host']}/v1.27/containers/#{@container.id}/attach/ws?#{query_params}"
# The ping value is measured in seconds and specifies how often a Ping frame should be sent.
# Internally, Faye::WebSocket uses EventMachine and the ping value is used to wake the EventMachine thread
socket = Faye::WebSocket::Client.new(socket_url, [], headers: headers, ping: 0.1)
Rails.logger.debug 'Opening Websocket on URL ' + socket_url
Rails.logger.debug "Opening Websocket on URL #{socket_url}"
socket.on :error do |event|
Rails.logger.info 'Websocket error: ' + event.message.to_s
Rails.logger.info "Websocket error: #{event.message}"
end
socket.on :close do |_event|
Rails.logger.info 'Websocket closed.'
@ -117,10 +116,10 @@ class DockerClient
# this is however not guaranteed and caused issues on the server already. Therefore create the necessary folders manually!
local_workspace_path = generate_local_workspace_path
FileUtils.mkdir(local_workspace_path)
FileUtils.chmod_R(0777, local_workspace_path)
FileUtils.chmod_R(0o777, local_workspace_path)
container = Docker::Container.create(container_creation_options(execution_environment, local_workspace_path))
container.start
container.start_time = Time.now
container.start_time = Time.zone.now
container.status = :created
container.execution_environment = execution_environment
container.re_use = true
@ -130,19 +129,19 @@ class DockerClient
timeout = Random.rand(MINIMUM_CONTAINER_LIFETIME..MAXIMUM_CONTAINER_LIFETIME) # seconds
sleep(timeout)
container.re_use = false
if container.status != :executing
container.docker_client.kill_container(container, false)
Rails.logger.info('Killing container in status ' + container.status.to_s + ' after ' + (Time.now - container.start_time).to_s + ' seconds.')
else
if container.status == :executing
Thread.new do
timeout = SELF_DESTROY_GRACE_PERIOD.to_i
sleep(timeout)
container.docker_client.kill_container(container, false)
Rails.logger.info('Force killing container in status ' + container.status.to_s + ' after ' + (Time.now - container.start_time).to_s + ' seconds.')
Rails.logger.info("Force killing container in status #{container.status} after #{Time.zone.now - container.start_time} seconds.")
ensure
# guarantee that the thread is releasing the DB connection after it is done
ActiveRecord::Base.connection_pool.release_connection
end
else
container.docker_client.kill_container(container, false)
Rails.logger.info("Killing container in status #{container.status} after #{Time.zone.now - container.start_time} seconds.")
end
ensure
# guarantee that the thread is releasing the DB connection after it is done
@ -150,8 +149,8 @@ class DockerClient
end
container
rescue Docker::Error::NotFoundError => error
Rails.logger.error('create_container: Got Docker::Error::NotFoundError: ' + error.to_s)
rescue Docker::Error::NotFoundError => e
Rails.logger.error("create_container: Got Docker::Error::NotFoundError: #{e}")
destroy_container(container)
# (tries += 1) <= RETRY_COUNT ? retry : raise(error)
end
@ -169,7 +168,7 @@ class DockerClient
end
FileUtils.chmod_R('+rwX', self.class.local_workspace_path(container))
rescue Docker::Error::NotFoundError => e
Rails.logger.info('create_workspace_files: Rescued from Docker::Error::NotFoundError: ' + e.to_s)
Rails.logger.info("create_workspace_files: Rescued from Docker::Error::NotFoundError: #{e}")
end
private :create_workspace_files
@ -187,7 +186,7 @@ class DockerClient
# create a temporary dir, put all files in it, and put it into the container. the dir is automatically removed when leaving the block.
Dir.mktmpdir do |dir|
submission.collect_files.each do |file|
disk_file = File.new(dir + '/' + (file.path || '') + file.name_with_extension, 'w')
disk_file = File.new("#{dir}/#{file.path || ''}#{file.name_with_extension}", 'w')
disk_file.write(file.content)
disk_file.close
end
@ -198,7 +197,7 @@ class DockerClient
# container.exec(['bash', '-c', 'chown -R python ' + CONTAINER_WORKSPACE_PATH])
# container.exec(['bash', '-c', 'chgrp -G python ' + CONTAINER_WORKSPACE_PATH])
rescue StandardError => e
Rails.logger.error('create workspace folder: Rescued from StandardError: ' + e.to_s)
Rails.logger.error("create workspace folder: Rescued from StandardError: #{e}")
end
# sleep 1000
@ -207,38 +206,39 @@ class DockerClient
# tar the files in dir and put the tar to CONTAINER_WORKSPACE_PATH in the container
container.archive_in(dir, CONTAINER_WORKSPACE_PATH, overwrite: false)
rescue StandardError => e
Rails.logger.error('insert tar: Rescued from StandardError: ' + e.to_s)
Rails.logger.error("insert tar: Rescued from StandardError: #{e}")
end
# Rails.logger.info('command: tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH)
begin
# untar the tar file placed in the CONTAINER_WORKSPACE_PATH
container.exec(['bash', '-c', 'tar -xf ' + CONTAINER_WORKSPACE_PATH + '/' + dir.split('/tmp/')[1] + ' -C ' + CONTAINER_WORKSPACE_PATH])
container.exec(['bash', '-c',
"tar -xf #{CONTAINER_WORKSPACE_PATH}/#{dir.split('/tmp/')[1]} -C #{CONTAINER_WORKSPACE_PATH}"])
rescue StandardError => e
Rails.logger.error('untar: Rescued from StandardError: ' + e.to_s)
Rails.logger.error("untar: Rescued from StandardError: #{e}")
end
# sleep 1000
end
rescue StandardError => e
Rails.logger.error('create_workspace_files_transmit: Rescued from StandardError: ' + e.to_s)
Rails.logger.error("create_workspace_files_transmit: Rescued from StandardError: #{e}")
end
def self.destroy_container(container)
@socket&.close
Rails.logger.info('destroying container ' + container.to_s)
Rails.logger.info("destroying container #{container}")
# Checks only if container assignment is not nil and not whether the container itself is still present.
if container && !DockerContainerPool.config[:active]
container.kill
container.port_bindings.values.each { |port| PortPool.release(port) }
container.port_bindings.each_value {|port| PortPool.release(port) }
begin
clean_container_workspace(container)
FileUtils.rmtree(local_workspace_path(container))
rescue Errno::ENOENT, Errno::EACCES => error
Sentry.capture_exception(error)
Rails.logger.error("clean_container_workspace: Got #{error.class.to_s}: #{error.to_s}")
rescue Errno::ENOENT, Errno::EACCES => e
Sentry.capture_exception(e)
Rails.logger.error("clean_container_workspace: Got #{e.class}: #{e}")
end
# Checks only if container assignment is not nil and not whether the container itself is still present.
@ -246,11 +246,11 @@ class DockerClient
elsif container
DockerContainerPool.destroy_container(container)
end
rescue Docker::Error::NotFoundError => error
Rails.logger.error('destroy_container: Rescued from Docker::Error::NotFoundError: ' + error.to_s)
rescue Docker::Error::NotFoundError => e
Rails.logger.error("destroy_container: Rescued from Docker::Error::NotFoundError: #{e}")
Rails.logger.error('No further actions are done concerning that.')
rescue Docker::Error::ConflictError => error
Rails.logger.error('destroy_container: Rescued from Docker::Error::ConflictError: ' + error.to_s)
rescue Docker::Error::ConflictError => e
Rails.logger.error("destroy_container: Rescued from Docker::Error::ConflictError: #{e}")
Rails.logger.error('No further actions are done concerning that.')
end
@ -263,21 +263,22 @@ class DockerClient
# only used by score and execute_arbitrary_command
def execute_command(command, before_execution_block, output_consuming_block)
# tries ||= 0
container_request_time = Time.now
container_request_time = Time.zone.now
@container = DockerContainerPool.get_container(@execution_environment)
waiting_for_container_time = Time.now - container_request_time
waiting_for_container_time = Time.zone.now - container_request_time
if @container
@container.status = :executing
before_execution_block.try(:call)
execution_request_time = Time.now
execution_request_time = Time.zone.now
command_result = send_command(command, @container, &output_consuming_block)
container_execution_time = Time.now - execution_request_time
container_execution_time = Time.zone.now - execution_request_time
command_result.merge!(waiting_for_container_time: waiting_for_container_time)
command_result.merge!(container_execution_time: container_execution_time)
command_result[:waiting_for_container_time] = waiting_for_container_time
command_result[:container_execution_time] = container_execution_time
command_result
else
{status: :container_depleted, waiting_for_container_time: waiting_for_container_time, container_execution_time: nil}
{status: :container_depleted, waiting_for_container_time: waiting_for_container_time,
container_execution_time: nil}
end
rescue Excon::Errors::SocketError => e
# socket errors seems to be normal when using exec
@ -298,7 +299,7 @@ class DockerClient
# Prevent catching this error here
raise
rescue StandardError => e
Rails.logger.error('execute_websocket_command: Rescued from StandardError caused by before_execution_block.call: ' + e.to_s)
Rails.logger.error("execute_websocket_command: Rescued from StandardError caused by before_execution_block.call: #{e}")
end
# TODO: catch exception if socket could not be created
@socket ||= create_socket(@container)
@ -309,16 +310,16 @@ class DockerClient
end
def kill_after_timeout(container)
"""
"
We need to start a second thread to kill the websocket connection,
as it is impossible to determine whether further input is requested.
"""
"
container.status = :executing
@thread = Thread.new do
timeout = (@execution_environment.permitted_execution_time.to_i) # seconds
timeout = @execution_environment.permitted_execution_time.to_i # seconds
sleep(timeout)
if container && container.status != :available
Rails.logger.info('Killing container after timeout of ' + timeout.to_s + ' seconds.')
Rails.logger.info("Killing container after timeout of #{timeout} seconds.")
# send timeout to the tubesock socket
# FIXME: 2nd thread to notify user.
@tubesock&.send_data JSON.dump({'cmd' => 'timeout'})
@ -338,7 +339,7 @@ class DockerClient
ActiveRecord::Base.connection_pool.release_connection
end
else
Rails.logger.info('Container' + container.to_s + ' already removed.')
Rails.logger.info("Container#{container} already removed.")
end
ensure
# guarantee that the thread is releasing the DB connection after it is done
@ -351,25 +352,30 @@ class DockerClient
end
def exit_container(container)
Rails.logger.debug('exiting container ' + container.to_s)
Rails.logger.debug("exiting container #{container}")
# exit the timeout thread if it is still alive
exit_thread_if_alive
@socket.close
# if we use pooling and recylce the containers, put it back. otherwise, destroy it.
DockerContainerPool.config[:active] && RECYCLE_CONTAINERS ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container)
if DockerContainerPool.config[:active] && RECYCLE_CONTAINERS
self.class.return_container(container,
@execution_environment)
else
self.class.destroy_container(container)
end
end
def kill_container(container, _create_new = true)
exit_thread_if_alive
Rails.logger.info('killing container ' + container.to_s)
Rails.logger.info("killing container #{container}")
self.class.destroy_container(container)
end
def execute_run_command(submission, filename, &block)
"""
"
Run commands by attaching a websocket to Docker.
"""
filepath = submission.collect_files.find { |f| f.name_with_extension == filename }.filepath
"
filepath = submission.collect_files.find {|f| f.name_with_extension == filename }.filepath
command = submission.execution_environment.run_command % command_substitutions(filepath)
create_workspace_files = proc { create_workspace_files(container, submission) }
open_websocket_connection(command, create_workspace_files, block)
@ -377,15 +383,15 @@ class DockerClient
end
def execute_test_command(submission, filename, &block)
"""
"
Stick to existing Docker API with exec command.
"""
file = submission.collect_files.find { |f| f.name_with_extension == filename }
"
file = submission.collect_files.find {|f| f.name_with_extension == filename }
filepath = file.filepath
command = submission.execution_environment.test_command % command_substitutions(filepath)
create_workspace_files = proc { create_workspace_files(container, submission) }
test_result = execute_command(command, create_workspace_files, block)
test_result.merge!(file_role: file.role)
test_result[:file_role] = file.role
test_result
end
@ -404,7 +410,7 @@ class DockerClient
end
def self.image_tags
Docker::Image.all.map { |image| image.info['RepoTags'] }.flatten.reject { |tag| tag.nil? || tag.include?('<none>') }
Docker::Image.all.map {|image| image.info['RepoTags'] }.flatten.reject {|tag| tag.nil? || tag.include?('<none>') }
end
def initialize(options = {})
@ -417,7 +423,7 @@ class DockerClient
def self.initialize_environment
# TODO: Move to DockerContainerPool
raise(Error, 'Docker configuration missing!') unless config[:connection_timeout] && config[:workspace_root]
raise Error.new('Docker configuration missing!') unless config[:connection_timeout] && config[:workspace_root]
Docker.url = config[:host] if config[:host]
# TODO: availability check disabled for performance reasons. Reconsider if this is necessary.
@ -427,11 +433,13 @@ class DockerClient
end
def local_file_path(options = {})
resulting_file_path = File.join(self.class.local_workspace_path(options[:container]), options[:file].path || '', options[:file].name_with_extension)
resulting_file_path = File.join(self.class.local_workspace_path(options[:container]), options[:file].path || '',
options[:file].name_with_extension)
absolute_path = File.expand_path(resulting_file_path)
unless absolute_path.start_with? self.class.local_workspace_path(options[:container]).to_s
raise(FilepathError, 'Filepath not allowed')
raise FilepathError.new('Filepath not allowed')
end
absolute_path
end
@ -457,12 +465,12 @@ class DockerClient
end
def self.return_container(container, execution_environment)
Rails.logger.debug('returning container ' + container.to_s)
Rails.logger.debug("returning container #{container}")
begin
clean_container_workspace(container)
rescue Docker::Error::NotFoundError => error
rescue Docker::Error::NotFoundError => e
# FIXME: Create new container?
Rails.logger.info('return_container: Rescued from Docker::Error::NotFoundError: ' + error.to_s)
Rails.logger.info("return_container: Rescued from Docker::Error::NotFoundError: #{e}")
Rails.logger.info('Nothing is done here additionally. The container will be exchanged upon its next retrieval.')
end
DockerContainerPool.return_container(container, execution_environment)
@ -477,21 +485,30 @@ class DockerClient
Timeout.timeout(@execution_environment.permitted_execution_time.to_i) do
# TODO: check phusion doku again if we need -i -t options here
# https://stackoverflow.com/questions/363223/how-do-i-get-both-stdout-and-stderr-to-go-to-the-terminal-and-a-log-file
output = container.exec(['bash', '-c', "#{command} 1> >(tee -a /tmp/stdout.log) 2> >(tee -a /tmp/stderr.log >&2); rm -f /tmp/std*.log"], tty: false)
output = container.exec(
['bash', '-c',
"#{command} 1> >(tee -a /tmp/stdout.log) 2> >(tee -a /tmp/stderr.log >&2); rm -f /tmp/std*.log"], tty: false
)
end
Rails.logger.debug 'output from container.exec'
Rails.logger.debug output
if output.nil?
kill_container(container)
else
result = {status: output[2] == 0 ? :ok : :failed, stdout: output[0].join.force_encoding('utf-8'), stderr: output[1].join.force_encoding('utf-8')}
result = {status: (output[2]).zero? ? :ok : :failed, stdout: output[0].join.force_encoding('utf-8'),
stderr: output[1].join.force_encoding('utf-8')}
end
# if we use pooling and recylce the containers, put it back. otherwise, destroy it.
DockerContainerPool.config[:active] && RECYCLE_CONTAINERS ? self.class.return_container(container, @execution_environment) : self.class.destroy_container(container)
if DockerContainerPool.config[:active] && RECYCLE_CONTAINERS
self.class.return_container(container,
@execution_environment)
else
self.class.destroy_container(container)
end
result
rescue Timeout::Error
Rails.logger.info('got timeout error for container ' + container.to_s)
Rails.logger.info("got timeout error for container #{container}")
stdout = container.exec(['cat', '/tmp/stdout.log'])[0].join.force_encoding('utf-8')
stderr = container.exec(['cat', '/tmp/stderr.log'])[0].join.force_encoding('utf-8')
kill_container(container)
@ -501,5 +518,6 @@ class DockerClient
private :send_command
class Error < RuntimeError; end
class FilepathError < RuntimeError; end
end