transferred Code Ocean from original repository to GitHub

spec/lib/assessor_spec.rb

@@ -0,0 +1,25 @@
+require 'rails_helper'
+
+describe Assessor do
+  describe '#calculate_score' do
+    let(:count) { 42 }
+    let(:passed) { 17 }
+    let(:test_outcome) { {count: count, passed: passed} }
+
+    context 'with a testing framework adapter' do
+      let(:assessor) { Assessor.new(execution_environment: FactoryGirl.build(:ruby)) }
+
+      it 'returns the correct score' do
+        expect(assessor.send(:calculate_score, test_outcome)).to eq(passed.to_f / count.to_f)
+      end
+    end
+
+    context 'without a testing framework adapter' do
+      let(:assessor) { Assessor.new(execution_environment: FactoryGirl.build(:execution_environment)) }
+
+      it 'raises an error' do
+        expect { assessor.send(:calculate_score, test_outcome) }.to raise_error
+      end
+    end
+  end
+end

spec/lib/docker_client_spec.rb

@@ -0,0 +1,255 @@
+require 'rails_helper'
+require 'seeds_helper'
+
+describe DockerClient, docker: true do
+  let(:command) { 'whoami' }
+  let(:docker_client) { DockerClient.new(execution_environment: FactoryGirl.build(:ruby), user: FactoryGirl.build(:admin)) }
+  let(:image) { double }
+  let(:submission) { FactoryGirl.create(:submission) }
+  let(:workspace_path) { '/tmp' }
+
+  describe '#bound_folders' do
+    context 'when executing a submission' do
+      before(:each) { docker_client.instance_variable_set(:@submission, submission) }
+
+      it 'returns a submission-specific mapping' do
+        mapping = docker_client.send(:bound_folders).first
+        expect(mapping).to include(submission.id.to_s)
+        expect(mapping).to end_with(DockerClient::CONTAINER_WORKSPACE_PATH)
+      end
+    end
+
+    context 'when executing a single command' do
+      it 'returns an empty mapping' do
+        expect(docker_client.send(:bound_folders)).to eq([])
+      end
+    end
+  end
+
+  describe '.check_availability!' do
+    context 'when a socket error occurs' do
+      it 'raises an error' do
+        expect(Docker).to receive(:version).and_raise(Excon::Errors::SocketError.new(StandardError.new))
+        expect { DockerClient.check_availability! }.to raise_error(DockerClient::Error)
+      end
+    end
+
+    context 'when a timeout occurs' do
+      it 'raises an error' do
+        expect(Docker).to receive(:version).and_raise(Timeout::Error)
+        expect { DockerClient.check_availability! }.to raise_error(DockerClient::Error)
+      end
+    end
+  end
+
+  describe '#clean_workspace' do
+    it 'removes the submission-specific directory' do
+      expect(docker_client).to receive(:local_workspace_path).and_return(workspace_path)
+      expect(FileUtils).to receive(:rm_rf).with(workspace_path)
+      docker_client.send(:clean_workspace)
+    end
+  end
+
+  describe '#create_container' do
+    let(:image_tag) { 'tag' }
+    before(:each) { docker_client.instance_variable_set(:@image, image) }
+
+    it 'creates a container' do
+      expect(image).to receive(:info).and_return({'RepoTags' => [image_tag]})
+      expect(Docker::Container).to receive(:create).with('Cmd' => command, 'Image' => image_tag)
+      docker_client.send(:create_container, command: command)
+    end
+  end
+
+  describe '#create_workspace' do
+    before(:each) { docker_client.instance_variable_set(:@submission, submission) }
+
+    it 'creates submission-specific directories' do
+      expect(docker_client).to receive(:local_workspace_path).at_least(:once).and_return(workspace_path)
+      expect(Dir).to receive(:mkdir).at_least(:once)
+      docker_client.send(:create_workspace)
+    end
+  end
+
+  describe '#create_workspace_file' do
+    let(:file) { FactoryGirl.build(:file, content: 'puts 42') }
+    let(:file_path) { File.join(workspace_path, file.name_with_extension) }
+
+    it 'creates a file' do
+      expect(docker_client).to receive(:local_workspace_path).and_return(workspace_path)
+      docker_client.send(:create_workspace_file, file: file)
+      expect(File.exist?(file_path)).to be true
+      expect(File.new(file_path, 'r').read).to eq(file.content)
+      File.delete(file_path)
+    end
+  end
+
+  describe '.destroy_container' do
+    let(:container) { docker_client.send(:create_container, {command: command}) }
+    after(:each) { DockerClient.destroy_container(container) }
+
+    it 'stops the container' do
+      expect(container).to receive(:stop).and_return(container)
+    end
+
+    it 'kills the container' do
+      expect(container).to receive(:kill)
+    end
+
+    it 'releases allocated ports' do
+      expect(container).to receive(:json).at_least(:once).and_return({'HostConfig' => {'PortBindings' => {foo: [{'HostPort' => '42'}]}}})
+      docker_client.send(:start_container, container)
+      expect(PortPool).to receive(:release)
+    end
+  end
+
+  describe '#execute_command' do
+    after(:each) { docker_client.send(:execute_command, command) }
+
+    it 'creates a container' do
+      expect(docker_client).to receive(:create_container).with(command: ['bash', '-c', command]).and_call_original
+    end
+
+    it 'starts the container' do
+      expect(docker_client).to receive(:start_container)
+    end
+  end
+
+  describe '#execute_in_workspace' do
+    let(:block) { Proc.new do; end }
+    let(:execute_in_workspace) { docker_client.send(:execute_in_workspace, submission, &block) }
+    after(:each) { execute_in_workspace }
+
+    it 'creates the workspace' do
+      expect(docker_client).to receive(:create_workspace)
+    end
+
+    it 'calls the block' do
+      expect(block).to receive(:call)
+    end
+
+    it 'cleans the workspace' do
+      expect(docker_client).to receive(:clean_workspace)
+    end
+  end
+
+  describe '#execute_run_command' do
+    let(:block) { Proc.new {} }
+    let(:filename) { submission.exercise.files.detect { |file| file.role == 'main_file' }.name_with_extension }
+    after(:each) { docker_client.send(:execute_run_command, submission, filename, &block) }
+
+    it 'is executed in the workspace' do
+      expect(docker_client).to receive(:execute_in_workspace)
+    end
+
+    it 'executes the run command' do
+      expect(docker_client).to receive(:execute_command).with(kind_of(String), &block)
+    end
+  end
+
+  describe '#execute_test_command' do
+    let(:filename) { submission.exercise.files.detect { |file| file.role == 'teacher_defined_test' }.name_with_extension }
+    after(:each) { docker_client.send(:execute_test_command, submission, filename) }
+
+    it 'is executed in the workspace' do
+      expect(docker_client).to receive(:execute_in_workspace)
+    end
+
+    it 'executes the test command' do
+      expect(docker_client).to receive(:execute_command).with(kind_of(String))
+    end
+  end
+
+  describe '.initialize_environment' do
+    let(:config) { {connection_timeout: 3, host: 'tcp://8.8.8.8:2375', workspace_root: '/'} }
+
+    context 'with complete configuration' do
+      before(:each) { expect(DockerClient).to receive(:config).at_least(:once).and_return(config) }
+
+      it 'does not raise an error' do
+        expect { DockerClient.initialize_environment }.not_to raise_error
+      end
+    end
+
+    context 'with incomplete configuration' do
+      before(:each) { expect(DockerClient).to receive(:config).at_least(:once).and_return({}) }
+
+      it 'raises an error' do
+        expect { DockerClient.initialize_environment }.to raise_error(DockerClient::Error)
+      end
+    end
+  end
+
+  describe '#local_workspace_path' do
+    before(:each) { docker_client.instance_variable_set(:@submission, submission) }
+
+    it 'includes the correct workspace root' do
+      expect(docker_client.send(:local_workspace_path)).to start_with(DockerClient::LOCAL_WORKSPACE_ROOT.to_s)
+    end
+
+    it 'is submission-specific' do
+      expect(docker_client.send(:local_workspace_path)).to end_with(submission.id.to_s)
+    end
+  end
+
+  describe '#remote_workspace_path' do
+    before(:each) { docker_client.instance_variable_set(:@submission, submission) }
+
+    it 'includes the correct workspace root' do
+      expect(docker_client.send(:remote_workspace_path)).to start_with(DockerClient.config[:workspace_root])
+    end
+
+    it 'is submission-specific' do
+      expect(docker_client.send(:remote_workspace_path)).to end_with(submission.id.to_s)
+    end
+  end
+
+  describe '#start_container' do
+    let(:container) { docker_client.send(:create_container, command: command) }
+    let(:start_container) { docker_client.send(:start_container, container) }
+
+    it 'configures bound folders' do
+      expect(container).to receive(:start).with(hash_including('Binds' => kind_of(Array))).and_call_original
+      start_container
+    end
+
+    it 'configures bound ports' do
+      expect(container).to receive(:start).with(hash_including('PortBindings' => kind_of(Hash))).and_call_original
+      start_container
+    end
+
+    it 'starts the container' do
+      expect(container).to receive(:start).and_call_original
+      start_container
+    end
+
+    it 'waits for the container to terminate' do
+      expect(container).to receive(:wait).with(kind_of(Numeric)).and_call_original
+      start_container
+    end
+
+    context 'when a timeout occurs' do
+      before(:each) { expect(container).to receive(:wait).and_raise(Docker::Error::TimeoutError) }
+
+      it 'kills the container' do
+        expect(container).to receive(:kill)
+        start_container
+      end
+
+      it 'returns a corresponding status' do
+        expect(start_container[:status]).to eq(:timeout)
+      end
+    end
+
+    context 'when the container terminates timely' do
+      it "returns the container's output" do
+        expect(start_container[:stderr]).to be_blank
+        expect(start_container[:stdout]).to start_with('root')
+      end
+
+      it 'returns a corresponding status' do
+        expect(start_container[:status]).to eq(:ok)
+      end
+    end
+  end
+end

spec/lib/generators/testing_framework_adapter_generator_spec.rb

@@ -0,0 +1,39 @@
+require 'rails/generators'
+require 'generators/testing_framework_adapter_generator'
+require 'rails_helper'
+
+describe TestingFrameworkAdapterGenerator do
+  describe '#create_testing_framework_adapter' do
+    let(:name) { 'TestUnit' }
+    let(:path) { Rails.root.join('lib', "#{name.underscore}_adapter.rb") }
+    let(:spec_path) { Rails.root.join('spec', 'lib', "#{name.underscore}_adapter_spec.rb") }
+
+    before(:each) do
+      Rails::Generators.invoke('testing_framework_adapter', [name])
+    end
+
+    after(:each) do
+      File.delete(path)
+      File.delete(spec_path)
+    end
+
+    it 'generates a correctly named file' do
+      expect(File.exist?(path)).to be true
+    end
+
+    it 'builds a correct class skeleton' do
+      file_content = File.new(path, 'r').read
+      expect(file_content).to start_with("class #{name}Adapter < TestingFrameworkAdapter")
+    end
+
+    it 'generates a corresponding test' do
+      expect(File.exist?(spec_path)).to be true
+    end
+
+    it 'builds a correct test skeleton' do
+      file_content = File.new(spec_path, 'r').read
+      expect(file_content).to include("describe #{name}Adapter")
+      expect(file_content).to include("describe '#parse_output'")
+    end
+  end
+end

spec/lib/junit_adapter_spec.rb

@@ -0,0 +1,26 @@
+require 'rails_helper'
+
+describe JunitAdapter do
+  let(:adapter) { JunitAdapter.new }
+
+  describe '#parse_output' do
+    context 'with failed tests' do
+      let(:count) { 42 }
+      let(:failed) { 25 }
+      let(:stdout) { "FAILURES!!!\nTests run: #{count},  Failures: #{failed}" }
+
+      it 'returns the correct numbers' do
+        expect(adapter.parse_output(stdout: stdout)).to eq({count: count, failed: failed})
+      end
+    end
+
+    context 'without failed tests' do
+      let(:count) { 42 }
+      let(:stdout) { "OK (#{count} tests)" }
+
+      it 'returns the correct numbers' do
+        expect(adapter.parse_output(stdout: stdout)).to eq({count: count, passed: count})
+      end
+    end
+  end
+end

spec/lib/nonce_store_spec.rb

@@ -0,0 +1,41 @@
+require 'rails_helper'
+
+describe NonceStore do
+  let(:nonce) { SecureRandom.hex }
+
+  describe '.add' do
+    it 'stores a nonce in the cache' do
+      expect(Rails.cache).to receive(:write)
+      NonceStore.add(nonce)
+    end
+  end
+
+  describe '.delete' do
+    it 'deletes a nonce from the cache' do
+      expect(Rails.cache).to receive(:write)
+      NonceStore.add(nonce)
+      NonceStore.delete(nonce)
+      expect(NonceStore.has?(nonce)).to be false
+    end
+  end
+
+  describe '.has?' do
+    it 'returns true for present nonces' do
+      NonceStore.add(nonce)
+      expect(NonceStore.has?(nonce)).to be true
+    end
+
+    it 'returns false for expired nonces' do
+      Lti.send(:remove_const, 'MAXIMUM_SESSION_AGE')
+      Lti::MAXIMUM_SESSION_AGE = 1
+      NonceStore.add(nonce)
+      expect(NonceStore.has?(nonce)).to be true
+      sleep(Lti::MAXIMUM_SESSION_AGE)
+      expect(NonceStore.has?(nonce)).to be false
+    end
+
+    it 'returns false for absent nonces' do
+      expect(NonceStore.has?(nonce)).to be false
+    end
+  end
+end

spec/lib/port_pool_spec.rb

@@ -0,0 +1,55 @@
+require 'rails_helper'
+
+describe PortPool do
+  describe '.available_port' do
+    it 'is synchronized' do
+      expect(PortPool.instance_variable_get(:@mutex)).to receive(:synchronize)
+      PortPool.available_port
+    end
+
+    context 'when a port is available' do
+      it 'returns the port' do
+        expect(PortPool.available_port).to be_a(Numeric)
+      end
+
+      it 'removes the port from the list of available ports' do
+        port = PortPool.available_port
+        expect(PortPool.instance_variable_get(:@available_ports)).not_to include(port)
+      end
+    end
+
+    context 'when no port is available' do
+      it 'returns the port' do
+        available_ports = PortPool.instance_variable_get(:@available_ports)
+        PortPool.instance_variable_set(:@available_ports, [])
+        expect(PortPool.available_port).to be_nil
+        PortPool.instance_variable_set(:@available_ports, available_ports)
+      end
+    end
+  end
+
+  describe '.release' do
+    context 'when the port has been obtained earlier' do
+      it 'adds the port to the list of available ports' do
+        port = PortPool.available_port
+        expect(PortPool.instance_variable_get(:@available_ports)).not_to include(port)
+        PortPool.release(port)
+        expect(PortPool.instance_variable_get(:@available_ports)).to include(port)
+      end
+    end
+
+    context 'when the port has not been obtained earlier' do
+      it 'does not add the port to the list of available ports' do
+        port = PortPool.instance_variable_get(:@available_ports).sample
+        expect { PortPool.release(port) }.not_to change { PortPool.instance_variable_get(:@available_ports).length }
+      end
+    end
+
+    context 'when the port is not included in the port range' do
+      it 'does not add the port to the list of available ports' do
+        port = nil
+        expect { PortPool.release(port) }.not_to change { PortPool.instance_variable_get(:@available_ports).length }
+      end
+    end
+  end
+end

spec/lib/py_unit_adapter_spec.rb

@@ -0,0 +1,14 @@
+require 'rails_helper'
+
+describe PyUnitAdapter do
+  let(:adapter) { PyUnitAdapter.new }
+  let(:count) { 42 }
+  let(:failed) { 25 }
+  let(:stderr) { "Ran #{count} tests in 0.1s\n\nFAILED (failures=#{failed})" }
+
+  describe '#parse_output' do
+    it 'returns the correct numbers' do
+      expect(adapter.parse_output(stderr: stderr)).to eq({count: count, failed: failed})
+    end
+  end
+end

spec/lib/rspec_adapter_spec.rb

@@ -0,0 +1,14 @@
+require 'rails_helper'
+
+describe RspecAdapter do
+  let(:adapter) { RspecAdapter.new }
+  let(:count) { 42 }
+  let(:failed) { 25 }
+  let(:stdout) { "Finished in 0.1 seconds (files took 0.1 seconds to load)\n#{count} examples, #{failed} failures" }
+
+  describe '#parse_output' do
+    it 'returns the correct numbers' do
+      expect(adapter.parse_output(stdout: stdout)).to eq({count: count, failed: failed})
+    end
+  end
+end

spec/lib/sql_result_set_comparator_adapter_spec.rb

@@ -0,0 +1,31 @@
+require 'rails_helper'
+
+describe SqlResultSetComparatorAdapter do
+  let(:adapter) { SqlResultSetComparatorAdapter.new }
+
+  describe '#parse_output' do
+    context 'with missing tuples' do
+      let(:stdout) { "Missing tuples: [1]\nUnexpected tuples: []" }
+
+      it 'considers the test as failed' do
+        expect(adapter.parse_output(stdout: stdout)).to eq({count: 1, failed: 1})
+      end
+    end
+
+    context 'with unexpected tuples' do
+      let(:stdout) { "Missing tuples: []\nUnexpected tuples: [1]" }
+
+      it 'considers the test as failed' do
+        expect(adapter.parse_output(stdout: stdout)).to eq({count: 1, failed: 1})
+      end
+    end
+
+    context 'without missing or unexpected tuples' do
+      let(:stdout) { "Missing tuples: []\nUnexpected tuples: []" }
+
+      it 'considers the test as passed' do
+        expect(adapter.parse_output(stdout: stdout)).to eq({count: 1, passed: 1})
+      end
+    end
+  end
+end

spec/lib/testing_framework_adapter_spec.rb

@@ -0,0 +1,41 @@
+require 'rails_helper'
+
+describe TestingFrameworkAdapter do
+  let(:adapter) { TestingFrameworkAdapter.new }
+  let(:count) { 42 }
+  let(:failed) { 25 }
+  let(:passed) { 17 }
+
+  describe '#augment_output' do
+    context 'when missing the count of all tests' do
+      it 'adds the count of all tests' do
+        expect(adapter.send(:augment_output, failed: failed, passed: passed)).to include(count: count)
+      end
+    end
+
+    context 'when missing the count of failed tests' do
+      it 'adds the count of failed tests' do
+        expect(adapter.send(:augment_output, count: count, passed: passed)).to include(failed: failed)
+      end
+    end
+
+    context 'when missing the count of passed tests' do
+      it 'adds the count of passed tests' do
+        expect(adapter.send(:augment_output, count: count, failed: failed)).to include(passed: passed)
+      end
+    end
+  end
+
+  describe '#parse_output' do
+    it 'requires subclasses to implement #parse_output' do
+      expect { adapter.send(:parse_output, '') }.to raise_error(NotImplementedError)
+    end
+  end
+
+  describe '#test_outcome' do
+    it 'calls the framework-specific implementation' do
+      expect(adapter).to receive(:parse_output).and_return(count: count, failed: failed, passed: passed)
+      adapter.test_outcome('')
+    end
+  end
+end

spec/lib/whistleblower_spec.rb

@@ -0,0 +1,28 @@
+require 'rails_helper'
+
+describe Whistleblower do
+  let(:hint) { FactoryGirl.create(:ruby_no_method_error) }
+  let(:stderr) { "undefined method `foo' for main:Object (NoMethodError)" }
+  let(:whistleblower) { Whistleblower.new(execution_environment: hint.execution_environment) }
+
+  describe '#find_hint' do
+    let(:find_hint) { whistleblower.send(:find_hint, stderr) }
+
+    it 'finds the hint' do
+      expect(find_hint).to eq(hint)
+    end
+
+    it 'stores the matches' do
+      find_hint
+      expect(whistleblower.instance_variable_get(:@matches)).to be_a(MatchData)
+    end
+  end
+
+  describe '#generate_hint' do
+    it 'returns the customized hint message' do
+      message = whistleblower.generate_hint(stderr)
+      expect(message[0..9]).to eq(hint.message[0..9])
+      expect(message[-10..-1]).to eq(hint.message[-10..-1])
+    end
+  end
+end