diff --git a/lib/docker_client.rb b/lib/docker_client.rb
index 28ff0ad3..00fd8872 100644
--- a/lib/docker_client.rb
+++ b/lib/docker_client.rb
@@ -68,6 +68,7 @@ class DockerClient
         PortPool.release(port)
       end
     end
+    container.delete(force: true)
   end
 
   def execute_command(command, &block)
diff --git a/spec/lib/docker_client_spec.rb b/spec/lib/docker_client_spec.rb
index cc770a15..d27284a4 100644
--- a/spec/lib/docker_client_spec.rb
+++ b/spec/lib/docker_client_spec.rb
@@ -98,9 +98,12 @@ describe DockerClient, docker: true do
 
     it 'releases allocated ports' do
       expect(container).to receive(:json).at_least(:once).and_return({'HostConfig' => {'PortBindings' => {foo: [{'HostPort' => '42'}]}}})
-      docker_client.send(:start_container, container)
       expect(PortPool).to receive(:release)
     end
+
+    it 'deletes the container' do
+      expect(container).to receive(:delete)
+    end
   end
 
   describe '#execute_command' do