#46 Add Prometheus exporter

This commit is contained in:
Tobias Kantusch
2021-02-18 18:02:56 +01:00
committed by Sebastian Serth
parent 39fcd255f9
commit 44b32b6f6a
26 changed files with 2121 additions and 45 deletions

View File

@ -20,3 +20,9 @@ Style/Documentation:
Enabled: false
Layout/SpaceInsideHashLiteralBraces:
EnforcedStyle: no_space
RSpec/MultipleExpectations:
Max: 2
Metrics/BlockLength:
Exclude:
- 'spec/**/*.rb'
- 'config/environments/*.rb'

View File

@ -21,6 +21,7 @@ gem 'nokogiri'
gem 'pagedown-bootstrap-rails'
gem 'pg'
gem 'proforma', git: 'https://github.com/openHPI/proforma.git', tag: 'v0.5'
gem 'prometheus_exporter'
gem 'pry-byebug'
gem 'puma'
gem 'pundit'

View File

@ -248,6 +248,8 @@ GEM
ast (~> 2.4.1)
path_expander (1.1.0)
pg (1.2.3)
prometheus_exporter (0.7.0)
webrick
pry (0.13.1)
coderay (~> 1.1)
method_source (~> 1.0)
@ -469,6 +471,7 @@ GEM
rack-proxy (>= 0.6.1)
railties (>= 5.2)
semantic_range (>= 2.3.0)
webrick (1.7.0)
websocket (1.2.9)
websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
@ -515,6 +518,7 @@ DEPENDENCIES
pagedown-bootstrap-rails
pg
proforma!
prometheus_exporter
pry-byebug
pry-rails
puma

View File

@ -9,7 +9,7 @@ CodeOcean
## Introduction
CodeOcean is an educational, web-based execution and development environment for practical programming exercises designed for the use in Massive Open Online Courses (MOOCs).
CodeOcean is an educational, web-based execution and development environment for practical programming exercises designed for the use in Massive Open Online Courses (MOOCs).
The programming courses offered on openHPI include practical programming exercises that are provided on a web-based code execution platform called CodeOcean. The platform has many advantages for learners and teachers alike:
@ -22,7 +22,7 @@ CodeOcean is mainly used in the context of MOOCs (such as those offered on openH
## Development Setup
Please refer to the [Local Setup Guide](docs/LOCAL_SETUP.md) for more details.
Please refer to the [Local Setup Guide](docs/LOCAL_SETUP.md) for more details.
### Mandatory Steps
@ -50,3 +50,9 @@ In order to execute code submissions using Docker, source code files are written
- create production configuration files (*database.production.yml*, …)
- customize *config/deploy/production.rb* if you want to deploy using [Capistrano](http://capistranorb.com/)
## Monitoring
- We use a [Prometheus Exporter](https://github.com/discourse/prometheus_exporter) and a [Telegraf Client](https://github.com/jgraichen/telegraf-ruby)
- The Telegraf client collects the data from the Prometheus endpoint, adds its own datasets and forwards them to an InfluxDB
- The Prometheus Exporter must be started separately **before** running the Rails server via `bundle exec prometheus_exporter`
- The InfluxDB data can be visualized using Grafana, for example. There is also an adapted [dashboard](docs/grafana/prometheus_exporter_grafana_dashboard.json) for this purpose

17
Vagrantfile vendored
View File

@ -1,14 +1,19 @@
# frozen_string_literal: true
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.box = "ubuntu/focal64"
config.vm.provider "virtualbox" do |v|
config.vm.box = 'ubuntu/focal64'
config.vm.provider 'virtualbox' do |v|
v.memory = 4096
v.cpus = 4
end
config.vm.network "forwarded_port", host_ip: "127.0.0.1", host: 3000, guest: 3000
config.vm.synced_folder ".", "/home/vagrant/codeocean"
config.vm.synced_folder "../dockercontainerpool", "/home/vagrant/dockercontainerpool"
config.vm.provision "shell", path: "provision/provision.vagrant.sh", privileged: false
config.vm.network 'forwarded_port',
host_ip: ENV['LISTEN_ADDRESS'] || '127.0.0.1',
host: 3000,
guest: 3000
config.vm.synced_folder '.', '/home/vagrant/codeocean'
config.vm.synced_folder '../dockercontainerpool', '/home/vagrant/dockercontainerpool'
config.vm.provision 'shell', path: 'provision/provision.vagrant.sh', privileged: false
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
class Comment < ApplicationRecord
# inherit the creation module: encapsulates that this is a polymorphic user, offers some aliases and makes sure that all necessary attributes are set.
include Creation
@ -8,4 +10,12 @@ class Comment < ApplicationRecord
belongs_to :file, class_name: 'CodeOcean::File'
belongs_to :user, polymorphic: true
# after_save :trigger_rfc_action_cable_from_comment
def request_for_comment
RequestForComment.find_by(submission_id: file.context.id)
end
def only_comment_for_rfc?
request_for_comment.comments.one?
end
end

View File

@ -1,29 +1,27 @@
# frozen_string_literal: true
class RequestForComment < ApplicationRecord
include Creation
include ActionCableHelper
# SOLVED: The author explicitly marked the RfC as solved.
# SOFT_SOLVED: The author did not mark the RfC as solved but reached the maximum score in the corresponding exercise at any time.
# ONGOING: The author did not mark the RfC as solved and did not reach the maximum score in the corresponding exercise yet.
STATE = [SOLVED = :solved, SOFT_SOLVED = :soft_solved, ONGOING = :unsolved].freeze
belongs_to :submission
belongs_to :exercise
belongs_to :file, class_name: 'CodeOcean::File'
has_many :comments, through: :submission
has_many :subscriptions
has_many :subscriptions, dependent: :destroy
scope :unsolved, -> { where(solved: [false, nil]) }
scope :in_range, -> (from, to) { where(created_at: from..to) }
scope :in_range, ->(from, to) { where(created_at: from..to) }
scope :with_comments, -> { select { |rfc| rfc.comments.any? } }
# after_save :trigger_rfc_action_cable
def self.last_per_user(n = 5)
from("(#{row_number_user_sql}) as request_for_comments")
.where("row_number <= ?", n)
.group('request_for_comments.id, request_for_comments.user_id, request_for_comments.user_type,
request_for_comments.exercise_id, request_for_comments.file_id, request_for_comments.question,
request_for_comments.created_at, request_for_comments.updated_at, request_for_comments.solved,
request_for_comments.full_score_reached, request_for_comments.submission_id, request_for_comments.row_number')
# ugly, but necessary
end
# not used right now, finds the last submission for the respective user and exercise.
# might be helpful to check whether the exercise has been solved in the meantime.
def last_submission
@ -46,27 +44,64 @@ class RequestForComment < ApplicationRecord
end
def comments_count
submission.files.map { |file| file.comments.size}.sum
submission.files.map { |file| file.comments.size }.sum
end
def commenters
comments.map(&:user).uniq
end
def self.with_last_activity
self.joins('join "submissions" s on s.id = request_for_comments.submission_id
def comments?
comments.any?
end
def to_s
"RFC-#{id}"
end
def current_state
state(solved, full_score_reached)
end
def old_state
state(solved_before_last_save, full_score_reached_before_last_save)
end
private
def state(solved, full_score_reached)
if solved
SOLVED
elsif full_score_reached
SOFT_SOLVED
else
ONGOING
end
end
class << self
def with_last_activity
joins('join "submissions" s on s.id = request_for_comments.submission_id
left outer join "files" f on f.context_id = s.id
left outer join "comments" c on c.file_id = f.id')
.group('request_for_comments.id')
.select('request_for_comments.*, max(c.updated_at) as last_comment')
end
end
def to_s
"RFC-" + self.id.to_s
end
def last_per_user(count = 5)
from("(#{row_number_user_sql}) as request_for_comments")
.where('row_number <= ?', count)
.group('request_for_comments.id, request_for_comments.user_id, request_for_comments.user_type,
request_for_comments.exercise_id, request_for_comments.file_id, request_for_comments.question,
request_for_comments.created_at, request_for_comments.updated_at, request_for_comments.solved,
request_for_comments.full_score_reached, request_for_comments.submission_id, request_for_comments.row_number')
# ugly, but necessary
end
private
def self.row_number_user_sql
select("id, user_id, user_type, exercise_id, file_id, question, created_at, updated_at, solved, full_score_reached, submission_id, row_number() OVER (PARTITION BY user_id, user_type ORDER BY created_at DESC) as row_number").to_sql
def row_number_user_sql
select('id, user_id, user_type, exercise_id, file_id, question, created_at, updated_at, solved, full_score_reached, submission_id, row_number() OVER (PARTITION BY user_id, user_type ORDER BY created_at DESC) as row_number').to_sql
end
end
end

View File

@ -7,3 +7,5 @@ test:
enabled: false
codeocean_events:
enabled: false
prometheus_exporter:
enabled: false

View File

@ -19,9 +19,15 @@ development:
codeharbor:
enabled: true
url: https://codeharbor.openhpi.de
prometheus_exporter:
enabled: false
production:
<<: *default
prometheus_exporter:
enabled: true
test:
<<: *default
prometheus_exporter:
enabled: false

View File

@ -1,24 +1,26 @@
# frozen_string_literal: true
Rails.application.configure do
# Verifies that versions and hashed value of the package contents in the project's package.json
config.webpacker.check_yarn_integrity = true
# Settings specified here will take precedence over those in config/application.rb.
config.web_console.whitelisted_ips = '192.168.0.0/16'
config.web_console.whitelisted_ips = '192.168.0.0/16'
# In the development environment your application's code is reloaded on
# every request. This slows down response time but is perfect for development
# since you don't have to restart the web server when you make code changes.
config.cache_classes = false
# Do not eager load code on boot.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
# Show full error reports.
config.consider_all_requests_local = true
# Enable/disable caching. By default caching is disabled.
# Run rails dev:cache to toggle caching.
if Rails.root.join('tmp', 'caching-dev.txt').exist?
if Rails.root.join('tmp/caching-dev.txt').exist?
config.action_controller.perform_caching = true
config.cache_store = :memory_store

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Verifies that versions and hashed value of the package contents in the project's package.json
config.webpacker.check_yarn_integrity = false
@ -10,6 +12,7 @@ Rails.application.configure do
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
# Eager load is also required for the prometheus exporter
config.eager_load = true
# Full error reports are disabled and caching is turned on.
@ -90,8 +93,8 @@ Rails.application.configure do
# require 'syslog/logger'
# config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
if ENV["RAILS_LOG_TO_STDOUT"].present?
logger = ActiveSupport::Logger.new(STDOUT)
if ENV['RAILS_LOG_TO_STDOUT'].present?
logger = ActiveSupport::Logger.new($stdout)
logger.formatter = config.log_formatter
config.logger = ActiveSupport::TaggedLogging.new(logger)
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
@ -11,9 +13,10 @@ Rails.application.configure do
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
#enable web console in staging
# enable web console in staging
config.web_console.development_only = false
# Show full error reports and disable caching.

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
@ -10,7 +12,8 @@ Rails.application.configure do
# Do not eager load code on boot. This avoids loading your whole application
# just for the purpose of running a single test. If you are using a tool that
# preloads Rails for running tests, you may have to set it to true.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
# Configure public file server for tests with Cache-Control for performance.
config.public_file_server.enabled = true
@ -47,7 +50,7 @@ Rails.application.configure do
# Raises error for missing translations
# config.action_view.raise_on_missing_translations = true
#config.logger = Logger.new(STDOUT)
# config.logger = Logger.new($stdout)
# Set log level
#config.log_level = :DEBUG
# config.log_level = :DEBUG
end

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
return unless CodeOcean::Config.new(:code_ocean).read[:prometheus_exporter][:enabled]
return if Rake.application.top_level_tasks.to_s.include?('db:')
# Add metric callbacks to all models
ApplicationRecord.include Prometheus::Record
# Initialize the counters according to the db
Prometheus::Controller.initialize_metrics

View File

@ -1,6 +1,6 @@
describe 'Exercise' do
it "outputs 'Hello World" do
expect(STDOUT).to receive(:puts).with('Hello World')
expect($stdout).to receive(:puts).with('Hello World')
require './exercise'
end
end

View File

@ -47,6 +47,14 @@ The default credentials for the administrator are:
- email: `admin@example.org`
- password: `admin`
For exporting metrics, start the prometeus exporter by running
```bash
bundle exec prometheus_exporter
```
in the CodeOcean folder before starting CodeOcean.
## Execution Environments
Every exercise is executed in an execution environment which is based on a docker image. In order to install a new image, have a look at the container of the openHPI team on [DockerHub](https://hub.docker.com/u/openhpi). For example you can add an [image for ruby](https://hub.docker.com/layers/openhpi/co_execenv_ruby/latest/images/sha256-70f597320567678bf8d0146d93fb1bd98457abe61c3b642e832d4e4fbe7f4526) by executing `docker pull openhpi/co_execenv_ruby:latest`.

File diff suppressed because it is too large Load Diff

3
lib/prometheus.rb Normal file
View File

@ -0,0 +1,3 @@
# frozen_string_literal: true
module Prometheus; end

View File

@ -0,0 +1,100 @@
# frozen_string_literal: true
require 'prometheus_exporter/client'
module Prometheus
module Controller
# TODO: currently active users - as Event
# TODO: active_in_last_hour
# TODO: autosaves_per_minute
class << self
def initialize_metrics
register_metrics
initialize_instance_count
initialize_rfc_metrics
end
def register_metrics
prometheus = PrometheusExporter::Client.default
@instance_count = prometheus.register(:gauge, :instance_count, help: 'Instance count')
# counts solved, soft_solved, ongoing
@rfc_count = prometheus.register(:gauge, :rfc_count, help: 'Count of RfCs in each state')
# counts commented
@rfc_commented_count = prometheus.register(:gauge, :rfc_commented_count, help: 'Count of commented RfCs')
end
def initialize_instance_count
ApplicationRecord.descendants.reject(&:abstract_class).each do |each|
@instance_count.observe(each.count, class: each.name)
end
end
def initialize_rfc_metrics
# Initialize rfc metric
@rfc_count.observe(RequestForComment.unsolved.where(full_score_reached: false).count,
state: RequestForComment::ONGOING)
@rfc_count.observe(RequestForComment.unsolved.where(full_score_reached: true).count,
state: RequestForComment::SOFT_SOLVED)
@rfc_count.observe(RequestForComment.where(solved: true).count,
state: RequestForComment::SOLVED)
# count of rfcs with comments
@rfc_commented_count.observe(RequestForComment.with_comments.count)
end
def update_notification(object)
Rails.logger.debug("Prometheus metric updated for #{object.class.name}")
case object
when RequestForComment
update_rfc(object)
end
end
def create_notification(object)
@instance_count.increment(class: object.class.name)
Rails.logger.debug("Prometheus instance count increased for #{object.class.name}")
case object
when RequestForComment
create_rfc(object)
when Comment
create_comment(object)
end
end
def destroy_notification(object)
@instance_count.decrement(class: object.class.name)
Rails.logger.debug("Prometheus instance count decreased for #{object.class.name}")
case object
when Comment
destroy_comment(object)
end
end
def create_rfc(rfc)
@rfc_count.increment(state: rfc.current_state)
end
def update_rfc(rfc)
@rfc_count.decrement(state: rfc.old_state)
# If the metrics are scraped when the execution is exactly at the place of this comment,
# the old state is already decremented while the new state is not yet incremented and
# the metric is therefore inconsistent. As this is only a temporarily off by one error
# in the metric and a solution (e.g. a mutex) would be complex, this is acceptable.
@rfc_count.increment(state: rfc.current_state)
end
def create_comment(comment)
@rfc_commented_count.increment if comment.only_comment_for_rfc?
end
def destroy_comment(comment)
@rfc_commented_count.decrement unless comment.request_for_comment.comments?
end
end
end
end

27
lib/prometheus/record.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Prometheus
module Record
extend ActiveSupport::Concern
included do
after_create_commit :create_notification
after_destroy_commit :destroy_notification
after_update_commit :update_notification
end
private
def create_notification
Prometheus::Controller.create_notification self
end
def destroy_notification
Prometheus::Controller.destroy_notification self
end
def update_notification
Prometheus::Controller.update_notification self
end
end
end

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
FactoryBot.define do
factory :rfc, class: RequestForComment do
factory :rfc, class: 'RequestForComment' do
association :user, factory: :external_user
association :submission
association :exercise, factory: :dummy
@ -7,5 +9,12 @@ FactoryBot.define do
sequence :question do |n|
"test question #{n}"
end
factory :rfc_with_comment, class: 'RequestForComment' do
after(:create) do |rfc|
rfc.file = rfc.submission.files.first
Comment.create(file: rfc.file, user: rfc.user, text: "comment for rfc #{rfc.question}")
end
end
end
end

View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require 'rails_helper'
describe Prometheus::Controller do
def stub_metrics
%i[increment decrement observe].each do |method|
%i[@instance_count @rfc_count @rfc_commented_count].each do |metric|
allow(described_class.instance_variable_get(metric)).to receive(method)
end
end
end
before do
ApplicationRecord.include Prometheus::Record
described_class.initialize_metrics
stub_metrics
end
describe 'instance count' do
it 'initializes the metrics with the current database entries' do
FactoryBot.create_list(:proxy_exercise, 3)
described_class.register_metrics
stub_metrics
described_class.initialize_instance_count
expect(described_class.instance_variable_get(:@instance_count)).to(have_received(:observe).with(ProxyExercise.count, class: ProxyExercise.name).once)
end
it 'gets notified when an object is created' do
allow(described_class).to receive(:create_notification)
proxy_exercise = FactoryBot.create(:proxy_exercise)
expect(described_class).to have_received(:create_notification).with(proxy_exercise).once
end
it 'gets notified when an object is destroyed' do
allow(described_class).to receive(:destroy_notification)
proxy_exercise = FactoryBot.create(:proxy_exercise).destroy
expect(described_class).to have_received(:destroy_notification).with(proxy_exercise).once
end
it 'increments gauge when creating a new instance' do
FactoryBot.create(:proxy_exercise)
expect(described_class.instance_variable_get(:@instance_count)).to(
have_received(:increment).with(class: ProxyExercise.name).once
)
end
it 'decrements gauge when deleting an object' do
FactoryBot.create(:proxy_exercise).destroy
expect(described_class.instance_variable_get(:@instance_count)).to(
have_received(:decrement).with(class: ProxyExercise.name).once
)
end
end
describe 'rfc count' do
context 'when initializing an rfc' do
it 'updates rfc count when creating an ongoing rfc' do
FactoryBot.create(:rfc)
expect(described_class.instance_variable_get(:@rfc_count)).to(
have_received(:increment).with(state: RequestForComment::ONGOING).once
)
end
end
context 'when changing the state of an rfc' do
let(:rfc) { FactoryBot.create(:rfc) }
it 'updates rfc count when soft-solving an rfc' do
rfc.full_score_reached = true
rfc.save
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:increment).with(state: RequestForComment::SOFT_SOLVED).once)
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:decrement).with(state: RequestForComment::ONGOING).once)
end
it 'updates rfc count when solving an rfc' do
rfc.solved = true
rfc.save
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:increment).with(state: RequestForComment::SOLVED).once)
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:decrement).with(state: RequestForComment::ONGOING).once)
end
end
context 'when commenting an rfc' do
it 'updates comment metric when commenting an rfc' do
FactoryBot.create(:rfc_with_comment)
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment)
end
it 'does not update comment metric when commenting an rfc that already has a comment' do
rfc = FactoryBot.create(:rfc_with_comment)
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment).once
Comment.create(file: rfc.file, user: rfc.user, text: "comment a for rfc #{rfc.question}")
Comment.create(file: rfc.file, user: rfc.user, text: "comment b for rfc #{rfc.question}")
# instance count has only been updated for the creation of the commented rfc and not for additional comments
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment).once
end
end
end
end

View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
require 'rails_helper'
describe RequestForComment do
let!(:rfc) { FactoryBot.create(:rfc) }
describe 'scope with_comments' do
let!(:rfc2) { FactoryBot.create(:rfc_with_comment) }
it 'includes all RfCs with comments' do
expect(described_class.with_comments).to include(rfc2)
end
it 'does not include any RfC without a comment' do
expect(described_class.with_comments).not_to include(rfc)
end
end
end

View File

@ -1,7 +1,10 @@
# frozen_string_literal: true
# This file is copied to spec/ when you run 'rails generate rspec:install'
ENV['RAILS_ENV'] ||= 'test'
require 'spec_helper'
require File.expand_path('../../config/environment', __FILE__)
require 'support/prometheus_client_stub'
require File.expand_path('../config/environment', __dir__)
require 'rspec/rails'
require 'pundit/rspec'
@ -12,7 +15,7 @@ require 'pundit/rspec'
# run twice. It is recommended that you do not name files matching this glob to
# end with _spec.rb. You can configure this pattern with with the --pattern
# option on the command line or in ~/.rspec, .rspec or `.rspec-local`.
Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
Dir[Rails.root.join('spec/support/**/*.rb')].sort.each { |f| require f }
# Checks for pending migrations before tests are run.
# If you are not using ActiveRecord, you can remove this line.

View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
require 'prometheus_exporter/client'
require 'rails_helper'
module Prometheus
# A stub to disable server functionality in the specs and stub all registered metrics
module StubClient
def send(str)
# Do nothing
end
end
PrometheusExporter::Client.prepend StubClient
end