#46 Add Prometheus exporter

This commit is contained in:
Tobias Kantusch
2021-02-18 18:02:56 +01:00
committed by Sebastian Serth
parent 39fcd255f9
commit 44b32b6f6a
26 changed files with 2121 additions and 45 deletions

View File

@ -20,3 +20,9 @@ Style/Documentation:
Enabled: false
Layout/SpaceInsideHashLiteralBraces:
EnforcedStyle: no_space
RSpec/MultipleExpectations:
Max: 2
Metrics/BlockLength:
Exclude:
- 'spec/**/*.rb'
- 'config/environments/*.rb'

View File

@ -21,6 +21,7 @@ gem 'nokogiri'
gem 'pagedown-bootstrap-rails'
gem 'pg'
gem 'proforma', git: 'https://github.com/openHPI/proforma.git', tag: 'v0.5'
gem 'prometheus_exporter'
gem 'pry-byebug'
gem 'puma'
gem 'pundit'

View File

@ -248,6 +248,8 @@ GEM
ast (~> 2.4.1)
path_expander (1.1.0)
pg (1.2.3)
prometheus_exporter (0.7.0)
webrick
pry (0.13.1)
coderay (~> 1.1)
method_source (~> 1.0)
@ -469,6 +471,7 @@ GEM
rack-proxy (>= 0.6.1)
railties (>= 5.2)
semantic_range (>= 2.3.0)
webrick (1.7.0)
websocket (1.2.9)
websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
@ -515,6 +518,7 @@ DEPENDENCIES
pagedown-bootstrap-rails
pg
proforma!
prometheus_exporter
pry-byebug
pry-rails
puma

View File

@ -50,3 +50,9 @@ In order to execute code submissions using Docker, source code files are written
- create production configuration files (*database.production.yml*, …)
- customize *config/deploy/production.rb* if you want to deploy using [Capistrano](http://capistranorb.com/)
## Monitoring
- We use a [Prometheus Exporter](https://github.com/discourse/prometheus_exporter) and a [Telegraf Client](https://github.com/jgraichen/telegraf-ruby)
- The Telegraf client collects the data from the Prometheus endpoint, adds its own datasets and forwards them to an InfluxDB
- The Prometheus Exporter must be started separately **before** running the Rails server via `bundle exec prometheus_exporter`
- The InfluxDB data can be visualized using Grafana, for example. There is also an adapted [dashboard](docs/grafana/prometheus_exporter_grafana_dashboard.json) for this purpose

17
Vagrantfile vendored
View File

@ -1,14 +1,19 @@
# frozen_string_literal: true
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.box = "ubuntu/focal64"
config.vm.provider "virtualbox" do |v|
config.vm.box = 'ubuntu/focal64'
config.vm.provider 'virtualbox' do |v|
v.memory = 4096
v.cpus = 4
end
config.vm.network "forwarded_port", host_ip: "127.0.0.1", host: 3000, guest: 3000
config.vm.synced_folder ".", "/home/vagrant/codeocean"
config.vm.synced_folder "../dockercontainerpool", "/home/vagrant/dockercontainerpool"
config.vm.provision "shell", path: "provision/provision.vagrant.sh", privileged: false
config.vm.network 'forwarded_port',
host_ip: ENV['LISTEN_ADDRESS'] || '127.0.0.1',
host: 3000,
guest: 3000
config.vm.synced_folder '.', '/home/vagrant/codeocean'
config.vm.synced_folder '../dockercontainerpool', '/home/vagrant/dockercontainerpool'
config.vm.provision 'shell', path: 'provision/provision.vagrant.sh', privileged: false
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
class Comment < ApplicationRecord
# inherit the creation module: encapsulates that this is a polymorphic user, offers some aliases and makes sure that all necessary attributes are set.
include Creation
@ -8,4 +10,12 @@ class Comment < ApplicationRecord
belongs_to :file, class_name: 'CodeOcean::File'
belongs_to :user, polymorphic: true
# after_save :trigger_rfc_action_cable_from_comment
def request_for_comment
RequestForComment.find_by(submission_id: file.context.id)
end
def only_comment_for_rfc?
request_for_comment.comments.one?
end
end

View File

@ -1,29 +1,27 @@
# frozen_string_literal: true
class RequestForComment < ApplicationRecord
include Creation
include ActionCableHelper
# SOLVED: The author explicitly marked the RfC as solved.
# SOFT_SOLVED: The author did not mark the RfC as solved but reached the maximum score in the corresponding exercise at any time.
# ONGOING: The author did not mark the RfC as solved and did not reach the maximum score in the corresponding exercise yet.
STATE = [SOLVED = :solved, SOFT_SOLVED = :soft_solved, ONGOING = :unsolved].freeze
belongs_to :submission
belongs_to :exercise
belongs_to :file, class_name: 'CodeOcean::File'
has_many :comments, through: :submission
has_many :subscriptions
has_many :subscriptions, dependent: :destroy
scope :unsolved, -> { where(solved: [false, nil]) }
scope :in_range, ->(from, to) { where(created_at: from..to) }
scope :with_comments, -> { select { |rfc| rfc.comments.any? } }
# after_save :trigger_rfc_action_cable
def self.last_per_user(n = 5)
from("(#{row_number_user_sql}) as request_for_comments")
.where("row_number <= ?", n)
.group('request_for_comments.id, request_for_comments.user_id, request_for_comments.user_type,
request_for_comments.exercise_id, request_for_comments.file_id, request_for_comments.question,
request_for_comments.created_at, request_for_comments.updated_at, request_for_comments.solved,
request_for_comments.full_score_reached, request_for_comments.submission_id, request_for_comments.row_number')
# ugly, but necessary
end
# not used right now, finds the last submission for the respective user and exercise.
# might be helpful to check whether the exercise has been solved in the meantime.
def last_submission
@ -53,20 +51,57 @@ class RequestForComment < ApplicationRecord
comments.map(&:user).uniq
end
def self.with_last_activity
self.joins('join "submissions" s on s.id = request_for_comments.submission_id
def comments?
comments.any?
end
def to_s
"RFC-#{id}"
end
def current_state
state(solved, full_score_reached)
end
def old_state
state(solved_before_last_save, full_score_reached_before_last_save)
end
private
def state(solved, full_score_reached)
if solved
SOLVED
elsif full_score_reached
SOFT_SOLVED
else
ONGOING
end
end
class << self
def with_last_activity
joins('join "submissions" s on s.id = request_for_comments.submission_id
left outer join "files" f on f.context_id = s.id
left outer join "comments" c on c.file_id = f.id')
.group('request_for_comments.id')
.select('request_for_comments.*, max(c.updated_at) as last_comment')
end
def to_s
"RFC-" + self.id.to_s
def last_per_user(count = 5)
from("(#{row_number_user_sql}) as request_for_comments")
.where('row_number <= ?', count)
.group('request_for_comments.id, request_for_comments.user_id, request_for_comments.user_type,
request_for_comments.exercise_id, request_for_comments.file_id, request_for_comments.question,
request_for_comments.created_at, request_for_comments.updated_at, request_for_comments.solved,
request_for_comments.full_score_reached, request_for_comments.submission_id, request_for_comments.row_number')
# ugly, but necessary
end
private
def self.row_number_user_sql
select("id, user_id, user_type, exercise_id, file_id, question, created_at, updated_at, solved, full_score_reached, submission_id, row_number() OVER (PARTITION BY user_id, user_type ORDER BY created_at DESC) as row_number").to_sql
def row_number_user_sql
select('id, user_id, user_type, exercise_id, file_id, question, created_at, updated_at, solved, full_score_reached, submission_id, row_number() OVER (PARTITION BY user_id, user_type ORDER BY created_at DESC) as row_number').to_sql
end
end
end

View File

@ -7,3 +7,5 @@ test:
enabled: false
codeocean_events:
enabled: false
prometheus_exporter:
enabled: false

View File

@ -19,9 +19,15 @@ development:
codeharbor:
enabled: true
url: https://codeharbor.openhpi.de
prometheus_exporter:
enabled: false
production:
<<: *default
prometheus_exporter:
enabled: true
test:
<<: *default
prometheus_exporter:
enabled: false

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Verifies that versions and hashed value of the package contents in the project's package.json
config.webpacker.check_yarn_integrity = true
@ -10,15 +12,15 @@ Rails.application.configure do
# since you don't have to restart the web server when you make code changes.
config.cache_classes = false
# Do not eager load code on boot.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
# Show full error reports.
config.consider_all_requests_local = true
# Enable/disable caching. By default caching is disabled.
# Run rails dev:cache to toggle caching.
if Rails.root.join('tmp', 'caching-dev.txt').exist?
if Rails.root.join('tmp/caching-dev.txt').exist?
config.action_controller.perform_caching = true
config.cache_store = :memory_store

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Verifies that versions and hashed value of the package contents in the project's package.json
config.webpacker.check_yarn_integrity = false
@ -10,6 +12,7 @@ Rails.application.configure do
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
# Eager load is also required for the prometheus exporter
config.eager_load = true
# Full error reports are disabled and caching is turned on.
@ -90,8 +93,8 @@ Rails.application.configure do
# require 'syslog/logger'
# config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
if ENV["RAILS_LOG_TO_STDOUT"].present?
logger = ActiveSupport::Logger.new(STDOUT)
if ENV['RAILS_LOG_TO_STDOUT'].present?
logger = ActiveSupport::Logger.new($stdout)
logger.formatter = config.log_formatter
config.logger = ActiveSupport::TaggedLogging.new(logger)
end

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
@ -11,7 +13,8 @@ Rails.application.configure do
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
# enable web console in staging
config.web_console.development_only = false

View File

@ -1,3 +1,5 @@
# frozen_string_literal: true
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
@ -10,7 +12,8 @@ Rails.application.configure do
# Do not eager load code on boot. This avoids loading your whole application
# just for the purpose of running a single test. If you are using a tool that
# preloads Rails for running tests, you may have to set it to true.
config.eager_load = false
# Eager load code for prometheus exporter
config.eager_load = true
# Configure public file server for tests with Cache-Control for performance.
config.public_file_server.enabled = true
@ -47,7 +50,7 @@ Rails.application.configure do
# Raises error for missing translations
# config.action_view.raise_on_missing_translations = true
#config.logger = Logger.new(STDOUT)
# config.logger = Logger.new($stdout)
# Set log level
# config.log_level = :DEBUG
end

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
return unless CodeOcean::Config.new(:code_ocean).read[:prometheus_exporter][:enabled]
return if Rake.application.top_level_tasks.to_s.include?('db:')
# Add metric callbacks to all models
ApplicationRecord.include Prometheus::Record
# Initialize the counters according to the db
Prometheus::Controller.initialize_metrics

View File

@ -1,6 +1,6 @@
describe 'Exercise' do
it "outputs 'Hello World" do
expect(STDOUT).to receive(:puts).with('Hello World')
expect($stdout).to receive(:puts).with('Hello World')
require './exercise'
end
end

View File

@ -47,6 +47,14 @@ The default credentials for the administrator are:
- email: `admin@example.org`
- password: `admin`
For exporting metrics, start the prometeus exporter by running
```bash
bundle exec prometheus_exporter
```
in the CodeOcean folder before starting CodeOcean.
## Execution Environments
Every exercise is executed in an execution environment which is based on a docker image. In order to install a new image, have a look at the container of the openHPI team on [DockerHub](https://hub.docker.com/u/openhpi). For example you can add an [image for ruby](https://hub.docker.com/layers/openhpi/co_execenv_ruby/latest/images/sha256-70f597320567678bf8d0146d93fb1bd98457abe61c3b642e832d4e4fbe7f4526) by executing `docker pull openhpi/co_execenv_ruby:latest`.

File diff suppressed because it is too large Load Diff

3
lib/prometheus.rb Normal file
View File

@ -0,0 +1,3 @@
# frozen_string_literal: true
module Prometheus; end

View File

@ -0,0 +1,100 @@
# frozen_string_literal: true
require 'prometheus_exporter/client'
module Prometheus
module Controller
# TODO: currently active users - as Event
# TODO: active_in_last_hour
# TODO: autosaves_per_minute
class << self
def initialize_metrics
register_metrics
initialize_instance_count
initialize_rfc_metrics
end
def register_metrics
prometheus = PrometheusExporter::Client.default
@instance_count = prometheus.register(:gauge, :instance_count, help: 'Instance count')
# counts solved, soft_solved, ongoing
@rfc_count = prometheus.register(:gauge, :rfc_count, help: 'Count of RfCs in each state')
# counts commented
@rfc_commented_count = prometheus.register(:gauge, :rfc_commented_count, help: 'Count of commented RfCs')
end
def initialize_instance_count
ApplicationRecord.descendants.reject(&:abstract_class).each do |each|
@instance_count.observe(each.count, class: each.name)
end
end
def initialize_rfc_metrics
# Initialize rfc metric
@rfc_count.observe(RequestForComment.unsolved.where(full_score_reached: false).count,
state: RequestForComment::ONGOING)
@rfc_count.observe(RequestForComment.unsolved.where(full_score_reached: true).count,
state: RequestForComment::SOFT_SOLVED)
@rfc_count.observe(RequestForComment.where(solved: true).count,
state: RequestForComment::SOLVED)
# count of rfcs with comments
@rfc_commented_count.observe(RequestForComment.with_comments.count)
end
def update_notification(object)
Rails.logger.debug("Prometheus metric updated for #{object.class.name}")
case object
when RequestForComment
update_rfc(object)
end
end
def create_notification(object)
@instance_count.increment(class: object.class.name)
Rails.logger.debug("Prometheus instance count increased for #{object.class.name}")
case object
when RequestForComment
create_rfc(object)
when Comment
create_comment(object)
end
end
def destroy_notification(object)
@instance_count.decrement(class: object.class.name)
Rails.logger.debug("Prometheus instance count decreased for #{object.class.name}")
case object
when Comment
destroy_comment(object)
end
end
def create_rfc(rfc)
@rfc_count.increment(state: rfc.current_state)
end
def update_rfc(rfc)
@rfc_count.decrement(state: rfc.old_state)
# If the metrics are scraped when the execution is exactly at the place of this comment,
# the old state is already decremented while the new state is not yet incremented and
# the metric is therefore inconsistent. As this is only a temporarily off by one error
# in the metric and a solution (e.g. a mutex) would be complex, this is acceptable.
@rfc_count.increment(state: rfc.current_state)
end
def create_comment(comment)
@rfc_commented_count.increment if comment.only_comment_for_rfc?
end
def destroy_comment(comment)
@rfc_commented_count.decrement unless comment.request_for_comment.comments?
end
end
end
end

27
lib/prometheus/record.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Prometheus
module Record
extend ActiveSupport::Concern
included do
after_create_commit :create_notification
after_destroy_commit :destroy_notification
after_update_commit :update_notification
end
private
def create_notification
Prometheus::Controller.create_notification self
end
def destroy_notification
Prometheus::Controller.destroy_notification self
end
def update_notification
Prometheus::Controller.update_notification self
end
end
end

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
FactoryBot.define do
factory :rfc, class: RequestForComment do
factory :rfc, class: 'RequestForComment' do
association :user, factory: :external_user
association :submission
association :exercise, factory: :dummy
@ -7,5 +9,12 @@ FactoryBot.define do
sequence :question do |n|
"test question #{n}"
end
factory :rfc_with_comment, class: 'RequestForComment' do
after(:create) do |rfc|
rfc.file = rfc.submission.files.first
Comment.create(file: rfc.file, user: rfc.user, text: "comment for rfc #{rfc.question}")
end
end
end
end

View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require 'rails_helper'
describe Prometheus::Controller do
def stub_metrics
%i[increment decrement observe].each do |method|
%i[@instance_count @rfc_count @rfc_commented_count].each do |metric|
allow(described_class.instance_variable_get(metric)).to receive(method)
end
end
end
before do
ApplicationRecord.include Prometheus::Record
described_class.initialize_metrics
stub_metrics
end
describe 'instance count' do
it 'initializes the metrics with the current database entries' do
FactoryBot.create_list(:proxy_exercise, 3)
described_class.register_metrics
stub_metrics
described_class.initialize_instance_count
expect(described_class.instance_variable_get(:@instance_count)).to(have_received(:observe).with(ProxyExercise.count, class: ProxyExercise.name).once)
end
it 'gets notified when an object is created' do
allow(described_class).to receive(:create_notification)
proxy_exercise = FactoryBot.create(:proxy_exercise)
expect(described_class).to have_received(:create_notification).with(proxy_exercise).once
end
it 'gets notified when an object is destroyed' do
allow(described_class).to receive(:destroy_notification)
proxy_exercise = FactoryBot.create(:proxy_exercise).destroy
expect(described_class).to have_received(:destroy_notification).with(proxy_exercise).once
end
it 'increments gauge when creating a new instance' do
FactoryBot.create(:proxy_exercise)
expect(described_class.instance_variable_get(:@instance_count)).to(
have_received(:increment).with(class: ProxyExercise.name).once
)
end
it 'decrements gauge when deleting an object' do
FactoryBot.create(:proxy_exercise).destroy
expect(described_class.instance_variable_get(:@instance_count)).to(
have_received(:decrement).with(class: ProxyExercise.name).once
)
end
end
describe 'rfc count' do
context 'when initializing an rfc' do
it 'updates rfc count when creating an ongoing rfc' do
FactoryBot.create(:rfc)
expect(described_class.instance_variable_get(:@rfc_count)).to(
have_received(:increment).with(state: RequestForComment::ONGOING).once
)
end
end
context 'when changing the state of an rfc' do
let(:rfc) { FactoryBot.create(:rfc) }
it 'updates rfc count when soft-solving an rfc' do
rfc.full_score_reached = true
rfc.save
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:increment).with(state: RequestForComment::SOFT_SOLVED).once)
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:decrement).with(state: RequestForComment::ONGOING).once)
end
it 'updates rfc count when solving an rfc' do
rfc.solved = true
rfc.save
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:increment).with(state: RequestForComment::SOLVED).once)
expect(described_class.instance_variable_get(:@rfc_count)).to(have_received(:decrement).with(state: RequestForComment::ONGOING).once)
end
end
context 'when commenting an rfc' do
it 'updates comment metric when commenting an rfc' do
FactoryBot.create(:rfc_with_comment)
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment)
end
it 'does not update comment metric when commenting an rfc that already has a comment' do
rfc = FactoryBot.create(:rfc_with_comment)
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment).once
Comment.create(file: rfc.file, user: rfc.user, text: "comment a for rfc #{rfc.question}")
Comment.create(file: rfc.file, user: rfc.user, text: "comment b for rfc #{rfc.question}")
# instance count has only been updated for the creation of the commented rfc and not for additional comments
expect(described_class.instance_variable_get(:@rfc_commented_count)).to have_received(:increment).once
end
end
end
end

View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
require 'rails_helper'
describe RequestForComment do
let!(:rfc) { FactoryBot.create(:rfc) }
describe 'scope with_comments' do
let!(:rfc2) { FactoryBot.create(:rfc_with_comment) }
it 'includes all RfCs with comments' do
expect(described_class.with_comments).to include(rfc2)
end
it 'does not include any RfC without a comment' do
expect(described_class.with_comments).not_to include(rfc)
end
end
end

View File

@ -1,7 +1,10 @@
# frozen_string_literal: true
# This file is copied to spec/ when you run 'rails generate rspec:install'
ENV['RAILS_ENV'] ||= 'test'
require 'spec_helper'
require File.expand_path('../../config/environment', __FILE__)
require 'support/prometheus_client_stub'
require File.expand_path('../config/environment', __dir__)
require 'rspec/rails'
require 'pundit/rspec'
@ -12,7 +15,7 @@ require 'pundit/rspec'
# run twice. It is recommended that you do not name files matching this glob to
# end with _spec.rb. You can configure this pattern with with the --pattern
# option on the command line or in ~/.rspec, .rspec or `.rspec-local`.
Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
Dir[Rails.root.join('spec/support/**/*.rb')].sort.each { |f| require f }
# Checks for pending migrations before tests are run.
# If you are not using ActiveRecord, you can remove this line.

View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
require 'prometheus_exporter/client'
require 'rails_helper'
module Prometheus
# A stub to disable server functionality in the specs and stub all registered metrics
module StubClient
def send(str)
# Do nothing
end
end
PrometheusExporter::Client.prepend StubClient
end