Active record collector (#94)

Active record collector (#94)

  • adds activerecord collector

  • fails for old rails versions

  • adds documentation

  • adds test cases

  • fixes namespace issues

  • adds config labels and custom labels

  • fixing bug replacing invalid constant

  • adding prefix for config metrics

diff --git a/README.md b/README.md
index ba865db..c70008c 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
   * [Rails integration](#rails-integration)
     * [Per-process stats](#per-process-stats)
     * [Sidekiq metrics](#sidekiq-metrics)
+    * [ActiveRecord Connection Pool Metrics](#activerecord-connection-pool-metrics)
     * [Delayed Job plugin](#delayed-job-plugin)
     * [Hutch metrics](#hutch-message-processing-tracer)
   * [Puma metrics](#puma-metrics)
@@ -175,6 +176,60 @@ Ensure you run the exporter in a monitored background process:
 $ bundle exec prometheus_exporter
 `‍``
 
+#### Activerecord Connection Pool Metrics
+
+This collects activerecord connection pool metrics.
+
+It supports injection of custom labels and the connection config options (`username`, `database`, `host`, `port`) as labels.
+
+For Puma single mode 
+`‍``ruby
+#in puma.rb
+require 'prometheus_exporter/instrumentation'
+PrometheusExporter::Instrumentation::ActiveRecord.start(
+  custom_labels: { type: "puma_single_mode" }, #optional params
+  config_labels: [:database, :host] #optional params
+)
+`‍``
+
+For Puma cluster mode
+
+`‍``ruby
+# in puma.rb
+on_worker_boot do
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "puma_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+`‍``
+
+For Unicorn / Passenger
+
+`‍``ruby
+after_fork do 
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::ActiveRecord.start(
+    custom_labels: { type: "unicorn_worker" }, #optional params
+    config_labels: [:database, :host] #optional params
+  )
+end
+`‍``
+
+For Sidekiq
+`‍``ruby
+Sidekiq.configure_server do |config|
+  config.on :startup do
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::ActiveRecord.start(
+      custom_labels: { type: "sidekiq" }, #optional params
+      config_labels: [:database, :host] #optional params
+    )
+  end
+end
+`‍``
+
 #### Per-process stats
 
 You may also be interested in per-process stats. This collects memory and GC stats:
diff --git a/lib/prometheus_exporter/instrumentation.rb b/lib/prometheus_exporter/instrumentation.rb
index 1e3a384..a38da40 100644
--- a/lib/prometheus_exporter/instrumentation.rb
+++ b/lib/prometheus_exporter/instrumentation.rb
@@ -8,3 +8,4 @@ require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
+require_relative "instrumentation/active_record"
diff --git a/lib/prometheus_exporter/instrumentation/active_record.rb b/lib/prometheus_exporter/instrumentation/active_record.rb
new file mode 100644
index 0000000..de59020
--- /dev/null
+++ b/lib/prometheus_exporter/instrumentation/active_record.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# collects stats from currently running process
+module PrometheusExporter::Instrumentation
+  class ActiveRecord
+    ALLOWED_CONFIG_LABELS = %i(database username host port)
+
+    def self.start(client: nil, frequency: 30, custom_labels: {}, config_labels: [])
+
+      # Not all rails versions support coonection pool stats
+      unless ::ActiveRecord::Base.connection_pool.respond_to?(:stat)
+        STDERR.puts("ActiveRecord connection pool stats not supported in your rails version")
+        return
+      end
+
+      config_labels.map!(&:to_sym)
+      validate_config_labels(config_labels)
+
+      active_record_collector = new(custom_labels, config_labels)
+
+      client ||= PrometheusExporter::Client.default
+
+      stop if @thread
+
+      @thread = Thread.new do
+        while true
+          begin
+            metrics = active_record_collector.collect
+            metrics.each { |metric| client.send_json metric }
+          rescue => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Process Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def self.validate_config_labels(config_labels)
+      return if config_labels.size == 0
+      raise "Invalid Config Labels, available options #{ALLOWED_CONFIG_LABELS}" if (config_labels - ALLOWED_CONFIG_LABELS).size > 0
+    end
+
+    def self.stop
+      if t = @thread
+        t.kill
+        @thread = nil
+      end
+    end
+
+    def initialize(metric_labels, config_labels)
+      @metric_labels = metric_labels
+      @config_labels = config_labels
+      @hostname = nil
+    end
+
+    def hostname
+      @hostname ||=
+        begin
+          `hostname`.strip
+        rescue => e
+          STDERR.puts "Unable to lookup hostname #{e}"
+          "unknown-host"
+        end
+    end
+
+    def collect
+      metrics = []
+      collect_active_record_pool_stats(metrics)
+      metrics
+    end
+
+    def pid
+      @pid = ::Process.pid
+    end
+
+    def collect_active_record_pool_stats(metrics)
+      ObjectSpace.each_object(::ActiveRecord::ConnectionAdapters::ConnectionPool) do |pool|
+        next if pool.connections.nil?
+
+        labels_from_config = pool.spec.config
+          .select { |k, v| @config_labels.include? k }
+          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
+
+        labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
+
+        metric = {
+          pid: pid,
+          type: "active_record",
+          hostname: hostname,
+          metric_labels: labels
+        }
+        metric.merge!(pool.stat)
+        metrics << metric
+      end
+    end
+  end
+end
diff --git a/lib/prometheus_exporter/server.rb b/lib/prometheus_exporter/server.rb
index d0eba6d..51a4518 100644
--- a/lib/prometheus_exporter/server.rb
+++ b/lib/prometheus_exporter/server.rb
@@ -13,3 +13,4 @@ require_relative "server/runner"
 require_relative "server/puma_collector"
 require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
+require_relative "server/active_record_collector"
diff --git a/lib/prometheus_exporter/server/active_record_collector.rb b/lib/prometheus_exporter/server/active_record_collector.rb
new file mode 100644
index 0000000..9cdb9ac
--- /dev/null
+++ b/lib/prometheus_exporter/server/active_record_collector.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ActiveRecordCollector < TypeCollector
+    MAX_ACTIVERECORD_METRIC_AGE = 60
+    ACTIVE_RECORD_GAUGES = {
+      connections: "Total connections in pool",
+      busy: "Connections in use in pool",
+      dead: "Dead connections in pool",
+      idle: "Idle connections in pool",
+      waiting: "Connection requests waiting",
+      size: "Maximum allowed connection pool size"
+    }
+
+    def initialize
+      @active_record_metrics = []
+    end
+
+    def type
+      "active_record"
+    end
+
+    def metrics
+      return [] if @active_record_metrics.length == 0
+
+      metrics = {}
+
+      @active_record_metrics.map do |m|
+        metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+
+        ACTIVE_RECORD_GAUGES.map do |k, help|
+          k = k.to_s
+          if v = m[k]
+            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("active_record_connection_pool_#{k}", help)
+            g.observe(v, metric_key)
+          end
+        end
+      end
+
+      metrics.values
+    end
+
+    def collect(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+
+      @active_record_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
+      end
+
+      @active_record_metrics << obj
+    end
+  end
+end

[... diff too long, it was truncated ...]

GitHub sha: 8fdee66d