FEATURE: mobystash upgrades

approved

#1

FEATURE: mobystash upgrades

This is a pretty extensive amount of changes to mobystash including

  • Shift to prometheus_exporter from the prometheus gem + frankenstein, this is somewhat faster than prom gem

  • Added a gemspec so we have a cleaner deployment story in docker as we can require the dependency locally

  • Moved loggerstash and and logstash_writer into this mobystash repo, this makes it a bit easier to maintain

  • Optimisation of regex_filter so it is a bit faster

diff --git a/Dockerfile b/Dockerfile
index 8837e8e..62c83ba 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,8 @@ FROM ruby:2.6.1-slim-stretch
 MAINTAINER Matt Palmer "matt.palmer@discourse.org"
 
 RUN useradd mobystash -s /bin/bash -m -U --create-home
-COPY Gemfile Gemfile.lock /home/mobystash/
+
+COPY Gemfile Gemfile.lock mobystash.gemspec /home/mobystash/
 
 ARG GIT_REVISION=invalid-build
 ENV MOBYSTASH_GIT_REVISION=$GIT_REVISION
@@ -16,8 +17,9 @@ RUN docker_group="$(getent group 999 | cut -d : -f 1)" \
   && cd /home/mobystash && su -l mobystash -c "bundle install --deployment --without development" \
   && apt-get purge -y --auto-remove build-essential
 
+COPY lib /home/mobystash/lib
+
 COPY bin/* /usr/local/bin/
-COPY lib/ /usr/local/lib/ruby/2.6.0/
 
 EXPOSE 9367
 LABEL org.discourse.service._prom-exp.port=9367 org.discourse.service._prom-exp.instance=mobystash org.discourse.mobystash.disable=yes
diff --git a/Gemfile b/Gemfile
index 5c0c08a..79b6fe8 100644
--- a/Gemfile
+++ b/Gemfile
@@ -2,17 +2,17 @@ ruby '> 2.3', '< 2.7'
 
 source 'https://rubygems.org'
 
+gem 'mobystash', path: "."
+
 gem 'deep_merge'
 gem 'docker-api', '~> 1.33'
 # See https://github.com/swipely/docker-api/issues/409
 gem 'excon', '~> 0.59'
-gem 'frankenstein', '~> 1.1'
-gem 'loggerstash', '~> 0.0.2'
-gem 'logstash_writer', '~> 0.0'
 gem 'murmurhash3', '~> 0.1'
 gem 'rbtrace', '~> 0.4'
 gem 'sigdump', require: 'sigdump/setup'
 gem 'stackprof'
+gem 'prometheus_exporter'
 
 group :development do
   gem 'byebug'
diff --git a/Gemfile.lock b/Gemfile.lock
index 19c95c3..85daa5a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,8 @@
+PATH
+  remote: .
+  specs:
+    mobystash (0.0.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -13,9 +18,6 @@ GEM
     excon (0.62.0)
     ffi (1.10.0)
     formatador (0.2.5)
-    frankenstein (1.2.0)
-      prometheus-client (= 0.8.0)
-      rack (~> 2.0)
     guard (2.15.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
@@ -39,10 +41,6 @@ GEM
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
       ruby_dep (~> 1.2)
-    loggerstash (0.0.8)
-      logstash_writer (~> 0.0)
-    logstash_writer (0.0.10)
-      prometheus-client (= 0.8.0)
     lumberjack (1.0.13)
     method_source (0.9.2)
     msgpack (1.2.6)
@@ -57,16 +55,13 @@ GEM
     parser (2.6.0.0)
       ast (~> 2.4.0)
     powerpack (0.1.2)
-    prometheus-client (0.8.0)
-      quantile (~> 0.2.1)
+    prometheus_exporter (0.4.3)
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     pry-byebug (3.6.0)
       byebug (~> 10.0)
       pry (~> 0.10)
-    quantile (0.2.1)
-    rack (2.0.6)
     rainbow (3.0.0)
     rake (12.3.2)
     rb-fsevent (0.10.3)
@@ -120,12 +115,11 @@ DEPENDENCIES
   deep_merge
   docker-api (~> 1.33)
   excon (~> 0.59)
-  frankenstein (~> 1.1)
   guard-rspec
   guard-rubocop
-  loggerstash (~> 0.0.2)
-  logstash_writer (~> 0.0)
+  mobystash!
   murmurhash3 (~> 0.1)
+  prometheus_exporter
   pry-byebug
   rake (>= 11)
   rbtrace (~> 0.4)
diff --git a/bin/mobystash b/bin/mobystash
index 940ce0a..5bec56a 100755
--- a/bin/mobystash
+++ b/bin/mobystash
@@ -11,10 +11,9 @@ end
 require 'sigdump'
 Sigdump.setup('SIGCONT', '+')
 
-require 'mobystash/system'
 require 'logger'
-require 'loggerstash'
 require 'rbtrace'
+require 'mobystash'
 
 logger = Logger.new($stderr)
 logger.level = Logger.const_get((ENV['MOBYSTASH_LOG_LEVEL'] || "INFO").upcase)
diff --git a/lib/mobystash.rb b/lib/mobystash.rb
new file mode 100644
index 0000000..bab225c
--- /dev/null
+++ b/lib/mobystash.rb
@@ -0,0 +1,13 @@
+require 'prometheus_exporter'
+require 'prometheus_exporter/metric'
+
+require_relative "mobystash/logstash_writer"
+require_relative "mobystash/loggerstash"
+require_relative "mobystash/log_exception"
+require_relative "mobystash/moby_chunk_parser"
+require_relative "mobystash/moby_event_worker"
+require_relative "mobystash/config"
+require_relative "mobystash/container"
+require_relative "mobystash/moby_watcher"
+require_relative "mobystash/sampler"
+require_relative "mobystash/system"
diff --git a/lib/mobystash/config.rb b/lib/mobystash/config.rb
index 646b13c..15688c9 100644
--- a/lib/mobystash/config.rb
+++ b/lib/mobystash/config.rb
@@ -1,9 +1,5 @@
 # frozen_string_literal: true
-
 require 'logger'
-require 'logstash_writer'
-require 'prometheus/client'
-require 'mobystash/sampler'
 
 module Mobystash
   # Encapsulates all common configuration parameters and shared metrics.
@@ -22,7 +18,7 @@ module Mobystash
 
     attr_reader :logger
 
-    attr_reader :metrics_registry
+    attr_reader :metrics
 
     attr_reader :read_event_exception_counter,
                 :log_entries_read_counter,
@@ -47,26 +43,69 @@ module Mobystash
     #
     # @raise [InvalidEnvironmentError] if any problems are detected with the
     #   environment variables found.
-    #
     def initialize(env, logger:)
       @logger = logger
 
       # Even if we're not actually *running* a metrics server, we still need
       # the registry in place, because conditionalising every metrics-related
       # operation on whether metrics are enabled is just... madness.
-      @metrics_registry = Prometheus::Client::Registry.new
       @sampler = Mobystash::Sampler.new(self)
 
       parse_env(env)
 
-      @read_event_exception_counter = @metrics_registry.counter(:mobystash_moby_read_exceptions_total, "Exception counts while attempting to read log entries from the Moby server")
-      @log_entries_read_counter     = @metrics_registry.counter(:mobystash_log_entries_read_total, "How many log entries have been received from Moby")
-      @log_entries_sent_counter     = @metrics_registry.counter(:mobystash_log_entries_sent_total, "How many log entries have been sent to the LogstashWriter")
-      @last_log_entry_at            = @metrics_registry.gauge(:mobystash_last_log_entry_at_seconds, "The time at which the last log entry was timestamped")
-      @sampled_entries_sent         = @metrics_registry.counter(:mobystash_sampled_entries_sent_total, "The number of sampled entries which have been sent")
-      @sampled_entries_dropped      = @metrics_registry.counter(:mobystash_sampled_entries_dropped_total, "The number of sampled log entries which didn't get sent")
-      @unsampled_entries            = @metrics_registry.counter(:mobystash_unsampled_entries_total, "How many log messages we've seen which didn't match any defined sample keys")
-      @sample_ratios                = @metrics_registry.gauge(:mobystash_sample_ratio, "The current sample ratio for each sample key")
+      @metrics = []
+
+      @read_event_exception_counter = add_counter(
+        "mobystash_moby_read_exceptions_total",
+        "Exception counts while attempting to read log entries from the Moby server"
+      )
+
+      @log_entries_read_counter = add_counter(
+        "mobystash_log_entries_read_total",
+        "How many log entries have been received from Moby"
+      )
+
+      @log_entries_sent_counter = add_counter(
+        "mobystash_log_entries_sent_total",
+        "How many log entries have been sent to the LogstashWriter"
+      )
+
+      @last_log_entry_at = add_gauge(
+        "mobystash_last_log_entry_at_seconds",
+        "The time at which the last log entry was timestamped"
+      )
+
+      @sampled_entries_sent = add_counter(
+        "mobystash_sampled_entries_sent_total",
+        "The number of sampled entries which have been sent"
+      )
+
+      @sampled_entries_dropped = add_counter(
+        "mobystash_sampled_entries_dropped_total",
+        "The number of sampled log entries which didn't get sent"
+      )
+
+      @unsampled_entries = add_counter(
+        "mobystash_unsampled_entries_total",
+        "How many log messages we've seen which didn't match any defined sample keys"
+      )
+
+      @sample_ratios = add_gauge(
+        "mobystash_sample_ratio",
+        "The current sample ratio for each sample key"
+      )
+    end
+
+    def add_counter(name, desc)

[... diff too long, it was truncated ...]

GitHub sha: 69249419


Approved #2