Add support for event tracking metrics

Add support for event tracking metrics

Use :pending_refresh_total to count how many images are pending refresh Use :pending_events_total to figure out how many inotify events have still not been processed

pending_events_total is useful in automation, allows us to determine if nefario has finished all work

Also adds local server for testing

diff --git a/Rakefile b/Rakefile
index e65165e..ebf04c8 100644
--- a/Rakefile
+++ b/Rakefile
@@ -22,6 +22,14 @@ task :guard do
   sh "guard --clear"
 end
 
+desc "Run sample local nefario"
+task :run_local do
+  $LOAD_PATH.unshift(File.expand_path(".", "lib"))
+  ENV["NEFARIO_CONFIG_DIRECTORY"] = File.expand_path(".", "sample/pods")
+  ENV["NEFARIO_METRICS_PORT"] = "9408"
+  load(File.expand_path(".", "bin/nefario"))
+end
+
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new :test do |t|
   t.pattern = "spec/**/*_spec.rb"
diff --git a/lib/nefario.rb b/lib/nefario.rb
index 7edcc9a..0f4cc20 100644
--- a/lib/nefario.rb
+++ b/lib/nefario.rb
@@ -11,6 +11,9 @@ class Nefario
   string :NEFARIO_HOST_HOSTNAME, default: Socket.gethostname
   integer :NEFARIO_IMAGE_REFRESH_INTERVAL, default: 15 * 60
 
+  gauge :pending_refresh_total, docstring: "number of pods pending refresh"
+  gauge :pending_events_total, docstring: "number of inotify events left to process"
+
   def self.register_ultravisor_children(ultravisor, config:, metrics_registry:)
     ultravisor.add_child(
       id: :moby_derp_runner,
diff --git a/lib/nefario/config_dir_watcher.rb b/lib/nefario/config_dir_watcher.rb
index bad9e20..f212d36 100644
--- a/lib/nefario/config_dir_watcher.rb
+++ b/lib/nefario/config_dir_watcher.rb
@@ -9,14 +9,19 @@ class Nefario::ConfigDirWatcher
     @logger = @config.logger
 
     @sd_r, @sd_w = IO.pipe
+
+    @inotify_queue = Queue.new
+    @processing_thread = create_processing_thread
   end
 
   def run
     notifier = INotify::Notifier.new
     notifier.watch(@config.config_directory, :move, :close_write, :delete) do |event|
-      process_inotify_event(event)
+      @metrics.pending_events_total.set(@inotify_queue.length)
+      @inotify_queue << event
     end
 
+
     logger.info(logloc) { "Refreshing all pods" }
     Pathname.new(@config.config_directory).each_child do |f|
       next unless f.basename.to_s =~ /(\A[^.]|\.yaml\z)/
@@ -47,6 +52,8 @@ class Nefario::ConfigDirWatcher
 
   def shutdown
     logger.debug(logloc) { "Received polite request to shutdown" }
+    @inotify_queue.close
+    @processing_thread.join
     @sd_w.putc("!")
   end
 
@@ -54,6 +61,24 @@ class Nefario::ConfigDirWatcher
 
   attr_reader :logger
 
+  def create_processing_thread
+    Thread.new do
+      while true
+        begin
+
+          break if @inotify_queue.closed?
+          event = @inotify_queue.pop
+          break if @inotify_queue.closed?
+
+          process_inotify_event(event)
+          @metrics.pending_events_total.set(@inotify_queue.length)
+        rescue => e
+          logger.warn(logloc) { "Failed to process inotify event: #{e}" }
+        end
+      end
+    end
+  end
+
   def process_inotify_event(event)
     logger.debug(logloc) { "Processing inotify event: #{event.flags} => #{event.name}" }
 
diff --git a/lib/nefario/image_watcher.rb b/lib/nefario/image_watcher.rb
index 9693ddc..5f27374 100644
--- a/lib/nefario/image_watcher.rb
+++ b/lib/nefario/image_watcher.rb
@@ -58,10 +58,17 @@ class Nefario::ImageWatcher
       end
     end
 
-    changed_pods.uniq.each do |p|
+    changed_pods.uniq!
+
+    @metrics.pending_refresh_total.set(changed_pods.length)
+    i = 0
+
+    changed_pods.each do |p|
       logger.info(logloc) { "Refreshing pod #{p} due to out-of-date image" }
 
       @ultravisor[:moby_derp_runner].cast.refresh_pod(p)
+      i += 1
+      @metrics.pending_refresh_total.set((changed_pods.length - i))
     end
   end
 
diff --git a/sample/pods/sample_container.yaml b/sample/pods/sample_container.yaml
new file mode 100644
index 0000000..4e88481
--- /dev/null
+++ b/sample/pods/sample_container.yaml
@@ -0,0 +1,15 @@
+hostname: testing
+root_labels:
+  test.label: '100'
+expose:
+  - 9997
+  - 9408
+containers:
+  green:
+    image: "debian:latest"
+    command: "/bin/sleep 102"
+    restart: always
+    environment:
+      FOO: bar22
+    labels:
+      inner.label: 'test'

GitHub sha: 0eda8349