FIX: ECS confirmance for mobystash parsed syslog messages

FIX: ECS confirmance for mobystash parsed syslog messages

  • mobystash’s “parse syslog” feature was generating documents that did not match ECS expectations for field names
  • now, documents should have similar fields regardless whether they came via mobystash+parse_syslog or syslogstash
  • some of this logic was cribbed from syslogstash
diff --git a/lib/mobystash/container.rb b/lib/mobystash/container.rb
index 7a9e2b1..ee2c926 100644
--- a/lib/mobystash/container.rb
+++ b/lib/mobystash/container.rb
@@ -2,6 +2,8 @@
 require 'deep_merge'
 require 'murmurhash3'
 
+TIMESTAMP_FORMAT = '%FT%T.%3NZ'
+
 module Mobystash
   # Hoovers up logs for a single container and passes them on to the writer.
   class Container
@@ -131,6 +133,23 @@ module Mobystash
       super
     end
 
+    def parse_timestamp(t) # copied from syslogstash
+      return Time.now.utc if t.nil?
+
+      begin
+        if t.start_with? '*'
+          # unsynced timestamp from IOS, is useless
+          Time.now.utc
+        else
+          # DateTime does a fairly sensible job of this
+          DateTime.parse(t)
+        end
+      rescue
+        # as good a fallback as any
+        Time.now.utc
+      end
+    end
+
     private
 
     def progname
@@ -285,11 +304,16 @@ module Mobystash
       if !@filter_regex || !msg.match?(@filter_regex)
         event = {
           message: msg,
-          "@timestamp": log_time.strftime("%FT%T.%NZ"),
           labels: {
             stream: stream.to_s,
           },
         }.deep_merge(syslog_fields).deep_merge(sampling_metadata).deep_merge!(@tags)
+        if event.key? :"@timestamp"
+          event.deep_merge({ event: { created: log_time.strftime("%FT%T.%NZ") } })
+        else
+          event[:"@timestamp"] = log_time.strftime("%FT%T.%NZ")
+        end
+
 
         # Can't calculate the document_id until you've got a constructed event...
         metadata = {
@@ -332,16 +356,27 @@ module Mobystash
         severity = flags % 8
         facility = flags / 8
 
-        [message, { syslog: {
-          timestamp: timestamp,
-          severity_id: severity,
-          severity_name: SYSLOG_SEVERITIES[severity],
-          facility_id: facility,
-          facility_name: SYSLOG_FACILITIES[facility],
-          hostname: hostname,
-          program: program,
-          pid: pid.nil? ? nil : pid.to_i,
-        }.select { |k, v| !v.nil? } }]
+        log = {
+           '@timestamp': parse_timestamp(timestamp).strftime(TIMESTAMP_FORMAT),
+           log: {
+             original: msg,
+             syslog: {
+               severity: {
+                 code: severity,
+                 name: SYSLOG_SEVERITIES[severity],
+               },
+               facility: {
+                 code: facility,
+                 name: SYSLOG_FACILITIES[facility],
+               },
+             },
+           }
+        }
+        log.deep_merge({ host: { hostname: hostname } }) unless hostname.nil?
+        log.deep_merge({ process: { name: program } }) unless program.nil?
+        log.deep_merge({ process: { pid: pid.to_i } }) unless pid.nil?
+
+        [message, log]
       else
         [msg, {}]
       end
diff --git a/spec/container_spec.rb b/spec/container_spec.rb
index 18e3017..196578e 100644
--- a/spec/container_spec.rb
+++ b/spec/container_spec.rb
@@ -495,26 +495,38 @@ describe Mobystash::Container do
             ecs: {
               version: '1.8',
             },
+            event: {
+              created: "2018-10-02T08:39:16.458228203Z"
+            },
+            host: {
+              hostname: "sumhost",
+            },
             labels: {
               stream: "stderr",
             },
-            syslog: {
-              severity_id: 6,
-              severity_name: "info",
-              facility_id: 18,
-              facility_name: "local2",
-              hostname: "sumhost",
-              timestamp: "Oct 11 10:10:35",
-              program: "ohai",
+            log: {
+              original: "<150>Oct 11 10:10:35 sumhost ohai[3656]: hello from syslog!",
+              syslog: {
+                facility: {
+                  code: 18,
+                  name: "local2",
+                },
+                severity: {
+                  code: 6,
+                  name: "info",
+                },
+              },
+            },
+            process: {
+              name: 'ohai',
               pid: 3656,
             },
-            "@timestamp": "2018-10-02T08:39:16.458228203Z",
+            "@timestamp": "2021-10-11T10:10:35.000Z",
             "@metadata": {
               document_id: match(DOC_ID_REGEX),
               event_type: "moby",
             },
           )
-
         container.run
       end
 
@@ -554,18 +566,29 @@ describe Mobystash::Container do
             ecs: {
               version: '1.8',
             },
+            event: {
+              created: "2018-10-02T08:39:16.458228203Z"
+            },
+            host: {
+              hostname: "sumhost",
+            },
             labels: {
               stream: "stderr",
             },
-            syslog: {
-              severity_id: 6,
-              severity_name: "info",
-              facility_id: 18,
-              facility_name: "local2",
-              hostname: "sumhost",
-              timestamp: "Oct 11 10:10:35",
+            log: {
+              original: "<150>Oct 11 10:10:35 sumhost hello from syslog!",
+              syslog: {
+                facility: {
+                  code: 18,
+                  name: "local2",
+                },
+                severity: {
+                  code: 6,
+                  name: "info",
+                },
+              },
             },
-            "@timestamp": "2018-10-02T08:39:16.458228203Z",
+            "@timestamp": "2021-10-11T10:10:35.000Z",
             "@metadata": {
               document_id: match(DOC_ID_REGEX),
               event_type: "moby",
@@ -611,18 +634,29 @@ describe Mobystash::Container do
             ecs: {
               version: '1.8',
             },
+            event: {
+              created: "2018-10-02T08:39:16.458228203Z"
+            },
             labels: {
               stream: "stderr",
             },
-            syslog: {
-              severity_id: 6,
-              severity_name: "info",
-              facility_id: 18,
-              facility_name: "local2",
-              timestamp: "Oct 11 10:10:35",
-              program: "ohai",
+            process: {
+              name: 'ohai',
+            },
+            log: {
+              original: "<150>Oct 11 10:10:35 ohai: hello from syslog!",
+              syslog: {
+                facility: {
+                  code: 18,
+                  name: "local2",
+                },
+                severity: {
+                  code: 6,
+                  name: "info",
+                },
+              },
             },
-            "@timestamp": "2018-10-02T08:39:16.458228203Z",
+            "@timestamp": "2021-10-11T10:10:35.000Z",
             "@metadata": {
               document_id: match(DOC_ID_REGEX),
               event_type: "moby",
@@ -668,17 +702,26 @@ describe Mobystash::Container do
             ecs: {
               version: '1.8',
             },
+            event: {
+              created: "2018-10-02T08:39:16.458228203Z"
+            },
             labels: {
               stream: "stderr",
             },
-            syslog: {
-              severity_id: 6,
-              severity_name: "info",
-              facility_id: 18,
-              facility_name: "local2",
-              timestamp: "Oct 11 10:10:35",
+            log: {
+              original: "<150>Oct 11 10:10:35 hellofromsyslog!",
+              syslog: {
+                facility: {
+                  code: 18,
+                  name: "local2",
+                },
+                severity: {
+                  code: 6,
+                  name: "info",
+                },
+              },
             },
-            "@timestamp": "2018-10-02T08:39:16.458228203Z",
+            "@timestamp": "2021-10-11T10:10:35.000Z",
             "@metadata": {
               document_id: match(DOC_ID_REGEX),
               event_type: "moby",

GitHub sha: 700cca65

This commit appears in #11 which was approved by davidtaylorhq. It was merged by Supermathie.