FEATURE: Add sidekiq_job_error_count counter metric

There is currently a `sidekiq_job_count` metrics which counts the number
of sidekiq jobs that ran successfully. It is useful to also count the
number of sidekiq jobs that encountered an error so that we can
calculate the error rate of sidekiq jobs.

Author: tgxworld

lib/collector.rb

@@ -196,25 +196,37 @@ def ensure_global_metrics
     def process_job(metric)
       ensure_job_metrics
       hash = { job_name: metric.job_name }
+
       if metric.scheduled
         @scheduled_job_duration_seconds.observe(metric.duration, hash)
         @scheduled_job_count.observe(metric.count, hash)
       else
         @sidekiq_job_duration_seconds.observe(metric.duration, hash)
-        @sidekiq_job_count.observe(metric.count, hash)
+
+        if metric.success
+          @sidekiq_job_count.observe(metric.count, hash)
+        else
+          @sidekiq_job_error_count.observe(metric.count, hash)
+        end
       end
     end
 
     def ensure_job_metrics
       unless @scheduled_job_count
         @scheduled_job_duration_seconds =
           Counter.new("scheduled_job_duration_seconds", "Total time spent in scheduled jobs")
+
         @scheduled_job_count =
-          Counter.new("scheduled_job_count", "Total number of scheduled jobs executued")
+          Counter.new("scheduled_job_count", "Total number of scheduled jobs that succeeded")
+
         @sidekiq_job_duration_seconds =
           Counter.new("sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs")
+
         @sidekiq_job_count =
-          Counter.new("sidekiq_job_count", "Total number of sidekiq jobs executed")
+          Counter.new("sidekiq_job_count", "Total number of sidekiq jobs that succeeded")
+
+        @sidekiq_job_error_count =
+          Counter.new("sidekiq_job_error_count", "Total number of sidekiq jobs that failed")
       end
     end
 
@@ -403,6 +415,7 @@ def job_metrics
           @scheduled_job_count,
           @sidekiq_job_duration_seconds,
           @sidekiq_job_count,
+          @sidekiq_job_error_count,
         ]
       else
         []

lib/internal_metric/job.rb

@@ -2,6 +2,6 @@
 
 module DiscoursePrometheus::InternalMetric
   class Job < Base
-    attribute :job_name, :scheduled, :duration, :count
+    attribute :job_name, :scheduled, :duration, :count, :success
   end
 end

plugin.rb

@@ -65,15 +65,27 @@ module ::DiscoursePrometheus
     metric.job_name = stat.name
     metric.duration = stat.duration_ms * 0.001
     metric.count = 1
+    metric.success = true
     $prometheus_client.send_json metric.to_h unless Rails.env.test?
   end
 
-  on(:sidekiq_job_ran) do |worker, msg, queue, duration|
+  on(:sidekiq_job_ran) do |worker, _msg, _queue, duration|
     metric = DiscoursePrometheus::InternalMetric::Job.new
     metric.scheduled = false
     metric.duration = duration
     metric.count = 1
     metric.job_name = worker.class.to_s
+    metric.success = true
+    $prometheus_client.send_json metric.to_h unless Rails.env.test?
+  end
+
+  on(:sidekiq_job_error) do |worker, _msg, _queue, duration|
+    metric = DiscoursePrometheus::InternalMetric::Job.new
+    metric.scheduled = false
+    metric.duration = duration
+    metric.count = 1
+    metric.job_name = worker.class.to_s
+    metric.success = false
     $prometheus_client.send_json metric.to_h unless Rails.env.test?
   end
 end

spec/lib/collector_spec.rb

@@ -46,6 +46,34 @@
     expect(counter.data).to eq(nil => 3)
   end
 
+  it "handles sidekiq job metrics" do
+    metric_1 = DiscoursePrometheus::InternalMetric::Job.new
+    metric_1.scheduled = false
+    metric_1.job_name = "Bob"
+    metric_1.duration = 1.778
+    metric_1.count = 1
+    metric_1.success = true
+
+    collector.process(metric_1.to_json)
+    metrics = collector.prometheus_metrics
+
+    metric_2 = DiscoursePrometheus::InternalMetric::Job.new
+    metric_2.scheduled = false
+    metric_2.job_name = "Bob"
+    metric_2.duration = 0.5
+    metric_2.count = 1
+    metric_2.success = false
+    collector.process(metric_2.to_json)
+
+    duration = metrics.find { |m| m.name == "sidekiq_job_duration_seconds" }
+    sidekiq_job_count = metrics.find { |m| m.name == "sidekiq_job_count" }
+    sidekiq_job_error_count = metrics.find { |m| m.name == "sidekiq_job_error_count" }
+
+    expect(duration.data).to eq({ job_name: "Bob" } => metric_1.duration + metric_2.duration)
+    expect(sidekiq_job_count.data).to eq({ job_name: "Bob" } => 1)
+    expect(sidekiq_job_error_count.data).to eq({ job_name: "Bob" } => 1)
+  end
+
   it "handles scheduled job metrics" do
     metric = DiscoursePrometheus::InternalMetric::Job.new