From f4a4639fcfcef490d65973d11e5961dee141b2ae Mon Sep 17 00:00:00 2001 From: Mark Szabo Date: Wed, 26 Feb 2020 20:38:49 +0100 Subject: [PATCH 1/4] raise exception after exiting watcher.each --- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb | 1 + lib/fluent/plugin/kubernetes_metadata_watch_pods.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb index 26c470f..2bcbf36 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb @@ -119,6 +119,7 @@ def process_namespace_watcher_notices(watcher) @stats.bump(:namespace_cache_watch_ignored) end end + raise end end end diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb index 02e1657..5fdb659 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb @@ -125,6 +125,7 @@ def process_pod_watcher_notices(watcher) @stats.bump(:pod_cache_watch_ignored) end end + raise end end end From d3475fcd7593a8a573b30945721b025612ec8734 Mon Sep 17 00:00:00 2001 From: Mark Szabo Date: Thu, 27 Feb 2020 02:04:08 +0100 Subject: [PATCH 2/4] raise exception when ERROR received while watching --- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb | 4 +++- lib/fluent/plugin/kubernetes_metadata_watch_pods.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb index 2bcbf36..281ab80 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb @@ -113,13 +113,15 @@ def process_namespace_watcher_notices(watcher) # ignore and let age out for cases where # deleted but still processing logs @stats.bump(:namespace_cache_watch_deletes_ignored) + when 'ERROR' + message = notice['object']['message'] if notice['object'] && notice['object']['message'] + raise "Error while watching namespaces: #{message}" else # Don't pay attention to creations, since the created namespace may not # be used by any namespace on this node. @stats.bump(:namespace_cache_watch_ignored) end end - raise end end end diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb index 5fdb659..c8f7256 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb @@ -119,13 +119,15 @@ def process_pod_watcher_notices(watcher) # ignore and let age out for cases where pods # deleted but still processing logs @stats.bump(:pod_cache_watch_delete_ignored) + when 'ERROR' + message = notice['object']['message'] if notice['object'] && notice['object']['message'] + raise "Error while watching pods: #{message}" else # Don't pay attention to creations, since the created pod may not # end up on this node. @stats.bump(:pod_cache_watch_ignored) end end - raise end end end From 2b6de68f23b746cd66b91c978ab15bd050929d32 Mon Sep 17 00:00:00 2001 From: Mark Szabo Date: Tue, 10 Mar 2020 00:38:08 +0100 Subject: [PATCH 3/4] reset watch_retry_count after resourceVersion retrieved from API server --- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb | 3 +++ lib/fluent/plugin/kubernetes_metadata_watch_pods.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb index 281ab80..7d82597 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb @@ -92,6 +92,9 @@ def get_namespaces_and_start_watcher @stats.bump(:namespace_cache_host_updates) end options[:resource_version] = namespaces.resourceVersion + # At this point a successful connection was made to the API server, + # reset namespace_watch_retry_count to zero + Thread.current[:namespace_watch_retry_count] = 0 watcher = @client.watch_namespaces(options) watcher end diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb index c8f7256..a07cf89 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb @@ -95,6 +95,9 @@ def get_pods_and_start_watcher @stats.bump(:pod_cache_host_updates) end options[:resource_version] = pods.resourceVersion + # At this point a successful connection was made to the API server, + # reset pod_watch_retry_count to zero + Thread.current[:pod_watch_retry_count] = 0 watcher = @client.watch_pods(options) watcher end From 501967663e50b3ead02fad561373e90679af973e Mon Sep 17 00:00:00 2001 From: Mark Szabo Date: Tue, 17 Mar 2020 17:54:37 +0100 Subject: [PATCH 4/4] remove watch_retry_count resets, it was handled in another PR which is already merged --- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb | 3 --- lib/fluent/plugin/kubernetes_metadata_watch_pods.rb | 3 --- 2 files changed, 6 deletions(-) diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb index 7d82597..281ab80 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb @@ -92,9 +92,6 @@ def get_namespaces_and_start_watcher @stats.bump(:namespace_cache_host_updates) end options[:resource_version] = namespaces.resourceVersion - # At this point a successful connection was made to the API server, - # reset namespace_watch_retry_count to zero - Thread.current[:namespace_watch_retry_count] = 0 watcher = @client.watch_namespaces(options) watcher end diff --git a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb index a07cf89..c8f7256 100644 --- a/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +++ b/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb @@ -95,9 +95,6 @@ def get_pods_and_start_watcher @stats.bump(:pod_cache_host_updates) end options[:resource_version] = pods.resourceVersion - # At this point a successful connection was made to the API server, - # reset pod_watch_retry_count to zero - Thread.current[:pod_watch_retry_count] = 0 watcher = @client.watch_pods(options) watcher end