From 2ebc2ca557275587b0b045da425e8d9341243221 Mon Sep 17 00:00:00 2001
From: Roman Gusev <roman.gusev@gmail.com>
Date: Tue, 27 Apr 2021 20:54:38 +0200
Subject: [PATCH] Keep non-discoverable metadata from the nodes

---
 tap_postgres/stream_utils.py |  6 ++++++
 tests/test_streams_utils.py  | 14 ++++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/tap_postgres/stream_utils.py b/tap_postgres/stream_utils.py
index 84a4f46e..43910da6 100644
--- a/tap_postgres/stream_utils.py
+++ b/tap_postgres/stream_utils.py
@@ -90,6 +90,12 @@ def refresh_streams_schema(conn_config: Dict, streams: List[Dict]):
                 if not metadatum['breadcrumb']:
                     meta.update(new_discovery[stream['tap_stream_id']]['metadata'][idx_met]['metadata'])
                     new_discovery[stream['tap_stream_id']]['metadata'][idx_met]['metadata'] = meta
+                elif metadatum['breadcrumb'] in md_map:
+                    # takes discoverable node's metadata from new_discover
+                    # and keeps remaining metadata from the original node (like 'selected')
+                    node_md_map = md_map.get(metadatum['breadcrumb'])
+                    node_md_map.update(metadatum['metadata'])
+                    new_discovery[stream['tap_stream_id']]['metadata'][idx_met]['metadata'] = node_md_map
 
             # 2nd step: now copy all the metadata from the updated new discovery to the original stream
             streams[idx]['metadata'] = copy.deepcopy(new_discovery[stream['tap_stream_id']]['metadata'])
diff --git a/tests/test_streams_utils.py b/tests/test_streams_utils.py
index 114ab5c2..aaa978be 100644
--- a/tests/test_streams_utils.py
+++ b/tests/test_streams_utils.py
@@ -29,7 +29,8 @@ def setUp(self):
 								  {"name": '"character-varying_name"', "type": "character varying"},
 								  {"name": '"varchar-name"', "type": "varchar(28)"},
 								  {"name": 'char_name', "type": "char(10)"},
-								  {"name": '"text-name"', "type": "text"}],
+								  {"name": '"text-name"', "type": "text"},
+								  {"name": '"column_to_exclude-name"', "type": "text"}],
 					  "name": self.table_name}
 
 		ensure_test_table(table_spec)
@@ -51,6 +52,10 @@ def test_refresh_streams_schema(self):
 							'table-key-properties': ['some_id'],
 							'row-count': 1000,
 						}
+					},
+					{
+						'breadcrumb': ['properties', 'column_to_exclude-name'],
+						'metadata': {'selected': False}
 					}
 				]
 			}
@@ -75,6 +80,10 @@ def test_refresh_streams_schema(self):
 			('properties', 'character-varying_name'): {'inclusion': 'available',
 													   'sql-datatype': 'character varying',
 													   'selected-by-default': True},
+		    ('properties', 'column_to_exclude-name'): {'inclusion': 'available',
+		                                               'selected': False,
+		                                               'selected-by-default': True,
+		                                               'sql-datatype': 'text'},
 			('properties', 'id'): {'inclusion': 'automatic',
 								   'sql-datatype': 'integer',
 								   'selected-by-default': True},
@@ -94,6 +103,7 @@ def test_refresh_streams_schema(self):
 										 'character-varying_name': {'type': ['null', 'string']},
 										 'varchar-name': {'type': ['null', 'string'], 'maxLength': 28},
 										 'char_name': {'type': ['null', 'string'], 'maxLength': 10},
-										 'text-name': {'type': ['null', 'string']}},
+										 'text-name': {'type': ['null', 'string']},
+										 'column_to_exclude-name': {'type': ['null', 'string']}},
 						  'type': 'object',
 						  'definitions': BASE_RECURSIVE_SCHEMAS}, streams[0].get('schema'))