Refactor telemetry and fixes

The installation metadata has been refactored:

- The installation metadata store now takes Datums of any
  type as input and output
- Move metadata functions from uuid.c -> metadata.c
- Make metadata functions return native types rather than text,
  including for tests

Telemetry tests for ssl and nossl have been combined.

Note that PG 9.6 does not have pg_backend_random() that gives us a
secure random numbers for UUIDs that we send in telemetry. Therefore,
we fall back to the generating the UUID from the timestamp if we are
on PG 9.6.

This change also fixes a number of test issues. For instance, in the
telemetry test the escape char `E` was passed on as part of the
response string when set as a variable with `\set`. This was not
detected before because the response parser didn't parse the start of
the response properly.

A number of fixes have been made to the formatting of log messages for
telemetry to conform to the PostgreSQL standard as well as being
consistent with other messages.

Numerous build issues on Windows have been resolved. There is also new
functionality to get OS version info on Windows (for telemetry),
including a SQL function get_os_info() to retrieve this information.

The net library will now allow connecting to a servicename, e.g., http
or https. A port is resolved from this service name via getaddrinfo().
An explicit port can still be given, and it that case it will not
resolve the service name.

Databases the are updated to the new version of the extension will
have an install_timestamp in their installation metadata that does not
reflect the actual original install date of the extension. To be able
to distinguish these updated installations from those that are freshly
installed, we add a bogus "epoch" install_timestamp in the update
script.

Parsing of the version string in the telemetry response has been
refactored to be more amenable to testing. Tests have been added.

Author: erimatnor

Diff for: .travis.yml

@@ -18,7 +18,7 @@ before_install:
   - git clone --branch ${PG_GIT_TAG} --depth 1 https://github.com/postgres/postgres.git /tmp/postgres
   - docker run -d --name pgbuild -v ${TRAVIS_BUILD_DIR}:/build -v /tmp/postgres:/postgres postgres:${PG_VERSION}-alpine
 install:
-  - docker exec -it pgbuild /bin/sh -c "apk add --no-cache --virtual .build-deps coreutils dpkg-dev gcc libc-dev make util-linux-dev diffutils cmake bison flex curl git && mkdir -p /build/debug"
+  - docker exec -it pgbuild /bin/sh -c "apk add --no-cache --virtual .build-deps coreutils dpkg-dev gcc libc-dev make util-linux-dev diffutils cmake bison flex curl git openssl-dev && mkdir -p /build/debug"
   - docker exec -it pgbuild /bin/sh -c "apk add --no-cache --virtual --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted lcov"
   # We only need to build the regress stuff
   - docker exec -it pgbuild /bin/sh -c "cd /postgres && ./configure --enable-coverage --enable-debug --enable-cassert --without-readline --without-zlib && make -C /postgres/src/test/regress"

Diff for: CMakeLists.txt

@@ -34,6 +34,15 @@ if (NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel" FORCE)
 endif ()
 
+if (CMAKE_BUILD_TYPE MATCHES Debug)
+  # CMAKE_BUILD_TYPE is set at CMake configuration type. But usage of CMAKE_C_FLAGS_DEBUG is
+  # determined at build time by running cmake --build . --config Debug (at least on Windows).
+  # Therefore, we only set these flags if the configuration-time CMAKE_BUILD_TYPE is set to
+  # Debug. Then Debug enabled builds will only happen on Windows if both the configuration-
+  # and build-time settings are Debug.
+  set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DUSE_ASSERT_CHECKING=1 -DDEBUG=1")
+endif (CMAKE_BUILD_TYPE MATCHES Debug)
+
 set(SUPPORTED_COMPILERS "GNU" "Clang" "AppleClang" "MSVC")
 
 # Check for a supported compiler

Diff for: README.md

@@ -145,6 +145,7 @@ See the Releases tab for the latest release.
 **Prerequisites**:
 
 - A standard [PostgreSQL 9.6 or 10 64-bit installation](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads#windows)
+- OpenSSL for Windows
 - Microsoft Visual Studio 2017 with CMake and Git components
 - OR Visual Studio 2015/2016 with [CMake](https://cmake.org/) version 3.4 or greater and Git
 - Make sure all relevant binaries are in your PATH: `pg_config`, `cmake`, `MSBuild`
@@ -163,16 +164,17 @@ cd timescaledb
 git checkout 0.8.0
 
 # Bootstrap the build system
-./bootstrap.bat
+bootstrap.bat
 
 # To build the extension from command line
-cd build
-MSBuild.exe timescaledb.sln
+cmake --build ./build --config Release
 
 # To install
-MSBuild.exe /p:Configuration=Release INSTALL.vcxproj
+cmake --build ./build --config Release --target install
 
-# Alternatively, open build/timescaledb.sln in Visual Studio and build
+
+# Alternatively, build in Visual Studio via its built-in support for CMake or by
+# opening the generated build/timescaledb.sln solution file.
 ```
 
 ### Additional documentation

Diff for: bootstrap.bat

@@ -29,5 +29,4 @@ cmake %SRC_DIR% -A x64 %*
 ECHO ---
 ECHO TimescaleDB build system initialized in %BUILD_DIR%.
 ECHO To compile, do:
-ECHO     cd %BUILD_DIR%
-ECHO     MSBuild.exe timescaledb.sln
+ECHO     cmake --build %BUILD_DIR% --config Release

Diff for: scripts/docker-build.sh

@@ -25,7 +25,7 @@ else
     docker run -d --name ${BUILD_CONTAINER_NAME} -v ${BASE_DIR}:/src postgres:${PG_IMAGE_TAG}
 
     # Install build dependencies
-    docker exec -u root -it ${BUILD_CONTAINER_NAME} /bin/bash -c "apk add --no-cache --virtual .build-deps gdb git coreutils dpkg-dev gcc libc-dev make cmake util-linux-dev diffutils && mkdir -p /build/debug"
+    docker exec -u root -it ${BUILD_CONTAINER_NAME} /bin/bash -c "apk add --no-cache --virtual .build-deps gdb git coreutils dpkg-dev gcc libc-dev make cmake util-linux-dev diffutils openssl-dev && mkdir -p /build/debug"
 
     docker commit -a $USER -m "TimescaleDB build base image version $PG_IMAGE_TAG" ${BUILD_CONTAINER_NAME} ${BUILD_IMAGE_NAME}:${PG_IMAGE_TAG}
 fi

Diff for: sql/CMakeLists.txt

@@ -107,12 +107,9 @@ function(cat_files SRC_FILE_LIST OUTPUT_FILE)
   if (WIN32)
     # Make list of files into string of files separated by "+"
     # to make Windows copy concatenate them
-
     file(TO_NATIVE_PATH "${SRC_FILE_LIST}" SRC_FILE_LIST_NATIVE)
-    string(REPLACE ";" ";+" SQL_LIST_JOINED "${SRC_FILE_LIST_NATIVE}")
-
+    string(REPLACE ";" ";+;" SQL_LIST_JOINED "${SRC_FILE_LIST_NATIVE}")
     file(TO_NATIVE_PATH "${OUTPUT_FILE}" OUTPUT_FILE_NATIVE)
-
     set(CAT_CMD copy /B /y ${SQL_LIST_JOINED} "\"${OUTPUT_FILE_NATIVE}\"" >NUL)
   else ()
     set(CAT_CMD cat ${SRC_FILE_LIST} > ${OUTPUT_FILE})

Diff for: sql/installation_metadata.sql

@@ -1,7 +1,10 @@
--- Insert uuid and install_timestamp on database creation.
--- Don't create exported_uuid because this shouldn't get called on a create
-CREATE FUNCTION _timescaledb_internal.generate_uuid_external() RETURNS TEXT
-AS '@MODULE_PATHNAME@', 'generate_uuid_external' LANGUAGE C VOLATILE STRICT;
-INSERT INTO _timescaledb_catalog.installation_metadata SELECT 'uuid', _timescaledb_internal.generate_uuid_external();
-INSERT INTO _timescaledb_catalog.installation_metadata SELECT 'install_timestamp', now();
-DROP FUNCTION _timescaledb_internal.generate_uuid_external();
+CREATE OR REPLACE FUNCTION _timescaledb_internal.generate_uuid() RETURNS UUID
+AS '@MODULE_PATHNAME@', 'ts_uuid_generate' LANGUAGE C VOLATILE STRICT;
+
+-- Insert uuid and install_timestamp on database creation. Don't
+-- create exported_uuid because it gets exported and installed during
+-- pg_dump, which would cause a conflict.
+INSERT INTO _timescaledb_catalog.installation_metadata
+SELECT 'uuid', _timescaledb_internal.generate_uuid() ON CONFLICT DO NOTHING;
+INSERT INTO _timescaledb_catalog.installation_metadata
+SELECT 'install_timestamp', now() ON CONFLICT DO NOTHING;

Diff for: sql/pre_install/tables.sql

@@ -180,8 +180,7 @@ CREATE TABLE IF NOT EXISTS _timescaledb_catalog.installation_metadata (
     key     NAME NOT NULL PRIMARY KEY,
     value   TEXT NOT NULL
 );
-SELECT
-pg_catalog.pg_extension_config_dump('_timescaledb_catalog.installation_metadata', $$WHERE key='exported_uuid'$$);
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.installation_metadata', $$WHERE key='exported_uuid'$$);
 
 -- Set table permissions
 GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_catalog TO PUBLIC;

Diff for: sql/updates/0.11.0--0.11.1-dev.sql

@@ -54,8 +54,11 @@ BEGIN
 users. For more information and how to disable, please see our docs https://docs.timescaledb.com/using-timescaledb/telemetry.\n';
 END;
 $$;
-CREATE FUNCTION _timescaledb_internal.generate_uuid_external() RETURNS TEXT
-AS '@MODULE_PATHNAME@', 'generate_uuid_external' LANGUAGE C VOLATILE STRICT;
-INSERT INTO _timescaledb_catalog.installation_metadata SELECT 'uuid', _timescaledb_internal.generate_uuid_external();
-INSERT INTO _timescaledb_catalog.installation_metadata SELECT 'install_timestamp', GetCurrentTimestamp();
-DROP FUNCTION _timescaledb_internal.generate_uuid_external();
+
+CREATE TABLE IF NOT EXISTS _timescaledb_catalog.installation_metadata (
+    key     NAME NOT NULL PRIMARY KEY,
+    value   TEXT NOT NULL
+);
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.installation_metadata', $$WHERE key='exported_uuid'$$);
+
+INSERT INTO _timescaledb_catalog.installation_metadata SELECT 'install_timestamp', to_timestamp(0);

Diff for: sql/version.sql

@@ -1,5 +1,8 @@
 CREATE OR REPLACE FUNCTION _timescaledb_internal.get_git_commit() RETURNS TEXT
     AS '@MODULE_PATHNAME@', 'get_git_commit' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
 
+CREATE OR REPLACE FUNCTION _timescaledb_internal.get_os_info() RETURNS TABLE(sysname TEXT, version TEXT, release TEXT)
+    AS '@MODULE_PATHNAME@', 'ts_get_os_info' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
 CREATE OR REPLACE FUNCTION get_report() RETURNS TEXT
     AS '@MODULE_PATHNAME@', 'ts_get_telemetry_report' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;

Diff for: src/CMakeLists.txt

@@ -1,5 +1,3 @@
-set(CMAKE_C_FLAGS_DEBUG "-DUSE_ASSERT_CHECKING=1 -DDEBUG=1")
-
 # Check if PostgreSQL has OpenSSL enabled
 file(STRINGS ${PG_INCLUDEDIR}/pg_config.h PG_USE_OPENSSL REGEX "#define USE_OPENSSL [01]")
 string(REGEX REPLACE "#define USE_OPENSSL ([01])" "\\1" USE_OPENSSL ${PG_USE_OPENSSL})
@@ -41,7 +39,7 @@ else ()
 endif ()
 
 if (WIN32)
-  set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LIBDIR}/postgres.lib ws2_32.lib")
+  set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LIBDIR}/postgres.lib ws2_32.lib Version.lib")
   set(CMAKE_C_FLAGS "-D_CRT_SECURE_NO_WARNINGS")
   include_directories(${PG_INCLUDEDIR_SERVER}/port/win32)
 
@@ -94,7 +92,8 @@ set(HEADERS
   subspace_store.h
   tablespace.h
   trigger.h
-  utils.h)
+  utils.h
+  version.h)
 
 set(SOURCES
   agg_bookend.c
@@ -142,7 +141,6 @@ set(SOURCES
   utils.c
   version.c)
 
-configure_file(version.h.in version.h)
 set(GITCOMMIT_H ${CMAKE_CURRENT_BINARY_DIR}/gitcommit.h)
 
 # Add test source code in Debug builds
@@ -168,22 +166,18 @@ else ()
     VERBATIM)
 endif (WIN32)
 
-add_library(${PROJECT_NAME} MODULE ${SOURCES} ${TEST_SOURCES} ${HEADERS} ${GITCOMMIT_H})
+add_library(${PROJECT_NAME} MODULE ${SOURCES} ${TEST_SOURCES} ${GITCOMMIT_H})
 
 if (OPENSSL_FOUND AND USE_OPENSSL)
     target_link_libraries(${PROJECT_NAME} ${OPENSSL_LIBRARIES})
 endif (OPENSSL_FOUND AND USE_OPENSSL)
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-add_subdirectory(bgw)
-add_subdirectory(net)
-add_subdirectory(telemetry)
-
 if (OPENSSL_FOUND AND USE_OPENSSL)
   target_link_libraries(${PROJECT_NAME} ${OPENSSL_LIBRARIES})
 endif (OPENSSL_FOUND AND USE_OPENSSL)
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
+  set(DEBUG 1)
   add_subdirectory(../test/src "${CMAKE_CURRENT_BINARY_DIR}/test/src")
 endif(CMAKE_BUILD_TYPE MATCHES Debug)
 
@@ -200,4 +194,10 @@ install(
   TARGETS ${PROJECT_NAME}
   DESTINATION ${PG_PKGLIBDIR})
 
+configure_file(config.h.in config.h)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+add_subdirectory(bgw)
+add_subdirectory(net)
+add_subdirectory(telemetry)
 add_subdirectory(loader)

Diff for: src/catalog.h

@@ -589,9 +589,6 @@ enum Anum_bgw_job_stat_pkey_idx
  ******************************/
 
 #define INSTALLATION_METADATA_TABLE_NAME		"installation_metadata"
-#define INSTALLATION_METADATA_UUID_KEY_NAME			"uuid"
-#define INSTALLATION_METADATA_EXPORTED_UUID_KEY_NAME	"exported_uuid"
-#define INSTALLATION_METADATA_TIMESTAMP_KEY_NAME		"install_timestamp"
 
 enum Anum_installation_metadata
 {
@@ -627,20 +624,11 @@ enum
 	_MAX_INSTALLATION_METADATA_INDEX,
 };
 
-#define MAX(a, b) \
-	((long)(a) > (long)(b) ? (a) : (b))
-
-#define _MAX_TABLE_INDEXES												\
-	MAX(_MAX_HYPERTABLE_INDEX,											\
-		MAX(_MAX_DIMENSION_INDEX,										\
-			MAX(_MAX_DIMENSION_SLICE_INDEX,								\
-				MAX(_MAX_CHUNK_CONSTRAINT_INDEX,						\
-					MAX(_MAX_CHUNK_INDEX_INDEX,							\
-						MAX(_MAX_TABLESPACE_INDEX,						\
-							MAX(_MAX_BGW_JOB_INDEX,						\
-								MAX(_MAX_BGW_JOB_STAT_INDEX,			\
-									MAX(_MAX_INSTALLATION_METADATA_INDEX, \
-										_MAX_CHUNK_INDEX)))))))))
+/*
+ * The maximum number of indexes a catalog table can have.
+ * This needs to be bumped in case of new catalog tables that have more indexes.
+ */
+#define _MAX_TABLE_INDEXES 5
 
 typedef enum CacheType
 {
@@ -659,8 +647,8 @@ typedef struct Catalog
 		const char *schema_name;
 		const char *name;
 		Oid			id;
-		Oid			index_ids[_MAX_TABLE_INDEXES];
 		Oid			serial_relid;
+		Oid			index_ids[_MAX_TABLE_INDEXES];
 	}			tables[_MAX_CATALOG_TABLES];
 
 	Oid			cache_schema_id;

Diff for: src/compat.h

@@ -1,6 +1,8 @@
 #ifndef TIMESCALEDB_COMPAT_H
 #define TIMESCALEDB_COMPAT_H
 
+#include <postgres.h>
+
 #include "export.h"
 
 #define is_supported_pg_version_96(version) ((version >= 90603) && (version < 100000))

Diff for: src/version.h.in renamed to src/config.h.in

@@ -1,5 +1,5 @@
-#ifndef TIMESCALEDB_VERSION_H
-#define TIMESCALEDB_VERSION_H
+#ifndef TIMESCALEDB_CONFIG_H
+#define TIMESCALEDB_CONFIG_H
 
 #define TIMESCALEDB_VERSION "@PROJECT_VERSION@"
 #define TIMESCALEDB_VERSION_MOD "@PROJECT_VERSION_MOD@"
@@ -10,8 +10,28 @@
 #define BUILD_OS_NAME "@CMAKE_SYSTEM_NAME@"
 #define BUILD_OS_VERSION "@CMAKE_SYSTEM_VERSION@"
 #define PG_VERSION "@PG_VERSION@"
-// Value should be set in package release scripts. Otherwise
-// defaults to "source"
+/*
+ * Value should be set in package release scripts. Otherwise
+ * defaults to "source"
+ */
 #define TIMESCALEDB_INSTALL_METHOD "@PROJECT_INSTALL_METHOD@"
 
-#endif /* TIMESCALEDB_VERSION_H */
+/* Platform */
+#ifndef WIN32
+#cmakedefine WIN32
+#endif
+#ifndef MSVC
+#cmakedefine MSVC
+#endif
+#ifndef UNIX
+#cmakedefine UNIX
+#endif
+#ifndef APPLE
+#cmakedefine APPLE
+#endif
+
+#ifndef DEBUG
+#cmakedefine DEBUG
+#endif
+
+#endif /* TIMESCALEDB_CONFIG_H */

Diff for: src/export.h

@@ -3,6 +3,8 @@
 
 #include <postgres.h>
 
+#include "config.h"
+
 /* Definitions for symbol exports */
 
 #define TS_CAT(x,y) x ## y

Diff for: src/extension.c

@@ -21,7 +21,7 @@
 #include "catalog.h"
 #include "extension.h"
 #include "guc.h"
-#include "version.h"
+#include "config.h"
 #include "extension_utils.c"
 #include "compat.h"
 

Diff for: src/guc.c

@@ -4,11 +4,23 @@
 
 #include "guc.h"
 #include "hypertable_cache.h"
+#include "telemetry/telemetry.h"
+
+typedef enum TelemetryLevel
+{
+	TELEMETRY_OFF,
+	TELEMETRY_BASIC,
+} TelemetryLevel;
+
+/* Define which level means on. We use this object to have at least one object
+ * of type TelemetryLevel in the code, otherwise pgindent won't work for the
+ * type */
+static const TelemetryLevel on_level = TELEMETRY_BASIC;
 
 bool
 telemetry_on()
 {
-	return guc_telemetry_level == TELEMETRY_BASIC;
+	return guc_telemetry_level == on_level;
 }
 
 static const struct config_enum_entry telemetry_level_options[] =
@@ -24,7 +36,7 @@ bool		guc_restoring = false;
 bool		guc_constraint_aware_append = true;
 int			guc_max_open_chunks_per_insert = 10;
 int			guc_max_cached_chunks_per_hypertable = 10;
-char	   *guc_telemetry_endpoint = "https://telemetry.timescale.com";
+char	   *guc_telemetry_endpoint = TELEMETRY_ENDPOINT;
 int			guc_telemetry_level = TELEMETRY_BASIC;
 
 static void
@@ -113,7 +125,7 @@ _guc_init(void)
 							   "URI for telemetry endpoint",
 							   "URI for telemetry endpoint",
 							   &guc_telemetry_endpoint,
-							   "https://telemetry.timescale.com",
+							   guc_telemetry_endpoint,
 							   PGC_INTERNAL,
 							   0,
 							   NULL,

Diff for: src/guc.h

@@ -1,12 +1,7 @@
 #ifndef TIMESCALEDB_GUC_H
 #define TIMESCALEDB_GUC_H
-#include <postgres.h>
 
-typedef enum TelemetryLevel
-{
-	TELEMETRY_OFF,
-	TELEMETRY_BASIC,
-} TelemetryLevel;
+#include <postgres.h>
 
 extern bool telemetry_on(void);