From fdc076b131a02a2cd7d32e527961f61c7ba641dc Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Tue, 20 Apr 2021 10:32:27 -0700 Subject: [PATCH 1/5] tls http proxy --- .../src/http_proxy_io.c | 115 +++++++++++++++++- uamqp/__init__.py | 2 +- uamqp/authentication/common.py | 49 ++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) diff --git a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/http_proxy_io.c b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/http_proxy_io.c index 1e682dd54..398cbf3c3 100644 --- a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/http_proxy_io.c +++ b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/http_proxy_io.c @@ -9,11 +9,18 @@ #include "azure_c_shared_utility/gballoc.h" #include "azure_c_shared_utility/xio.h" #include "azure_c_shared_utility/socketio.h" +#include "azure_c_shared_utility/tlsio.h" +#include "azure_c_shared_utility/platform.h" #include "azure_c_shared_utility/crt_abstractions.h" #include "azure_c_shared_utility/http_proxy_io.h" #include "azure_c_shared_utility/azure_base64.h" +#include "azure_c_shared_utility/shared_util_options.h" static const char* const OPTION_UNDERLYING_IO_OPTIONS = "underlying_io_options"; +static const char* const OPTION_USE_TLS_HTTP_PROXY = "use_tls_http_proxy"; +static const char* const OPTION_TLS_HTTP_PROXY_TRUSTED_CERT = "tls_http_proxy_TrustedCerts"; +static const char* const OPTION_TLS_HTTP_PROXY_X509_CERT = "tls_http_proxy_x509certificate"; +static const char* const OPTION_TLS_HTTP_PROXY_X509_PRIVATE_KEY = "tls_http_proxy_x509privatekey"; typedef enum HTTP_PROXY_IO_STATE_TAG { @@ -45,6 +52,7 @@ typedef struct HTTP_PROXY_IO_INSTANCE_TAG XIO_HANDLE underlying_io; unsigned char* receive_buffer; size_t receive_buffer_size; + bool use_tls_http_proxy; } HTTP_PROXY_IO_INSTANCE; static CONCRETE_IO_HANDLE http_proxy_io_create(void* io_create_parameters) @@ -188,6 +196,7 @@ static CONCRETE_IO_HANDLE http_proxy_io_create(void* io_create_parameters) result->receive_buffer = NULL; result->receive_buffer_size = 0; result->http_proxy_io_state = HTTP_PROXY_IO_STATE_CLOSED; + result->use_tls_http_proxy = false; } } } @@ -918,6 +927,110 @@ static int http_proxy_io_set_option(CONCRETE_IO_HANDLE http_proxy_io, const char result = 0; } } + else if (strcmp(option_name, OPTION_USE_TLS_HTTP_PROXY) == 0) + { + bool use_tls_http_proxy = *((bool*)value); + if(http_proxy_io_instance->use_tls_http_proxy) + { + LogError("use tls http proxy has already been specified"); + result = MU_FAILURE; + } + else + { + if(use_tls_http_proxy) + { + // close and desotry the original socket io + (void)xio_close(http_proxy_io_instance->underlying_io, NULL, NULL); + xio_destroy(http_proxy_io_instance->underlying_io); + http_proxy_io_instance->use_tls_http_proxy = true; + const IO_INTERFACE_DESCRIPTION* underlying_io_interface; + underlying_io_interface = platform_get_default_tlsio(); + // create a new tls io + TLSIO_CONFIG tls_io_config; + XIO_HANDLE tls_io; + tls_io_config.hostname = http_proxy_io_instance->proxy_hostname; + tls_io_config.port = http_proxy_io_instance->proxy_port; + tls_io_config.underlying_io_interface = NULL; + tls_io_config.underlying_io_parameters = NULL; + tls_io = xio_create(underlying_io_interface, &tls_io_config); + if (tls_io == NULL) + { + LogError("xio_create failed"); + result = MU_FAILURE; + } + else + { + http_proxy_io_instance->underlying_io = tls_io; + result = 0; + } + } + else + { + // setting false to false, do nothing + result = 0; + } + } + } + else if (strcmp(option_name, OPTION_TLS_HTTP_PROXY_TRUSTED_CERT) == 0) + { + if (!http_proxy_io_instance->use_tls_http_proxy) + { + LogError("Invalid option %s", option_name); + result = MU_FAILURE; + } + else + { + if (xio_setoption(http_proxy_io_instance->underlying_io, OPTION_TRUSTED_CERT, value) != 0) + { + LogError("Setting option %s failed", option_name); + result = MU_FAILURE; + } + else + { + result = 0; + } + } + } + else if (strcmp(option_name, OPTION_TLS_HTTP_PROXY_X509_CERT) == 0) + { + if (!http_proxy_io_instance->use_tls_http_proxy) + { + LogError("Invalid option %s", option_name); + result = MU_FAILURE; + } + else + { + if (xio_setoption(http_proxy_io_instance->underlying_io, SU_OPTION_X509_CERT, value) != 0) + { + LogError("Setting option %s failed", option_name); + result = MU_FAILURE; + } + else + { + result = 0; + } + } + } + else if (strcmp(option_name, OPTION_TLS_HTTP_PROXY_X509_PRIVATE_KEY) == 0) + { + if (!http_proxy_io_instance->use_tls_http_proxy) + { + LogError("Invalid option %s", option_name); + result = MU_FAILURE; + } + else + { + if (xio_setoption(http_proxy_io_instance->underlying_io, SU_OPTION_X509_PRIVATE_KEY, value) != 0) + { + LogError("Setting option %s failed", option_name); + result = MU_FAILURE; + } + else + { + result = 0; + } + } + } /* Codes_SRS_HTTP_PROXY_IO_01_043: [ If the option_name argument indicates an option that is not handled by http_proxy_io_set_option, then xio_setoption shall be called on the underlying IO created in http_proxy_io_create, passing the option name and value to it. ]*/ /* Codes_SRS_HTTP_PROXY_IO_01_056: [ The value argument shall be allowed to be NULL. ]*/ else if (xio_setoption(http_proxy_io_instance->underlying_io, option_name, value) != 0) @@ -949,7 +1062,7 @@ static void* http_proxy_io_clone_option(const char* name, const void* value) } else { - if (strcmp(name, OPTION_UNDERLYING_IO_OPTIONS) == 0) + if (strcmp(name, OPTION_UNDERLYING_IO_OPTIONS) == 0 || (strcmp(name, OPTION_USE_TLS_HTTP_PROXY) == 0)) { result = (void*)value; } diff --git a/uamqp/__init__.py b/uamqp/__init__.py index a8e9ab766..27b702d9c 100644 --- a/uamqp/__init__.py +++ b/uamqp/__init__.py @@ -35,7 +35,7 @@ pass # Async not supported. -__version__ = "1.3.0" +__version__ = "1.4.0b1" _logger = logging.getLogger(__name__) diff --git a/uamqp/authentication/common.py b/uamqp/authentication/common.py index ca99ba89f..a6d2b6924 100644 --- a/uamqp/authentication/common.py +++ b/uamqp/authentication/common.py @@ -65,6 +65,37 @@ def _build_proxy_config(self, hostname, port, proxy_settings): def _encode(self, value): return value.encode(self._encoding) if isinstance(value, six.text_type) else value + @staticmethod + def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_client_cert=None, proxy_client_private_key=None): + if proxy_client_cert or proxy_client_private_key and not all((proxy_client_cert, proxy_client_private_key)): + raise ValueError("Client cert and key must both present.") + + underlying_xio.set_option("use_tls_http_proxy", True) + + if proxy_server_cert: + with open(proxy_server_cert, 'rb') as proxy_server_cert_handle: + proxy_server_cert_data = proxy_server_cert_handle.read() + try: + underlying_xio.set_option("tls_http_proxy_TrustedCerts", proxy_server_cert_data) + except ValueError: + _logger.warning('Unable to set external proxy certificates.') + + if proxy_client_cert: + with open(proxy_client_cert, 'rb') as proxy_client_cert_handle: + proxy_client_cert_data = proxy_client_cert_handle.read() + try: + underlying_xio.set_option("tls_http_proxy_x509certificate", proxy_client_cert_data) + except ValueError: + _logger.warning('Unable to set external proxy x509certificates.') + + if proxy_client_private_key: + with open(proxy_client_private_key, 'rb') as proxy_client_private_key_handle: + proxy_client_private_key_data = proxy_client_private_key_handle.read() + try: + underlying_xio.set_option("tls_http_proxy_x509privatekey", proxy_client_private_key_data) + except ValueError: + _logger.warning('Unable to set external x509privatekey.') + def set_io(self, hostname, port, http_proxy, transport_type): if transport_type == TransportType.AmqpOverWebsocket or http_proxy is not None: self.set_wsio(hostname, port or constants.DEFAULT_AMQP_WSS_PORT, http_proxy) @@ -89,14 +120,32 @@ def set_wsio(self, hostname, port, http_proxy): _tlsio_config.hostname = hostname _tlsio_config.port = port + proxy_server_cert = None + proxy_client_cert = None + proxy_client_private_key = None + use_tls_http_proxy = False + if http_proxy: proxy_config = self._build_proxy_config(hostname, port, http_proxy) + proxy_server_cert = http_proxy.get("proxy_verify") + # TODO: allow passing a tuple of client cert/key files, order matters + # check content to see if it's a key or cert file? + proxy_client_cert, proxy_client_private_key = http_proxy.get("proxy_cert") + use_tls_http_proxy = any((proxy_server_cert, proxy_client_cert, proxy_client_private_key)) _tlsio_config.set_proxy_config(proxy_config) _wsio_config.set_tlsio_config(_default_tlsio, _tlsio_config) _underlying_xio = c_uamqp.xio_from_wsioconfig(_wsio_config) # pylint: disable=attribute-defined-outside-init + if http_proxy and use_tls_http_proxy: + self._configure_tls_http_proxy( + _underlying_xio, + proxy_server_cert, + proxy_client_cert, + proxy_client_private_key + ) + cert = self.cert_file or certifi.where() with open(cert, 'rb') as cert_handle: cert_data = cert_handle.read() From d8742ea19fff2040b466306efc619a326f26468f Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Tue, 20 Apr 2021 12:43:48 -0700 Subject: [PATCH 2/5] update cython impl --- src/xio.pyx | 12 ++++++++++++ uamqp/authentication/common.py | 12 +++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/xio.pyx b/src/xio.pyx index 4a04311d3..e868d129d 100644 --- a/src/xio.pyx +++ b/src/xio.pyx @@ -81,6 +81,18 @@ cdef class XIO(StructBase): if c_xio.xio_setoption(self._c_value, option_name, option_value) != 0: raise self._value_error("Failed to set option {}".format(option_name)) + cpdef set_bool_value_option(self, bytes name, bint value): + cdef char *option_name = name + cdef bint option_value = value + if c_xio.xio_setoption(self._c_value, option_name, (&option_value)) != 0: + raise self._value_error("Failed to set option {}".format(name)) + + cpdef set_bytes_value_option(self, bytes name, bytes value): + cdef char *option_name = name + cdef char *option_value = value + if c_xio.xio_setoption(self._c_value, option_name, option_value) != 0: + raise self._value_error("Failed to set option {}".format(name)) + cpdef set_certificates(self, bytes value): cdef char *certificate = value if c_xio.xio_setoption(self._c_value, b'TrustedCerts', certificate) != 0: diff --git a/uamqp/authentication/common.py b/uamqp/authentication/common.py index a6d2b6924..6e5fbb57c 100644 --- a/uamqp/authentication/common.py +++ b/uamqp/authentication/common.py @@ -70,13 +70,13 @@ def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_clie if proxy_client_cert or proxy_client_private_key and not all((proxy_client_cert, proxy_client_private_key)): raise ValueError("Client cert and key must both present.") - underlying_xio.set_option("use_tls_http_proxy", True) + underlying_xio.set_bool_value_option(b"use_tls_http_proxy", True) if proxy_server_cert: with open(proxy_server_cert, 'rb') as proxy_server_cert_handle: proxy_server_cert_data = proxy_server_cert_handle.read() try: - underlying_xio.set_option("tls_http_proxy_TrustedCerts", proxy_server_cert_data) + underlying_xio.set_bytes_value_option(b"tls_http_proxy_TrustedCerts", proxy_server_cert_data) except ValueError: _logger.warning('Unable to set external proxy certificates.') @@ -84,7 +84,7 @@ def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_clie with open(proxy_client_cert, 'rb') as proxy_client_cert_handle: proxy_client_cert_data = proxy_client_cert_handle.read() try: - underlying_xio.set_option("tls_http_proxy_x509certificate", proxy_client_cert_data) + underlying_xio.set_bytes_value_option(b"tls_http_proxy_x509certificate", proxy_client_cert_data) except ValueError: _logger.warning('Unable to set external proxy x509certificates.') @@ -92,7 +92,7 @@ def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_clie with open(proxy_client_private_key, 'rb') as proxy_client_private_key_handle: proxy_client_private_key_data = proxy_client_private_key_handle.read() try: - underlying_xio.set_option("tls_http_proxy_x509privatekey", proxy_client_private_key_data) + underlying_xio.set_bytes_value_option(b"tls_http_proxy_x509privatekey", proxy_client_private_key_data) except ValueError: _logger.warning('Unable to set external x509privatekey.') @@ -130,7 +130,9 @@ def set_wsio(self, hostname, port, http_proxy): proxy_server_cert = http_proxy.get("proxy_verify") # TODO: allow passing a tuple of client cert/key files, order matters # check content to see if it's a key or cert file? - proxy_client_cert, proxy_client_private_key = http_proxy.get("proxy_cert") + proxy_cert = http_proxy.get("proxy_cert") + if proxy_cert is not None: + proxy_client_cert, proxy_client_private_key = proxy_cert use_tls_http_proxy = any((proxy_server_cert, proxy_client_cert, proxy_client_private_key)) _tlsio_config.set_proxy_config(proxy_config) From 147de87448d40b28e1d5dd4591df3f723bb12372 Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Wed, 21 Apr 2021 07:08:56 -0700 Subject: [PATCH 3/5] update condition check for client cert&pkey --- uamqp/authentication/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uamqp/authentication/common.py b/uamqp/authentication/common.py index 6e5fbb57c..8e7416441 100644 --- a/uamqp/authentication/common.py +++ b/uamqp/authentication/common.py @@ -67,7 +67,7 @@ def _encode(self, value): @staticmethod def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_client_cert=None, proxy_client_private_key=None): - if proxy_client_cert or proxy_client_private_key and not all((proxy_client_cert, proxy_client_private_key)): + if any((proxy_client_cert, proxy_client_private_key)) and (not all((proxy_client_cert, proxy_client_private_key))): raise ValueError("Client cert and key must both present.") underlying_xio.set_bool_value_option(b"use_tls_http_proxy", True) From 546fe0bbb479256ed9fefeb734bcb25c44900cf4 Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Wed, 21 Apr 2021 07:47:49 -0700 Subject: [PATCH 4/5] fix pylint --- uamqp/authentication/common.py | 71 ++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/uamqp/authentication/common.py b/uamqp/authentication/common.py index 8e7416441..a826f99a4 100644 --- a/uamqp/authentication/common.py +++ b/uamqp/authentication/common.py @@ -66,35 +66,44 @@ def _encode(self, value): return value.encode(self._encoding) if isinstance(value, six.text_type) else value @staticmethod - def _configure_tls_http_proxy(underlying_xio, proxy_server_cert=None, proxy_client_cert=None, proxy_client_private_key=None): - if any((proxy_client_cert, proxy_client_private_key)) and (not all((proxy_client_cert, proxy_client_private_key))): - raise ValueError("Client cert and key must both present.") - - underlying_xio.set_bool_value_option(b"use_tls_http_proxy", True) - - if proxy_server_cert: - with open(proxy_server_cert, 'rb') as proxy_server_cert_handle: - proxy_server_cert_data = proxy_server_cert_handle.read() - try: - underlying_xio.set_bytes_value_option(b"tls_http_proxy_TrustedCerts", proxy_server_cert_data) - except ValueError: - _logger.warning('Unable to set external proxy certificates.') - - if proxy_client_cert: - with open(proxy_client_cert, 'rb') as proxy_client_cert_handle: - proxy_client_cert_data = proxy_client_cert_handle.read() - try: - underlying_xio.set_bytes_value_option(b"tls_http_proxy_x509certificate", proxy_client_cert_data) - except ValueError: - _logger.warning('Unable to set external proxy x509certificates.') - - if proxy_client_private_key: - with open(proxy_client_private_key, 'rb') as proxy_client_private_key_handle: - proxy_client_private_key_data = proxy_client_private_key_handle.read() - try: - underlying_xio.set_bytes_value_option(b"tls_http_proxy_x509privatekey", proxy_client_private_key_data) - except ValueError: - _logger.warning('Unable to set external x509privatekey.') + def _configure_tls_http_proxy( + underlying_xio, + proxy_server_cert=None, + proxy_client_cert=None, + proxy_client_private_key=None + ): + if any((proxy_client_cert, proxy_client_private_key)) and\ + (not all((proxy_client_cert, proxy_client_private_key))): + raise ValueError("Client cert and key must both present.") + + underlying_xio.set_bool_value_option(b"use_tls_http_proxy", True) + + proxy_server_cert = proxy_server_cert or certifi.where() + with open(proxy_server_cert, 'rb') as proxy_server_cert_handle: + proxy_server_cert_data = proxy_server_cert_handle.read() + try: + underlying_xio.set_bytes_value_option(b"tls_http_proxy_TrustedCerts", proxy_server_cert_data) + except ValueError: + _logger.warning('Unable to set external proxy certificates.') + + if proxy_client_cert: + with open(proxy_client_cert, 'rb') as proxy_client_cert_handle: + proxy_client_cert_data = proxy_client_cert_handle.read() + try: + underlying_xio.set_bytes_value_option(b"tls_http_proxy_x509certificate", proxy_client_cert_data) + except ValueError: + _logger.warning('Unable to set external proxy x509certificates.') + + if proxy_client_private_key: + with open(proxy_client_private_key, 'rb') as proxy_client_private_key_handle: + proxy_client_private_key_data = proxy_client_private_key_handle.read() + try: + underlying_xio.set_bytes_value_option( + b"tls_http_proxy_x509privatekey", + proxy_client_private_key_data + ) + except ValueError: + _logger.warning('Unable to set external x509privatekey.') def set_io(self, hostname, port, http_proxy, transport_type): if transport_type == TransportType.AmqpOverWebsocket or http_proxy is not None: @@ -129,7 +138,9 @@ def set_wsio(self, hostname, port, http_proxy): proxy_config = self._build_proxy_config(hostname, port, http_proxy) proxy_server_cert = http_proxy.get("proxy_verify") # TODO: allow passing a tuple of client cert/key files, order matters - # check content to see if it's a key or cert file? + # assuming the first one is client certificate, the second is the client private key + # should we 1. be smart and check content to see if it's a key or cert file? follow requests + # 2. or split the argument to proxy_cert_certificates/proxy_cert_private_key proxy_cert = http_proxy.get("proxy_cert") if proxy_cert is not None: proxy_client_cert, proxy_client_private_key = proxy_cert From 70f78f143cd9719913306c8610e9f8e468f18988 Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Fri, 23 Apr 2021 09:41:44 -0700 Subject: [PATCH 5/5] raise Error on invalid cert input --- uamqp/authentication/common.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/uamqp/authentication/common.py b/uamqp/authentication/common.py index a826f99a4..a3374d5d3 100644 --- a/uamqp/authentication/common.py +++ b/uamqp/authentication/common.py @@ -137,12 +137,14 @@ def set_wsio(self, hostname, port, http_proxy): if http_proxy: proxy_config = self._build_proxy_config(hostname, port, http_proxy) proxy_server_cert = http_proxy.get("proxy_verify") - # TODO: allow passing a tuple of client cert/key files, order matters - # assuming the first one is client certificate, the second is the client private key - # should we 1. be smart and check content to see if it's a key or cert file? follow requests - # 2. or split the argument to proxy_cert_certificates/proxy_cert_private_key proxy_cert = http_proxy.get("proxy_cert") if proxy_cert is not None: + if not (isinstance(proxy_cert, tuple) and len(proxy_cert) == 2): + raise ValueError( + "proxy_cert must be a tuple containing both of certificate and private key file path", + ", and certificate file path must be put in front of the private key file path. ", + "E.g. proxy_cert=(, )" + ) proxy_client_cert, proxy_client_private_key = proxy_cert use_tls_http_proxy = any((proxy_server_cert, proxy_client_cert, proxy_client_private_key)) _tlsio_config.set_proxy_config(proxy_config)