diff --git a/helm/polaris/README.md b/helm/polaris/README.md index f67736614f..e6707091ff 100644 --- a/helm/polaris/README.md +++ b/helm/polaris/README.md @@ -48,7 +48,7 @@ A Helm chart for Apache Polaris (incubating). ### Optional -When using EclipseLink backed metastore a custom `persistence.xml` is required, a Kubernetes Secret must be created for it. Below is a sample command: +When using a custom `persistence.xml`, a Kubernetes Secret must be created for it. Below is a sample command: ```bash kubectl create secret generic polaris-secret -n polaris --from-file=persistence.xml ``` @@ -67,7 +67,7 @@ helm unittest helm/polaris The below instructions assume Kind and Helm are installed. Simply run the `run.sh` script from the Polaris repo root, making sure to specify the -`--eclipse-link-deps` if using EclipseLink backed metastore, option: +`--eclipse-link-deps` option: ```bash ./run.sh @@ -186,8 +186,8 @@ kubectl delete namespace polaris ## Values - Key | Type | Default | Description | -|-----|------|-----|-------------| +| Key | Type | Default | Description | +|-----|------|---------|-------------| | advancedConfig | object | `{}` | Advanced configuration. You can pass here any valid Polaris or Quarkus configuration property. Any property that is defined here takes precedence over all the other configuration values generated by this chart. Properties can be passed "flattened" or as nested YAML objects (see examples below). Note: values should be strings; avoid using numbers, booleans, or other types. | | affinity | object | `{}` | Affinity and anti-affinity for polaris pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity. | | authentication | object | `{"authenticator":{"type":"default"},"tokenBroker":{"maxTokenGeneration":"PT1H","secret":{"name":null,"privateKey":"private.pem","publicKey":"public.pem","secretKey":"secret"},"type":"rsa-key-pair"},"tokenService":{"type":"default"}}` | Polaris authentication configuration. | @@ -343,4 +343,4 @@ kubectl delete namespace polaris | tracing.attributes | object | `{}` | Resource attributes to identify the polaris service among other tracing sources. See https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#service. If left empty, traces will be attached to a service named "Apache Polaris"; to change this, provide a service.name attribute here. | | tracing.enabled | bool | `false` | Specifies whether tracing for the polaris server should be enabled. | | tracing.endpoint | string | `"http://otlp-collector:4317"` | The collector endpoint URL to connect to (required). The endpoint URL must have either the http:// or the https:// scheme. The collector must talk the OpenTelemetry protocol (OTLP) and the port must be its gRPC port (by default 4317). See https://quarkus.io/guides/opentelemetry for more information. | -| tracing.sample | string | `"1.0d"` | Which requests should be sampled. Valid values are: "all", "none", or a ratio between 0.0 and "1.0d" (inclusive). E.g. "0.5d" means that 50% of the requests will be sampled. Note: avoid entering numbers here, always prefer a string representation of the ratio. | \ No newline at end of file +| tracing.sample | string | `"1.0d"` | Which requests should be sampled. Valid values are: "all", "none", or a ratio between 0.0 and "1.0d" (inclusive). E.g. "0.5d" means that 50% of the requests will be sampled. Note: avoid entering numbers here, always prefer a string representation of the ratio. | diff --git a/helm/polaris/templates/configmap.yaml b/helm/polaris/templates/configmap.yaml index 58ba96c0b6..ae732760cb 100644 --- a/helm/polaris/templates/configmap.yaml +++ b/helm/polaris/templates/configmap.yaml @@ -36,12 +36,12 @@ data: {{- $_ = set $map "polaris.realm-context.realms" (join "," .Values.realmContext.realms) -}} {{- /* Features */ -}} - {{- range $k, $v := .Values.features.defaults -}} - {{- $_ = set $map (printf "polaris.features.defaults.\"%s\"" $k) (toJson $v) -}} + {{- range $k, $v := .Values.features -}} + {{- $_ = set $map (printf "polaris.features.\"%s\"" $k) (toJson $v) -}} {{- end -}} {{- range $realm, $overrides := .Values.features.realmOverrides -}} {{- range $k, $v := $overrides -}} - {{- $_ = set $map (printf "polaris.config.realm-overrides.\"%s\".\"%s\"" $realm $k) (toJson $v) -}} + {{- $_ = set $map (printf "polaris.features.realm-overrides.\"%s\".\"%s\"" $realm $k) (toJson $v) -}} {{- end -}} {{- end -}} diff --git a/helm/polaris/tests/configmap_test.yaml b/helm/polaris/tests/configmap_test.yaml index ef725ec4f3..562adad220 100644 --- a/helm/polaris/tests/configmap_test.yaml +++ b/helm/polaris/tests/configmap_test.yaml @@ -89,19 +89,18 @@ tests: - it: should configure features set: features: - defaults: - feature1: true - feature2: 42 + feature1: true + feature2: 42 realmOverrides: realm1: feature1: false realm2: feature2: 43 asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.defaults.\"feature1\"=true" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.defaults.\"feature2\"=42" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.config.realm-overrides.\"realm1\".\"feature1\"=false" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.config.realm-overrides.\"realm2\".\"feature2\"=43" } + - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.\"feature1\"=true" } + - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.\"feature2\"=42" } + - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.realm-overrides.\"realm1\".\"feature1\"=false" } + - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.features.realm-overrides.\"realm2\".\"feature2\"=43" } - it: should configure persistence set: diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java index c99ff780ee..fd983e6e88 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java @@ -177,10 +177,17 @@ public Builder catalogConfigUnsafe(String catalogConfig) { return this; } - public FeatureConfiguration buildFeatureConfiguration() { + private void validateOrThrow() { if (key == null || description == null || defaultValue == null) { throw new IllegalArgumentException("key, description, and defaultValue are required"); } + if (key.contains(".")) { + throw new IllegalArgumentException("key cannot contain `.`"); + } + } + + public FeatureConfiguration buildFeatureConfiguration() { + validateOrThrow(); FeatureConfiguration config = new FeatureConfiguration<>( key, description, defaultValue, catalogConfig, catalogConfigUnsafe); @@ -189,9 +196,7 @@ public FeatureConfiguration buildFeatureConfiguration() { } public BehaviorChangeConfiguration buildBehaviorChangeConfiguration() { - if (key == null || description == null || defaultValue == null) { - throw new IllegalArgumentException("key, description, and defaultValue are required"); - } + validateOrThrow(); if (catalogConfig.isPresent() || catalogConfigUnsafe.isPresent()) { throw new IllegalArgumentException( "catalog configs are not valid for behavior change configs"); diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index 4419d4d590..b172114cab 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -31,16 +31,16 @@ polaris.persistence.type=in-memory polaris.secrets-manager.type=in-memory -polaris.features.defaults."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false -polaris.features.defaults."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false -polaris.features.defaults."ALLOW_OVERLAPPING_CATALOG_URLS"=true -polaris.features.defaults."ALLOW_SPECIFYING_FILE_IO_IMPL"=true -polaris.features.defaults."ALLOW_WILDCARD_LOCATION"=true -polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true -polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true -polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true -polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] -polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true +polaris.features."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false +polaris.features."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false +polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true +polaris.features."ALLOW_SPECIFYING_FILE_IO_IMPL"=true +polaris.features."ALLOW_WILDCARD_LOCATION"=true +polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true +polaris.features."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true +polaris.features."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true +polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.realm-context.realms=POLARIS,OTHER diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 11f23de1b8..64f6f24742 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -108,10 +108,10 @@ polaris.realm-context.realms=POLARIS polaris.realm-context.header-name=Polaris-Realm polaris.realm-context.require-header=false -polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] -# polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true -polaris.features.defaults."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] +polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false +polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +# polaris.features."ENABLE_CATALOG_FEDERATION"=true +polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # realm overrides # polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusBehaviorChangesConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusBehaviorChangesConfiguration.java index 1e4fe946e0..8e4d059f5c 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusBehaviorChangesConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusBehaviorChangesConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithParentName; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -33,9 +34,16 @@ // QuarkusFeaturesConfiguration public interface QuarkusBehaviorChangesConfiguration { + @WithParentName Map defaults(); - Map realmOverrides(); + Map realmOverrides(); + + interface QuarkusRealmOverrides extends FeaturesConfiguration.RealmOverrides { + @WithParentName + @Override + Map overrides(); + } default Map parseDefaults(ObjectMapper objectMapper) { return convertMap(objectMapper, defaults()); diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java index 2ae58c8fe7..cbcfbc0576 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java @@ -19,15 +19,23 @@ package org.apache.polaris.service.quarkus.config; import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithParentName; import java.util.Map; import org.apache.polaris.service.config.FeaturesConfiguration; @ConfigMapping(prefix = "polaris.features") public interface QuarkusFeaturesConfiguration extends FeaturesConfiguration { + @WithParentName @Override Map defaults(); @Override - Map realmOverrides(); + Map realmOverrides(); + + interface QuarkusRealmOverrides extends RealmOverrides { + @WithParentName + @Override + Map overrides(); + } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedBehaviorChangesConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedBehaviorChangesConfiguration.java new file mode 100644 index 0000000000..b4a4078a38 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedBehaviorChangesConfiguration.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Wraps around {@link QuarkusBehaviorChangesConfiguration} but removes properties from `defaults` + * that shouldn't be there + */ +@ApplicationScoped +@Alternative +@Priority(1) +public class QuarkusResolvedBehaviorChangesConfiguration + implements QuarkusBehaviorChangesConfiguration { + + private final Map cleanedDefaults; + private final Map + realmOverrides; + + public QuarkusResolvedBehaviorChangesConfiguration(QuarkusBehaviorChangesConfiguration raw) { + this.realmOverrides = raw.realmOverrides(); + + // Filter out any keys that look like realm overrides + this.cleanedDefaults = + raw.defaults().entrySet().stream() + .filter(e -> e.getKey().split("\\.").length == 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public Map defaults() { + return cleanedDefaults; + } + + @Override + public Map + realmOverrides() { + return realmOverrides; + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedFeaturesConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedFeaturesConfiguration.java new file mode 100644 index 0000000000..567c30972e --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusResolvedFeaturesConfiguration.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.polaris.service.config.FeaturesConfiguration; + +/** + * Wraps around {@link QuarkusFeaturesConfiguration} but removes properties from `defaults` that + * shouldn't be there + */ +@ApplicationScoped +@Alternative +@Priority(1) +public class QuarkusResolvedFeaturesConfiguration implements FeaturesConfiguration { + + private final Map cleanedDefaults; + private final Map realmOverrides; + + public QuarkusResolvedFeaturesConfiguration(QuarkusFeaturesConfiguration raw) { + this.realmOverrides = raw.realmOverrides(); + + // Filter out any keys that look like realm overrides + this.cleanedDefaults = + raw.defaults().entrySet().stream() + .filter(e -> e.getKey().split("\\.").length == 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public Map defaults() { + return cleanedDefaults; + } + + @Override + public Map realmOverrides() { + return realmOverrides; + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 91f0a33d61..b5ef68a6bd 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -113,9 +113,9 @@ public Set> getEnabledAlternatives() { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", + "polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java index a296feca64..04c84c5f60 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -106,11 +106,11 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", + "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", - "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 08dd790e90..cbfaa41f3b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -88,9 +88,9 @@ public static class Profile extends PolarisAuthzTestBase.Profile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", + "polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index a2d50d9a4f..b54d032a81 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -169,13 +169,13 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", + "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", - "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]", - "polaris.features.defaults.\"LIST_PAGINATION_ENABLED\"", + "polaris.features.\"LIST_PAGINATION_ENABLED\"", "true", "polaris.event-listener.type", "test"); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index a7bdec4f10..8b87b75a74 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -100,15 +100,15 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_WILDCARD_LOCATION\"", + "polaris.features.\"ALLOW_WILDCARD_LOCATION\"", "true", - "polaris.features.defaults.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true", - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", + "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", - "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]", "polaris.event-listener.type", "test"); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 0ea91bb19c..308971c7cd 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -121,11 +121,11 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", + "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", - "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index 04f4a8680e..e533d52142 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -20,15 +20,74 @@ import static org.assertj.core.api.Assertions.assertThat; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.time.Clock; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; +import org.apache.polaris.service.config.FeaturesConfiguration; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +@QuarkusTest +@TestProfile(DefaultConfigurationStoreTest.Profile.class) public class DefaultConfigurationStoreTest { + private static final String falseByDefaultKey = "ALLOW_SPECIFYING_FILE_IO_IMPL"; + private static final String trueByDefaultKey = "ENABLE_GENERIC_TABLES"; + private static final String realmOne = "realm1"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.realm-context.realms", + "realm1,realm2", + String.format("polaris.features.\"%s\"", falseByDefaultKey), + "true", + String.format("polaris.features.\"%s\"", trueByDefaultKey), + "false", + String.format( + "polaris.features.realm-overrides.\"%s\".\"%s\"", realmOne, falseByDefaultKey), + "false"); + } + } + + private PolarisCallContext polarisContext; + + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagServices; + @Inject FeaturesConfiguration featuresConfiguration; + + @BeforeEach + public void before(TestInfo testInfo) { + String realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(java.lang.reflect.Method::getName).orElse("test"), + System.nanoTime()); + RealmContext realmContext = () -> realmName; + QuarkusMock.installMockForType(realmContext, RealmContext.class); + polarisContext = + new PolarisCallContext( + managerFactory.getOrCreateSessionSupplier(realmContext).get(), + diagServices, + configurationStore, + Clock.systemDefaultZone()); + } @Test public void testGetConfiguration() { @@ -111,4 +170,64 @@ public void testGetRealmConfiguration() { String keyTwoRealm3 = defaultConfigurationStore.getConfiguration(realm3Ctx, "key2"); assertThat(keyTwoRealm3).isEqualTo(defaultKeyTwoValue); } + + // TODO simplify once DefaultConfigrationStore doesn't rely on CallContext.getCurrentContext + private void setCurrentRealm(String realmName) { + RealmContext realmContext = () -> realmName; + QuarkusMock.installMockForType(realmContext, RealmContext.class); + CallContext.setCurrentContext( + new CallContext() { + @Override + public RealmContext getRealmContext() { + return realmContext; + } + + @Override + public PolarisCallContext getPolarisCallContext() { + return CallContext.getCurrentContext().getPolarisCallContext(); + } + + @Override + public Map contextVariables() { + return CallContext.getCurrentContext().contextVariables(); + } + }); + } + + @Test + public void testInjectedConfigurationStore() { + // Feature override makes this `false` + boolean featureOverrideValue = + configurationStore.getConfiguration(polarisContext, trueByDefaultKey); + assertThat(featureOverrideValue).isFalse(); + + // Feature override value makes this `true` + setCurrentRealm("not-" + realmOne); + boolean realmOverrideValue = + configurationStore.getConfiguration(polarisContext, falseByDefaultKey); + assertThat(realmOverrideValue).isTrue(); + + // Now, realm override value makes this `false` + setCurrentRealm(realmOne); + realmOverrideValue = configurationStore.getConfiguration(polarisContext, falseByDefaultKey); + assertThat(realmOverrideValue).isFalse(); + + assertThat(configurationStore).isInstanceOf(DefaultConfigurationStore.class); + } + + @Test + public void testInjectedFeaturesConfiguration() { + assertThat(featuresConfiguration).isInstanceOf(QuarkusResolvedFeaturesConfiguration.class); + + assertThat(featuresConfiguration.defaults()) + .containsKeys(falseByDefaultKey, trueByDefaultKey) + .allSatisfy((key, value) -> assertThat(value).doesNotContain(realmOne)); + + assertThat(featuresConfiguration.realmOverrides()).hasSize(1); + assertThat(featuresConfiguration.realmOverrides()).containsKey(realmOne); + + assertThat(featuresConfiguration.realmOverrides().get(realmOne).overrides()).hasSize(1); + assertThat(featuresConfiguration.realmOverrides().get(realmOne).overrides()) + .containsKey(falseByDefaultKey); + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java index c98da4d5a9..594efc2219 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java @@ -53,8 +53,8 @@ public Map getConfigOverrides() { return Map.of( "quarkus.http.limits.max-body-size", "1000000", "polaris.realm-context.realms", "POLARIS,OTHER", - "polaris.features.defaults.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true", - "polaris.features.defaults.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true"); + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusManagementServiceIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusManagementServiceIntegrationTest.java index 33735636d5..88421b8fed 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusManagementServiceIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusManagementServiceIntegrationTest.java @@ -34,9 +34,9 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of( - "polaris.features.defaults.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true", - "polaris.features.defaults.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", "true", "polaris.storage.gcp.token", "token", diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java index 11dd821ef2..27e29fd613 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java @@ -32,8 +32,7 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of( - "polaris.features.defaults.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false"); + return Map.of("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false"); } } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAwsIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAwsIntegrationTest.java index b86c313a15..16380c24de 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAwsIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAwsIntegrationTest.java @@ -38,7 +38,7 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]"); + return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAzureIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAzureIntegrationTest.java index 8cc89b9c6b..49ff842b7d 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAzureIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewAzureIntegrationTest.java @@ -37,7 +37,7 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"AZURE\"]"); + return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"AZURE\"]"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java index 4444bcbee3..3987e94d28 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java @@ -36,7 +36,7 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); + return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewGcpIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewGcpIntegrationTest.java index 75168043b8..ef04d745e0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewGcpIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewGcpIntegrationTest.java @@ -36,7 +36,7 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"GCS\"]"); + return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"GCS\"]"); } } diff --git a/site/content/in-dev/unreleased/configuration.md b/site/content/in-dev/unreleased/configuration.md index a06cb12b90..e9802c2b28 100644 --- a/site/content/in-dev/unreleased/configuration.md +++ b/site/content/in-dev/unreleased/configuration.md @@ -88,8 +88,8 @@ read-only mode, as Polaris only reads the configuration file once, at startup. | `polaris.realm-context.type` | `default` | Define the type of the Polaris realm to use. | | `polaris.realm-context.realms` | `POLARIS` | Define the list of realms to use. | | `polaris.realm-context.header-name` | `Polaris-Realm` | Define the header name defining the realm context. | -| `polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"` | `false` | Flag to enforce check if credential rotation. | -| `polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"` | `FILE` | Define the catalog supported storage. Supported values are `S3`, `GCS`, `AZURE`, `FILE`. | +| `polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"` | `false` | Flag to enforce check if credential rotation. | +| `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"` | `FILE` | Define the catalog supported storage. Supported values are `S3`, `GCS`, `AZURE`, `FILE`. | | `polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"` | `true` | "Override" realm features, here the catalog init default flag. | | `polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"` | `true` | "Override" realm features, here the skip credential subscoping indirection flag. | | `polaris.authentication.authenticator.type` | `default` | Define the Polaris authenticator type. | @@ -138,8 +138,6 @@ There are non Polaris configuration properties that can be useful: | `quarkus.management.port` | `8182` | Define the port number of the Polaris management server. | | `quarkus.management.root-path` | | Define the root path where `/metrics` and `/health` endpoints are based on. | | `quarkus.otel.sdk.disabled` | `true` | Enable the OpenTelemetry layer. | - -## Java Runtime Configuration > Note: This section is only relevant for Polaris Docker images and Kubernetes deployments. diff --git a/site/content/in-dev/unreleased/configuring-polaris-for-production.md b/site/content/in-dev/unreleased/configuring-polaris-for-production.md index d38dd33810..84a3cec5eb 100644 --- a/site/content/in-dev/unreleased/configuring-polaris-for-production.md +++ b/site/content/in-dev/unreleased/configuring-polaris-for-production.md @@ -213,7 +213,7 @@ curl -X POST http://localhost:8181/api/catalog/v1/oauth/tokens \ When deploying Polaris in production, consider adjusting the following configurations: -#### `polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"` +#### `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"` - By default, Polaris catalogs are allowed to be located in local filesystem with the `FILE` storage type. This should be disabled for production systems.