From 29e9049e1c0f3d14533ad619837d75072891d36a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 15 Jul 2022 13:56:13 -0400 Subject: [PATCH 01/14] Add github test skeleton --- .github/workflows/object_store.yml | 96 ++++++++++++++++++++++++++++++ object_store/src/azure.rs | 49 ++++++++++++++- object_store/src/gcp.rs | 16 ++++- 3 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/object_store.yml diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml new file mode 100644 index 000000000000..99a078712a90 --- /dev/null +++ b/.github/workflows/object_store.yml @@ -0,0 +1,96 @@ +--- +name: "object_store tests" + +on: + pull_request: + +jobs: + # This job functions as "proof of life" that things are working --no need to port + lint: + name: Lint (cargo fmt) + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v2 + - name: Setup toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Run + run: cargo fmt --all -- --check + + # test the crate + linux-test: + name: Emulator Tests + runs-on: ubuntu-latest + services: + fake-gcs: + image: fsouza/fake-gcs-server + ports: + - 4443:4443 + localstack: + image: localstack/localstack:0.14.4 + ports: + - 4566:4566 + azurite: + image: mcr.microsoft.com/azure-storage/azurite + ports: + - 10000:10002 + container: + image: amd64/rust + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + RUST_BACKTRACE: "1" + # Run integration tests + TEST_INTEGRATION: 1 + AWS_DEFAULT_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_ENDPOINT: http://localstack:4566 + AZURE_USE_EMULATOR: "1" + AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" + AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001" + GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" + GCP_ACCEPT_INVALID_CERTIFICATES: "true" + OBJECT_STORE_BUCKET: test-bucket + + steps: + - uses: actions/checkout@v2 + + - name: Configure Fake GCS Server (GCP emulation) + # the magical connection string is from + # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings + run: | + curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b" + echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" + + - name: Setup LocalStack (AWS emulation) + run: | + cd /tmp + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install + aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket + + - name: Configure Azurite (Azure emulation) + # the magical connection string is from + # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;' + + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + + - name: Run tests + run: | + # run tests + cargo test --workspace --features=aws,azure,azure_test,gcp diff --git a/object_store/src/azure.rs b/object_store/src/azure.rs index 5f4327982cd0..9bca7e8bdef9 100644 --- a/object_store/src/azure.rs +++ b/object_store/src/azure.rs @@ -38,6 +38,8 @@ use futures::{ use snafu::{ResultExt, Snafu}; use std::collections::BTreeSet; use std::{convert::TryInto, sync::Arc}; +use tokio::io::AsyncWrite; +use url::Url; /// A specialized `Error` for Azure object store-related errors #[derive(Debug, Snafu)] @@ -158,6 +160,18 @@ enum Error { "Azurite (azure emulator) support not compiled in, please add `azure_test` feature" ))] NoEmulatorFeature, + + #[snafu(display( + "Unable parse emulator url {}={}, Error: {}", + env_name, + env_value, + source + ))] + UnableToParseEmulatorUrl { + env_name: String, + env_value: String, + source: url::ParseError, + }, } impl From for super::Error { @@ -507,6 +521,19 @@ fn check_if_emulator_works() -> Result<()> { Err(Error::NoEmulatorFeature.into()) } +/// Parses the contents of the environment variable `env_name` as a URL +/// if present, otherwise falls back to default_url +fn url_from_env(env_name: &str, default_url: &str) -> Result { + let url = match std::env::var(env_name) { + Ok(env_value) => Url::parse(&env_value).context(UnableToParseEmulatorUrlSnafu { + env_name, + env_value, + })?, + Err(_) => Url::parse(default_url).expect("Failed to parse default URL"), + }; + Ok(url) +} + /// Configure a connection to container with given name on Microsoft Azure /// Blob store. /// @@ -524,7 +551,25 @@ pub fn new_azure( let (is_emulator, storage_account_client) = if use_emulator { check_if_emulator_works()?; - (true, StorageAccountClient::new_emulator_default()) + // Allow overriding defaults. Values taken from + // from https://docs.rs/azure_storage/0.2.0/src/azure_storage/core/clients/storage_account_client.rs.html#129-141 + let http_client = azure_core::new_http_client(); + let blob_storage_url = url_from_env("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000")?; + let queue_storage_url = + url_from_env("AZURITE_QUEUE_STORAGE_URL", "http://127.0.0.1:10001")?; + let table_storage_url = + url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10002")?; + let filesystem_url = url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10004")?; + + let storage_client = StorageAccountClient::new_emulator( + http_client, + &blob_storage_url, + &table_storage_url, + &queue_storage_url, + &filesystem_url, + ); + + (true, storage_client) } else { ( false, @@ -544,6 +589,8 @@ pub fn new_azure( .trim_end_matches('/') .to_string(); + println!("AAL using remote blob base URL: {}", blob_base_url); + let container_name = container_name.into(); let container_client = storage_client.as_container_client(&container_name); diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs index 84fb572bd77b..f727d2cb1dcc 100644 --- a/object_store/src/gcp.rs +++ b/object_store/src/gcp.rs @@ -73,6 +73,9 @@ enum Error { #[snafu(display("Error decoding object size: {}", source))] InvalidSize { source: std::num::ParseIntError }, + + #[snafu(display("Error creating http client: {}", source))] + CreatingHttpClient { source: reqwest::Error }, } impl From for super::Error { @@ -498,13 +501,24 @@ fn reader_credentials_file( Ok(serde_json::from_reader(reader).context(DecodeCredentialsSnafu)?) } +fn get_client() -> Result { + let mut builder = Client::builder(); + + if std::env::var("GCP_ACCEPT_INVALID_CERTIFICATES").is_ok() { + println!("WARNING: accepting invalid certifcates"); + builder = builder.danger_accept_invalid_certs(true); + } + let client = builder.build().context(CreatingHttpClientSnafu)?; + Ok(client) +} + /// Configure a connection to Google Cloud Storage. pub fn new_gcs( service_account_path: impl AsRef, bucket_name: impl Into, ) -> Result { let credentials = reader_credentials_file(service_account_path)?; - let client = Client::new(); + let client = get_client()?; // TODO: https://cloud.google.com/storage/docs/authentication#oauth-scopes let scope = "https://www.googleapis.com/auth/devstorage.full_control"; From fdff42dcdc4d4f42380bd3a048de78e5139f1a62 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 14:55:47 -0500 Subject: [PATCH 02/14] Cleanups and fmt --- object_store/src/azure.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/object_store/src/azure.rs b/object_store/src/azure.rs index 9bca7e8bdef9..ec65b18c7241 100644 --- a/object_store/src/azure.rs +++ b/object_store/src/azure.rs @@ -525,10 +525,12 @@ fn check_if_emulator_works() -> Result<()> { /// if present, otherwise falls back to default_url fn url_from_env(env_name: &str, default_url: &str) -> Result { let url = match std::env::var(env_name) { - Ok(env_value) => Url::parse(&env_value).context(UnableToParseEmulatorUrlSnafu { - env_name, - env_value, - })?, + Ok(env_value) => { + Url::parse(&env_value).context(UnableToParseEmulatorUrlSnafu { + env_name, + env_value, + })? + } Err(_) => Url::parse(default_url).expect("Failed to parse default URL"), }; Ok(url) @@ -554,12 +556,14 @@ pub fn new_azure( // Allow overriding defaults. Values taken from // from https://docs.rs/azure_storage/0.2.0/src/azure_storage/core/clients/storage_account_client.rs.html#129-141 let http_client = azure_core::new_http_client(); - let blob_storage_url = url_from_env("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000")?; + let blob_storage_url = + url_from_env("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000")?; let queue_storage_url = url_from_env("AZURITE_QUEUE_STORAGE_URL", "http://127.0.0.1:10001")?; let table_storage_url = url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10002")?; - let filesystem_url = url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10004")?; + let filesystem_url = + url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10004")?; let storage_client = StorageAccountClient::new_emulator( http_client, @@ -589,8 +593,6 @@ pub fn new_azure( .trim_end_matches('/') .to_string(); - println!("AAL using remote blob base URL: {}", blob_base_url); - let container_name = container_name.into(); let container_client = storage_client.as_container_client(&container_name); From 5f17eb9f20449b990c92d95e945f2aa48d546dce Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 14:59:26 -0500 Subject: [PATCH 03/14] Run on changes to object_store --- .github/workflows/object_store.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 99a078712a90..eb323fe0fbe6 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -1,8 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + --- name: "object_store tests" on: pull_request: + paths: + # Only run when object store files or github workflows change + - object_store/** + -.github/** jobs: # This job functions as "proof of life" that things are working --no need to port From d863a0dc1e66d66ae197fb7effc153689db5b624 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:13:17 -0500 Subject: [PATCH 04/14] Update name --- .github/workflows/object_store.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index eb323fe0fbe6..bd3e27f937c4 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -16,7 +16,7 @@ # under the License. --- -name: "object_store tests" +name: "Object Store" on: pull_request: From ab251f9bfd608ad560f2fef29bc333fe74fc8ef0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:14:23 -0500 Subject: [PATCH 05/14] Broken yaml? --- .github/workflows/object_store.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index bd3e27f937c4..4c55271aea07 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -23,7 +23,7 @@ on: paths: # Only run when object store files or github workflows change - object_store/** - -.github/** + - .github/** jobs: # This job functions as "proof of life" that things are working --no need to port From e32289fe11eac58b5af994802f6799862c3d742f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:15:31 -0500 Subject: [PATCH 06/14] Remove uneeded lint job --- .github/workflows/object_store.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 4c55271aea07..dc11da57b037 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -26,22 +26,6 @@ on: - .github/** jobs: - # This job functions as "proof of life" that things are working --no need to port - lint: - name: Lint (cargo fmt) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v2 - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt - - name: Run - run: cargo fmt --all -- --check - # test the crate linux-test: name: Emulator Tests From b5e5f709e25e0f6e32a74cf9ca7db000da87f023 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:26:25 -0500 Subject: [PATCH 07/14] Run only object store tests --- .github/workflows/object_store.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index dc11da57b037..c64ce160cd99 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -95,7 +95,7 @@ jobs: rustup toolchain install stable rustup default stable - - name: Run tests + - name: Run object_store tests run: | # run tests - cargo test --workspace --features=aws,azure,azure_test,gcp + cargo test -p object_store --features=aws,azure,azure_test,gcp From bf1729e0b0fd23e4b00276398317a39e97fa416a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:50:02 -0500 Subject: [PATCH 08/14] Add local gcp test instructions --- object_store/CONTRIBUTING.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/object_store/CONTRIBUTING.md b/object_store/CONTRIBUTING.md index 2e216dd48662..7c2832cf7ef1 100644 --- a/object_store/CONTRIBUTING.md +++ b/object_store/CONTRIBUTING.md @@ -90,5 +90,24 @@ $ cargo test --features azure ### GCP -We don't have a good story yet for testing the GCP integration locally. You will need to create a GCS bucket, a -service account that has access to it, and use this to run the tests. +To test the GCS integration, we use [Fake GCS Server](https://github.com/fsouza/fake-gcs-server) + +Startup the fake server: + +```shell +docker run -p 4443:4443 fsouza/fake-gcs-server +``` + +Configure the account: +```shell +curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://localhost:4443/storage/v1/b" +echo '{"gcs_base_url": "https://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > /tmp/gcs.json +``` + +Now run the tests: +```shell +TEST_INTEGRATION=1 \ +OBJECT_STORE_BUCKET=test-bucket \ +GOOGLE_SERVICE_ACCOUNT=/tmp/gcs.json \ +cargo test -p object_store --features=gcp +``` From 56e46ca67e013c2b84cbb59a1fb8bf6a37a89ba0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 15:59:32 -0500 Subject: [PATCH 09/14] Allow custom http client for gcs --- .github/workflows/object_store.yml | 3 -- object_store/src/gcp.rs | 45 ++++++++++++++++++------------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index c64ce160cd99..359406292748 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -62,15 +62,12 @@ jobs: AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001" GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" - GCP_ACCEPT_INVALID_CERTIFICATES: "true" OBJECT_STORE_BUCKET: test-bucket steps: - uses: actions/checkout@v2 - name: Configure Fake GCS Server (GCP emulation) - # the magical connection string is from - # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings run: | curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b" echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs index f727d2cb1dcc..00ee11cd5876 100644 --- a/object_store/src/gcp.rs +++ b/object_store/src/gcp.rs @@ -501,24 +501,21 @@ fn reader_credentials_file( Ok(serde_json::from_reader(reader).context(DecodeCredentialsSnafu)?) } -fn get_client() -> Result { - let mut builder = Client::builder(); - - if std::env::var("GCP_ACCEPT_INVALID_CERTIFICATES").is_ok() { - println!("WARNING: accepting invalid certifcates"); - builder = builder.danger_accept_invalid_certs(true); - } - let client = builder.build().context(CreatingHttpClientSnafu)?; - Ok(client) -} - /// Configure a connection to Google Cloud Storage. pub fn new_gcs( service_account_path: impl AsRef, bucket_name: impl Into, +) -> Result { + new_gcs_with_client(service_account_path, bucket_name, Client::new()) +} + +/// Configure a connection to Google Cloud Storage with the specified HTTP client. +pub fn new_gcs_with_client( + service_account_path: impl AsRef, + bucket_name: impl Into, + client: Client, ) -> Result { let credentials = reader_credentials_file(service_account_path)?; - let client = get_client()?; // TODO: https://cloud.google.com/storage/docs/authentication#oauth-scopes let scope = "https://www.googleapis.com/auth/devstorage.full_control"; @@ -589,6 +586,18 @@ mod test { service_account: String, } + impl GoogleCloudConfig { + fn build(self) -> Result { + // ignore HTTPS errors in tests so we can use fake-gcs server + let client = Client::builder() + .danger_accept_invalid_certs(true) + .build() + .expect("Error creating http client for testing"); + + new_gcs_with_client(self.service_account, self.bucket, client) + } + } + // Helper macro to skip tests if TEST_INTEGRATION and the GCP environment variables are not set. macro_rules! maybe_skip_integration { () => {{ @@ -636,7 +645,7 @@ mod test { #[tokio::test] async fn gcs_test() { let config = maybe_skip_integration!(); - let integration = new_gcs(config.service_account, config.bucket).unwrap(); + let integration = config.build().unwrap(); put_get_delete_list(&integration).await.unwrap(); list_uses_directories_correctly(&integration).await.unwrap(); @@ -647,7 +656,7 @@ mod test { #[tokio::test] async fn gcs_test_get_nonexistent_location() { let config = maybe_skip_integration!(); - let integration = new_gcs(config.service_account, &config.bucket).unwrap(); + let integration = config.build().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -664,7 +673,7 @@ mod test { async fn gcs_test_get_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = new_gcs(config.service_account, &config.bucket).unwrap(); + let integration = config.build().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -682,7 +691,7 @@ mod test { #[tokio::test] async fn gcs_test_delete_nonexistent_location() { let config = maybe_skip_integration!(); - let integration = new_gcs(config.service_account, &config.bucket).unwrap(); + let integration = config.build().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -698,7 +707,7 @@ mod test { async fn gcs_test_delete_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = new_gcs(config.service_account, &config.bucket).unwrap(); + let integration = config.build().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -714,7 +723,7 @@ mod test { async fn gcs_test_put_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = new_gcs(config.service_account, &config.bucket).unwrap(); + let integration = config.build().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); let data = Bytes::from("arbitrary data"); From d17fd079cb0968533ca3f1da3942d70f64aeda3e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 16:00:15 -0500 Subject: [PATCH 10/14] remove unused error --- object_store/src/gcp.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs index 00ee11cd5876..803e9ddfb203 100644 --- a/object_store/src/gcp.rs +++ b/object_store/src/gcp.rs @@ -73,9 +73,6 @@ enum Error { #[snafu(display("Error decoding object size: {}", source))] InvalidSize { source: std::num::ParseIntError }, - - #[snafu(display("Error creating http client: {}", source))] - CreatingHttpClient { source: reqwest::Error }, } impl From for super::Error { From 555c8f667ddff8ae1fda8da3336a8558eaf484c5 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 23 Jul 2022 16:40:17 -0500 Subject: [PATCH 11/14] Also run clippy --- .github/workflows/object_store.yml | 16 ++++++++++++++++ object_store/src/azure.rs | 1 - 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 359406292748..1b84f3ef083f 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -26,6 +26,22 @@ on: - .github/** jobs: + clippy: + name: Clippy + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v2 + - name: Setup Rust toolchain with clippy + run: | + rustup toolchain install stable + rustup default stable + rustup component add clippy + - name: Run clippy + run: | + cargo clippy -p object_store --all-features + # test the crate linux-test: name: Emulator Tests diff --git a/object_store/src/azure.rs b/object_store/src/azure.rs index ec65b18c7241..75dafef8694b 100644 --- a/object_store/src/azure.rs +++ b/object_store/src/azure.rs @@ -38,7 +38,6 @@ use futures::{ use snafu::{ResultExt, Snafu}; use std::collections::BTreeSet; use std::{convert::TryInto, sync::Arc}; -use tokio::io::AsyncWrite; use url::Url; /// A specialized `Error` for Azure object store-related errors From 8823f302b927ff9d091d06d940f9f2535f8515e2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 25 Jul 2022 06:22:45 -0400 Subject: [PATCH 12/14] Update object_store/src/gcp.rs Co-authored-by: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> --- object_store/src/gcp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs index 803e9ddfb203..2df5b9d20a06 100644 --- a/object_store/src/gcp.rs +++ b/object_store/src/gcp.rs @@ -584,7 +584,7 @@ mod test { } impl GoogleCloudConfig { - fn build(self) -> Result { + fn build_test(self) -> Result { // ignore HTTPS errors in tests so we can use fake-gcs server let client = Client::builder() .danger_accept_invalid_certs(true) From 3faf4c7b5c9b7794a796c60fc199259dd22bc04f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 25 Jul 2022 06:37:40 -0400 Subject: [PATCH 13/14] rename more --- object_store/src/gcp.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs index 2df5b9d20a06..e836caba7b44 100644 --- a/object_store/src/gcp.rs +++ b/object_store/src/gcp.rs @@ -642,7 +642,7 @@ mod test { #[tokio::test] async fn gcs_test() { let config = maybe_skip_integration!(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); put_get_delete_list(&integration).await.unwrap(); list_uses_directories_correctly(&integration).await.unwrap(); @@ -653,7 +653,7 @@ mod test { #[tokio::test] async fn gcs_test_get_nonexistent_location() { let config = maybe_skip_integration!(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -670,7 +670,7 @@ mod test { async fn gcs_test_get_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -688,7 +688,7 @@ mod test { #[tokio::test] async fn gcs_test_delete_nonexistent_location() { let config = maybe_skip_integration!(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -704,7 +704,7 @@ mod test { async fn gcs_test_delete_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); @@ -720,7 +720,7 @@ mod test { async fn gcs_test_put_nonexistent_bucket() { let mut config = maybe_skip_integration!(); config.bucket = NON_EXISTENT_NAME.into(); - let integration = config.build().unwrap(); + let integration = config.build_test().unwrap(); let location = Path::from_iter([NON_EXISTENT_NAME]); let data = Bytes::from("arbitrary data"); From 427107d88c235be19f76e3a5059ba355200fea16 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 25 Jul 2022 06:46:41 -0400 Subject: [PATCH 14/14] Fixup test --- object_store/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs index 4a56b03bfb72..2dc65069a998 100644 --- a/object_store/src/lib.rs +++ b/object_store/src/lib.rs @@ -695,8 +695,8 @@ mod tests { #[tokio::test] async fn test_list_lifetimes() { let store = memory::InMemory::new(); - let stream = list_store(&store, "path").await.unwrap(); - assert_eq!(stream.count().await, 0); + let mut stream = list_store(&store, "path").await.unwrap(); + assert!(stream.next().await.is_none()); } // Tests TODO: