Improve CI build

Author: elpiel

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
         uses: sudo-bot/action-docker-compose@latest
         with:
           # https://docs.docker.com/compose/reference/overview/
-          cli-args: "-f docker-compose.harness.yml up -d"
+          cli-args: "-f docker-compose.harness.yml up -d --build"
       - uses: actions-rs/toolchain@v1
         with:
           # No need to add `toolchain`, it will use `rust-toolchain` file instead
@@ -26,7 +26,4 @@ jobs:
       - name: Run `cargo make ci-flow`
         # Running cargo make doesn't successfully start `ganache`
         run: |
-          cargo make ci-flow
-        # set environment variables for `primitives` postgres tests
-        env:
-          POSTGRES_DB: sentry_leader
+          cargo make ci-flow

adapter/Makefile.toml

@@ -14,7 +14,7 @@ dependencies = [
 
 [tasks.ganache-up]
 script = '''
-docker-compose -f ../docker-compose.harness.yml up -d ganache \
+docker-compose -f ../docker-compose.harness.yml up --renew-anon-volumes -d ganache \
 && sleep 10
 '''
 

docker-compose.harness.yml

@@ -2,18 +2,19 @@ version: '3.8'
 
 services:
   adex-postgres:
-    image: postgres
+    build: ./scripts/postgres
+    # image: adex-postges
     container_name: adex-postgres
     restart: always
-    volumes:
-      - ./scripts/postgres:/docker-entrypoint-initdb.d
+    # volumes:
+      # - ./scripts/postgres:/docker-entrypoint-initdb.d
     ports:
       - "5432:5432"
     environment:
       POSTGRES_HOST: 'localhost'
       POSTGRES_USER: 'postgres'
       POSTGRES_PASSWORD: 'postgres'
-      POSTGRES_MULTIPLE_DATABASES: sentry_leader,sentry_follower
+      POSTGRES_MULTIPLE_DATABASES: harness_leader,harness_follower,sentry_leader,primitives
     networks:
       - adex-external
 

primitives/Makefile.toml

@@ -13,7 +13,7 @@ dependencies = [
 ]
 
 [tasks.test]
-env = { "POSTGRES_DB" = "sentry_leader" }
+env = { "POSTGRES_DB" = "primitives" }
 
 [tasks.services-up]
 script = "docker-compose -f ../docker-compose.harness.yml up -d adex-postgres && sleep 6"

scripts/postgres/Dockerfile

@@ -0,0 +1,5 @@
+FROM postgres:latest
+COPY ./create-multiple-postgres-db.sh /docker-entrypoint-initdb.d
+
+
+CMD ["docker-entrypoint.sh", "postgres"]

scripts/postgres/create-multiple-postgres-db.sh

@@ -4,19 +4,20 @@ set -e
 set -u
 
 function create_user_and_database() {
-	local database=$1
-	echo "  Creating user and database '$database'"
-	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
-	    CREATE USER $database;
-	    CREATE DATABASE $database;
-	    GRANT ALL PRIVILEGES ON DATABASE $database TO $database;
+    local database=$1
+
+    echo "Creating user and database '$database'"
+    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+        CREATE USER $database;
+        CREATE DATABASE $database;
+        GRANT ALL PRIVILEGES ON DATABASE $database TO $database;
 EOSQL
 }
 
 if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
-	echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
-	for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
-		create_user_and_database $db
-	done
-	echo "Multiple databases created"
+    echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
+    for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
+        create_user_and_database $db
+    done
+    echo "Multiple databases created"
 fi

sentry/Makefile.toml

@@ -1,6 +1,3 @@
-[tasks.test]
-env = { "POSTGRES_DB" = "sentry_leader" }
-
 [tasks.dev-test-flow]
 description = "Development testing flow will first format the code, and than run cargo build and test"
 category = "Development"
@@ -15,6 +12,9 @@ dependencies = [
   "services-down",
 ]
 
+[tasks.test]
+env = { "POSTGRES_DB" = "sentry_leader" }
+
 [tasks.services-up]
 script = "docker-compose -f ../docker-compose.harness.yml up -d adex-redis adex-postgres && sleep 3"
 

sentry/src/db.rs

@@ -29,7 +29,7 @@ pub use tokio_postgres::Config as PostgresConfig;
 // Re-export the Postgres PoolError for easier usages
 pub use deadpool_postgres::PoolError;
 // Re-export the redis RedisError for easier usage
-pub use redis::RedisError;
+pub use redis::{RedisError, cmd as redis_cmd};
 
 pub type DbPool = deadpool_postgres::Pool;
 

sentry/src/lib.rs

@@ -562,7 +562,7 @@ pub mod test_util {
     use adapter::DummyAdapter;
     use primitives::{
         adapter::DummyAdapterOptions,
-        config::{configuration, DEVELOPMENT_CONFIG},
+        config::DEVELOPMENT_CONFIG,
         util::tests::{discard_logger, prep_db::IDS},
     };
 
@@ -575,7 +575,8 @@ pub mod test_util {
         Application,
     };
 
-    /// Uses development and therefore the goreli testnet addresses of the tokens
+    /// Uses development and therefore the goerli testnet addresses of the tokens
+    /// It still uses DummyAdapter.
     pub async fn setup_dummy_app() -> Application<DummyAdapter> {
         let config = DEVELOPMENT_CONFIG.clone();
         let adapter = DummyAdapter::init(

test_harness/Makefile.toml

@@ -10,6 +10,10 @@ dependencies = [
   "local-test-flow",
 ]
 
+[tasks.test]
+# run tests in release because of slow unlock time of Ethereum adapter (i.e. keystore decryption)
+args = ["test", "--release"]
+
 [tasks.local-test-flow]
 dependencies = ["services-up", "test-flow", "services-down"]
 
@@ -20,7 +24,7 @@ dependencies = ["services-up", "test-flow", "services-down"]
 # This forces the snapshot from previous unsuccessful test runs to get destroyed.
 script = '''
 docker-compose -f ../docker-compose.harness.yml up --renew-anon-volumes -d ganache adex-redis adex-postgres \
-&& sleep 9
+&& sleep 6
 '''
 
 [tasks.services-down]

test_harness/src/lib.rs

@@ -100,7 +100,7 @@ pub static VALIDATORS: Lazy<HashMap<&'static str, TestValidator>> = Lazy::new(||
                     redis_url: "redis://127.0.0.1:6379/1".parse().unwrap(),
                 },
                 logger_prefix: "sentry-leader".into(),
-                db_name: "sentry_leader".into(),
+                db_name: "harness_leader".into(),
             },
         ),
         (
@@ -115,7 +115,7 @@ pub static VALIDATORS: Lazy<HashMap<&'static str, TestValidator>> = Lazy::new(||
                     redis_url: "redis://127.0.0.1:6379/2".parse().unwrap(),
                 },
                 logger_prefix: "sentry-follower".into(),
-                db_name: "sentry_follower".into(),
+                db_name: "harness_follower".into(),
             },
         ),
     ]
@@ -728,7 +728,10 @@ pub mod run {
         };
 
         let postgres = postgres_connection(42, postgres_config).await;
-        let redis = redis_connection(app_config.redis_url).await?;
+        let mut redis = redis_connection(app_config.redis_url).await?;
+        sentry::db::redis_cmd("FLUSHDB")
+                .query_async::<_, String>(&mut redis)
+                .await.unwrap();
         let campaign_remaining = CampaignRemaining::new(redis.clone());
 
         setup_test_migrations(postgres.clone())