HADOOP-19527. S3A: Skip AAL tests if encryption is set. (#7592) (#7604)

Contributed by Ahmar Suhail.

Author: ahmarsuhail

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java

@@ -23,12 +23,15 @@
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.enableAnalyticsAccelerator;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipForAnyEncryptionExceptSSES3;
 
 /**
- * S3A contract tests for vectored reads with the Analytics stream. The analytics stream does
- * not explicitly implement the vectoredRead() method, or currently do and vectored-read specific
- * optimisations (such as range coalescing). However, this test ensures that the base implementation
- * of readVectored {@link org.apache.hadoop.fs.PositionedReadable} still works.
+ * S3A contract tests for vectored reads with the Analytics stream.
+ * The analytics stream does not explicitly implement the vectoredRead() method,
+ * or currently do and vectored-read specific optimisations
+ * (such as range coalescing). However, this test ensures that the base
+ * implementation of readVectored {@link org.apache.hadoop.fs.PositionedReadable}
+ * still works.
  */
 public class ITestS3AContractAnalyticsStreamVectoredRead extends AbstractContractVectoredReadTest {
 
@@ -44,6 +47,16 @@ public ITestS3AContractAnalyticsStreamVectoredRead(String bufferType) {
   protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     enableAnalyticsAccelerator(conf);
+    // If encryption is set, some AAL tests will fail.
+    // This is because AAL caches the head request response, and uses
+    // the eTag when making a GET request. When using encryption, the eTag is
+    // no longer a hash of the object content, and is not always the same when
+    // the same object is created multiple times. This test creates the file
+    // vectored_file.txt before running each test, which will have a
+    // different eTag when using encryption, leading to preconditioned failures.
+    // This issue is tracked in:
+    // https://github.com/awslabs/analytics-accelerator-s3/issues/218
+    skipForAnyEncryptionExceptSSES3(conf);
     conf.set("fs.contract.vector-io-early-eof-check", "false");
     return conf;
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java

@@ -72,6 +72,7 @@ public class ITestS3AAnalyticsAcceleratorStreamReading extends AbstractS3ATestBa
   @Before
   public void setUp() throws Exception {
     super.setup();
+    skipIfClientSideEncryption();
     externalTestFile = getExternalData(getConfiguration());
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java

@@ -106,6 +106,7 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.impl.FlagSet.createFlagSet;
+import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_S3;
 import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Analytics;
 import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Prefetch;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
@@ -1728,6 +1729,30 @@ public static void skipIfEncryptionNotSet(Configuration configuration,
     }
   }
 
+  /**
+   * Skip a test if encryption algorithm is not empty, or if it is set to
+   * anything other than AES256.
+   *
+   * @param configuration configuration
+   */
+  public static void skipForAnyEncryptionExceptSSES3(Configuration configuration) {
+    String bucket = getTestBucketName(configuration);
+    try {
+      final EncryptionSecrets secrets = buildEncryptionSecrets(bucket, configuration);
+      S3AEncryptionMethods s3AEncryptionMethods = secrets.getEncryptionMethod();
+
+      if (s3AEncryptionMethods.getMethod().equals(SSE_S3.getMethod())
+              || s3AEncryptionMethods.getMethod().isEmpty()) {
+        return;
+      }
+
+      skip("Encryption method is set to " + s3AEncryptionMethods.getMethod());
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+
+  }
+
   /**
    * Get the input stream statistics of an input stream.
    * Raises an exception if the inner stream is not an S3A input stream