fix(clients): allow chunked requests on WithTransformation methods (generated)

algolia/api-clients-automation#5011

Co-authored-by: algolia-bot <[email protected]>
Co-authored-by: Clément Vannicatte <[email protected]>

Author: algolia-bot

packages/algoliasearch/__tests__/algoliasearch.common.test.ts

@@ -168,86 +168,6 @@ describe('api', () => {
         }),
       ).rejects.toThrow('`transformation.region` must be provided at client instantiation before calling this method.');
     });
-
-    test('exposes the transformation methods at the root of the client', async () => {
-      const ingestionClient = algoliasearch('APP_ID', 'API_KEY', {
-        requester: browserEchoRequester(),
-        transformation: { region: 'us' },
-      });
-
-      expect(ingestionClient.saveObjectsWithTransformation).not.toBeUndefined();
-
-      let res = (await ingestionClient.saveObjectsWithTransformation({
-        indexName: 'foo',
-        objects: [{ objectID: 'bar', baz: 42 }],
-        waitForTasks: true,
-      })) as unknown as EchoResponse;
-
-      expect(res.headers).toEqual(
-        expect.objectContaining({
-          'x-algolia-application-id': 'APP_ID',
-          'x-algolia-api-key': 'API_KEY',
-        }),
-      );
-      expect(res.url.startsWith('https://data.us.algolia.com/1/push/foo?watch=true')).toBeTruthy();
-      expect(res.data).toEqual({
-        action: 'addObject',
-        records: [
-          {
-            baz: 42,
-            objectID: 'bar',
-          },
-        ],
-      });
-      expect(ingestionClient.partialUpdateObjectsWithTransformation).not.toBeUndefined();
-
-      res = (await ingestionClient.partialUpdateObjectsWithTransformation({
-        indexName: 'foo',
-        objects: [{ objectID: 'bar', baz: 42 }],
-        waitForTasks: true,
-        createIfNotExists: true,
-      })) as unknown as EchoResponse;
-
-      expect(res.headers).toEqual(
-        expect.objectContaining({
-          'x-algolia-application-id': 'APP_ID',
-          'x-algolia-api-key': 'API_KEY',
-        }),
-      );
-      expect(res.url.startsWith('https://data.us.algolia.com/1/push/foo?watch=true')).toBeTruthy();
-      expect(res.data).toEqual({
-        action: 'partialUpdateObject',
-        records: [
-          {
-            baz: 42,
-            objectID: 'bar',
-          },
-        ],
-      });
-
-      res = (await ingestionClient.partialUpdateObjectsWithTransformation({
-        indexName: 'foo',
-        objects: [{ objectID: 'bar', baz: 42 }],
-        waitForTasks: true,
-      })) as unknown as EchoResponse;
-
-      expect(res.headers).toEqual(
-        expect.objectContaining({
-          'x-algolia-application-id': 'APP_ID',
-          'x-algolia-api-key': 'API_KEY',
-        }),
-      );
-      expect(res.url.startsWith('https://data.us.algolia.com/1/push/foo?watch=true')).toBeTruthy();
-      expect(res.data).toEqual({
-        action: 'partialUpdateObjectNoCreate',
-        records: [
-          {
-            baz: 42,
-            objectID: 'bar',
-          },
-        ],
-      });
-    });
   });
 });
 

packages/algoliasearch/builds/browser.ts

@@ -1,7 +1,6 @@
 // Code generated by OpenAPI Generator (https://openapi-generator.tech), manual changes will be lost - read more on https://github.com/algolia/api-clients-automation. DO NOT EDIT.
 
-import type { ApiError, ClientOptions, RequestOptions } from '@algolia/client-common';
-import { createIterablePromise } from '@algolia/client-common';
+import type { ClientOptions, RequestOptions } from '@algolia/client-common';
 
 import type { AbtestingClient } from '@algolia/client-abtesting';
 import { abtestingClient } from '@algolia/client-abtesting';
@@ -23,13 +22,12 @@ import type { RecommendClient } from '@algolia/recommend';
 import { recommendClient } from '@algolia/recommend';
 
 import type {
-  ChunkedBatchOptions,
   PartialUpdateObjectsOptions,
   ReplaceAllObjectsOptions,
   ReplaceAllObjectsWithTransformationResponse,
   SaveObjectsOptions,
 } from '@algolia/client-search';
-import type { PushTaskRecords, WatchResponse } from '@algolia/ingestion';
+import type { WatchResponse } from '@algolia/ingestion';
 
 import type {
   AbtestingRegionOptions,
@@ -65,12 +63,12 @@ export type Algoliasearch = SearchClient & {
    * @param saveObjects.objects - The array of `objects` to store in the given Algolia `indexName`.
    * @param saveObjects.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
    * @param saveObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
-   * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `batch` method and merged with the transporter requestOptions.
+   * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `push` method and merged with the transporter requestOptions.
    */
   saveObjectsWithTransformation: (
     options: SaveObjectsOptions,
     requestOptions?: RequestOptions | undefined,
-  ) => Promise<WatchResponse>;
+  ) => Promise<Array<WatchResponse>>;
 
   /**
    * Helper: Similar to the `partialUpdateObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
@@ -82,12 +80,12 @@ export type Algoliasearch = SearchClient & {
    * @param partialUpdateObjects.createIfNotExists - To be provided if non-existing objects are passed, otherwise, the call will fail..
    * @param partialUpdateObjects.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
    * @param partialUpdateObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
-   * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions.
+   * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `push` method and merged with the transporter requestOptions.
    */
   partialUpdateObjectsWithTransformation: (
     options: PartialUpdateObjectsOptions,
     requestOptions?: RequestOptions | undefined,
-  ) => Promise<WatchResponse>;
+  ) => Promise<Array<WatchResponse>>;
 
   /**
    * Helper: Similar to the `replaceAllObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
@@ -104,20 +102,6 @@ export type Algoliasearch = SearchClient & {
     options: ReplaceAllObjectsOptions,
     requestOptions?: RequestOptions | undefined,
   ) => Promise<ReplaceAllObjectsWithTransformationResponse>;
-
-  /**
-   * Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
-   *
-   * @summary Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
-   * @param chunkedPush - The `chunkedPush` object.
-   * @param chunkedPush.indexName - The `indexName` to replace `objects` in.
-   * @param chunkedPush.objects - The array of `objects` to store in the given Algolia `indexName`.
-   * @param chunkedPush.action - The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
-   * @param chunkedPush.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
-   * @param chunkedPush.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
-   * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getEvent` method and merged with the transporter requestOptions.
-   */
-  chunkedPush: (options: ChunkedBatchOptions, requestOptions?: RequestOptions) => Promise<Array<WatchResponse>>;
 };
 
 export type TransformationOptions = {
@@ -158,7 +142,10 @@ export function algoliasearch(
   return {
     ...client,
 
-    async saveObjectsWithTransformation({ indexName, objects, waitForTasks }, requestOptions): Promise<WatchResponse> {
+    async saveObjectsWithTransformation(
+      { indexName, objects, waitForTasks },
+      requestOptions,
+    ): Promise<Array<WatchResponse>> {
       if (!ingestionTransporter) {
         throw new Error('`transformation.region` must be provided at client instantiation before calling this method.');
       }
@@ -167,23 +154,16 @@ export function algoliasearch(
         throw new Error('`region` must be provided when leveraging the transformation pipeline');
       }
 
-      return ingestionTransporter?.push(
-        {
-          indexName,
-          watch: waitForTasks,
-          pushTaskPayload: {
-            action: 'addObject',
-            records: objects as PushTaskRecords[],
-          },
-        },
+      return ingestionTransporter.chunkedPush(
+        { indexName, objects, action: 'addObject', waitForTasks },
         requestOptions,
       );
     },
 
     async partialUpdateObjectsWithTransformation(
       { indexName, objects, createIfNotExists, waitForTasks },
       requestOptions,
-    ): Promise<WatchResponse> {
+    ): Promise<Array<WatchResponse>> {
       if (!ingestionTransporter) {
         throw new Error('`transformation.region` must be provided at client instantiation before calling this method.');
       }
@@ -192,86 +172,17 @@ export function algoliasearch(
         throw new Error('`region` must be provided when leveraging the transformation pipeline');
       }
 
-      return ingestionTransporter?.push(
+      return ingestionTransporter.chunkedPush(
         {
           indexName,
-          watch: waitForTasks,
-          pushTaskPayload: {
-            action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate',
-            records: objects as PushTaskRecords[],
-          },
+          objects,
+          action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate',
+          waitForTasks,
         },
         requestOptions,
       );
     },
 
-    async chunkedPush(
-      { indexName, objects, action = 'addObject', waitForTasks, batchSize = 1000 }: ChunkedBatchOptions,
-      requestOptions?: RequestOptions,
-    ): Promise<Array<WatchResponse>> {
-      if (!ingestionTransporter) {
-        throw new Error('`transformation.region` must be provided at client instantiation before calling this method.');
-      }
-
-      if (!options?.transformation?.region) {
-        throw new Error('`region` must be provided when leveraging the transformation pipeline');
-      }
-
-      let records: Array<PushTaskRecords> = [];
-      const responses: Array<WatchResponse> = [];
-
-      const objectEntries = objects.entries();
-      for (const [i, obj] of objectEntries) {
-        records.push(obj as PushTaskRecords);
-        if (records.length === batchSize || i === objects.length - 1) {
-          responses.push(
-            await ingestionTransporter.push(
-              { indexName, pushTaskPayload: { action, records }, watch: waitForTasks },
-              requestOptions,
-            ),
-          );
-          records = [];
-        }
-      }
-
-      let retryCount = 0;
-
-      if (waitForTasks) {
-        for (const resp of responses) {
-          if (!resp.eventID) {
-            throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
-          }
-
-          await createIterablePromise({
-            func: async () => {
-              if (resp.eventID === undefined || !resp.eventID) {
-                throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
-              }
-
-              return ingestionTransporter
-                .getEvent({ runID: resp.runID, eventID: resp.eventID })
-                .catch((error: ApiError) => {
-                  if (error.status === 404) {
-                    return undefined;
-                  }
-
-                  throw error;
-                });
-            },
-            validate: (response) => response !== undefined,
-            aggregator: () => (retryCount += 1),
-            error: {
-              validate: () => retryCount >= 50,
-              message: () => `The maximum number of retries exceeded. (${retryCount}/${50})`,
-            },
-            timeout: (): number => Math.min(retryCount * 500, 5000),
-          });
-        }
-      }
-
-      return responses;
-    },
-
     async replaceAllObjectsWithTransformation(
       { indexName, objects, batchSize, scopes }: ReplaceAllObjectsOptions,
       requestOptions?: RequestOptions | undefined,
@@ -304,8 +215,8 @@ export function algoliasearch(
           requestOptions,
         );
 
-        const watchResponses = await this.chunkedPush(
-          { indexName: tmpIndexName, objects, waitForTasks: true, batchSize },
+        const watchResponses = await ingestionTransporter.chunkedPush(
+          { indexName: tmpIndexName, objects, waitForTasks: true, batchSize, referenceIndexName: indexName },
           requestOptions,
         );