From 8c4e6c70021d128cdcf184095c602bada02c70fa Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Sat, 12 Apr 2025 23:02:54 +0900 Subject: [PATCH 01/20] wip --- package-lock.json | 1321 ++++++++++------- packages/cdk/lambda/nova-sonic-lambda.ts | 538 +++++++ packages/cdk/lib/construct/events.ts | 61 + packages/cdk/lib/construct/index.ts | 1 + .../cdk/lib/generative-ai-use-cases-stack.ts | 6 + packages/cdk/package.json | 4 + .../web/public/audio-processor.worklet.js | 124 ++ packages/web/public/audio-recorder.worklet.js | 127 ++ packages/web/src/components/AuthWithSAML.tsx | 8 + .../web/src/components/AuthWithUserpool.tsx | 8 + packages/web/src/hooks/useNovaSonic.ts | 552 +++++++ packages/web/src/main.tsx | 7 +- packages/web/src/pages/SpeechToSpeechPage.tsx | 37 + 13 files changed, 2221 insertions(+), 573 deletions(-) create mode 100644 packages/cdk/lambda/nova-sonic-lambda.ts create mode 100644 packages/cdk/lib/construct/events.ts create mode 100644 packages/web/public/audio-processor.worklet.js create mode 100644 packages/web/public/audio-recorder.worklet.js create mode 100644 packages/web/src/hooks/useNovaSonic.ts create mode 100644 packages/web/src/pages/SpeechToSpeechPage.tsx diff --git a/package-lock.json b/package-lock.json index d89b5e8f2..c12e80353 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,9 +63,9 @@ } }, "node_modules/@aws-amplify/analytics": { - "version": "7.0.77", - "resolved": "https://registry.npmjs.org/@aws-amplify/analytics/-/analytics-7.0.77.tgz", - "integrity": "sha512-iahOh4HdPa0D8pJo1+BWQ8g1+iFnKe9UAHJ2ks9HAQ14utbV39OG57onhkBA5bTSXYS9zSEjVgMreC0Wg67NIQ==", + "version": "7.0.78", + "resolved": "https://registry.npmjs.org/@aws-amplify/analytics/-/analytics-7.0.78.tgz", + "integrity": "sha512-ia6ZTVIq0lhlxuw1pgnGOhD2svaFJ8MypsrscTUgyziHfzmDFqhswhxhYjqie+9ilcLjrOypMBke0dBLCY8d9A==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-firehose": "3.621.0", @@ -117,13 +117,13 @@ } }, "node_modules/@aws-amplify/api": { - "version": "6.3.8", - "resolved": "https://registry.npmjs.org/@aws-amplify/api/-/api-6.3.8.tgz", - "integrity": "sha512-AtYbdrniz6SHWTgz/v4H3FODHfq+0q5aNY5P9O9DVWQkqkIwwkMGVGt5e//nuiF10e80qTPesg4g6ygPNIUtwA==", + "version": "6.3.9", + "resolved": "https://registry.npmjs.org/@aws-amplify/api/-/api-6.3.9.tgz", + "integrity": "sha512-QRt9Eo8W7ztwshnAzxf8mYYAsK5gysmxN2cM2eSNiPd6wZZJZFVQy7KzXuonWBB7SVGGThu0GG1ZtUOGXO/bsg==", "license": "Apache-2.0", "dependencies": { - "@aws-amplify/api-graphql": "4.7.12", - "@aws-amplify/api-rest": "4.1.1", + "@aws-amplify/api-graphql": "4.7.13", + "@aws-amplify/api-rest": "4.1.2", "@aws-amplify/data-schema": "^1.7.0", "rxjs": "^7.8.1", "tslib": "^2.5.0" @@ -133,13 +133,13 @@ } }, "node_modules/@aws-amplify/api-graphql": { - "version": "4.7.12", - "resolved": "https://registry.npmjs.org/@aws-amplify/api-graphql/-/api-graphql-4.7.12.tgz", - "integrity": "sha512-iqjLWRb60SU6EB2pM53lsCGaRtOXncXpe5PEYY4l6MjHaBgtJmdKdtHp1h4/fMUBtWdmJY2KVpQAKfhu2Hg3fw==", + "version": "4.7.13", + "resolved": "https://registry.npmjs.org/@aws-amplify/api-graphql/-/api-graphql-4.7.13.tgz", + "integrity": "sha512-n3QyBZ5kMQyUiplQcrGXdXNmEu2o7luEdIWCMMr5Nevo2tYJtj9319HebizfuTKZ98qukZ3FnaT0SA2Jt4Nkbw==", "license": "Apache-2.0", "dependencies": { - "@aws-amplify/api-rest": "4.1.1", - "@aws-amplify/core": "6.11.1", + "@aws-amplify/api-rest": "4.1.2", + "@aws-amplify/core": "6.11.2", "@aws-amplify/data-schema": "^1.7.0", "@aws-sdk/types": "3.387.0", "graphql": "15.8.0", @@ -174,9 +174,9 @@ } }, "node_modules/@aws-amplify/api-rest": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@aws-amplify/api-rest/-/api-rest-4.1.1.tgz", - "integrity": "sha512-OHtkZRAYZJBwFEoI1Tgcue3hbegNX0VvRolYb/qpsJIVfYTMfOZVD7JwQfRDFinuzHn1v6FBKI3Eo1q2apiLog==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@aws-amplify/api-rest/-/api-rest-4.1.2.tgz", + "integrity": "sha512-R4WBFrsF874vvKuBsk+f7jyvOI2+x3sbA1lHHhFGUhBrtESZ0G0jz4UuMvp52h4KumDfB9Sxt976QUyYDPA07A==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.5.0" @@ -186,9 +186,9 @@ } }, "node_modules/@aws-amplify/auth": { - "version": "6.12.1", - "resolved": "https://registry.npmjs.org/@aws-amplify/auth/-/auth-6.12.1.tgz", - "integrity": "sha512-VjFJPVOWBAzcoW/DoNADPNBT5Map6oeuR4w5H8uQHBaNTp/QOzgTB2pUt4YZRnf2z1G7AcHNB/xUyzTHV+8gtg==", + "version": "6.12.2", + "resolved": "https://registry.npmjs.org/@aws-amplify/auth/-/auth-6.12.2.tgz", + "integrity": "sha512-bVuPcM/mnn8437KdkRmWys2fZ3s1YogQOmJU20HVpO0UEre5GPQnSd6SRAVGX3ZQCTjYcOvlbLGewypvVi9eYQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -212,9 +212,9 @@ } }, "node_modules/@aws-amplify/core": { - "version": "6.11.1", - "resolved": "https://registry.npmjs.org/@aws-amplify/core/-/core-6.11.1.tgz", - "integrity": "sha512-4JCANRSrCJX/arDaNbiqvJgvQE2KW3tBTVrESsh63PyAm8S4sg2bplpGJYF5e7fjqYs/qp0iwO9VFH/57HbroA==", + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/@aws-amplify/core/-/core-6.11.2.tgz", + "integrity": "sha512-aRjacDe8/lEF2+xJYdUzwdcrtsXhv+OkrOBzGz3Yn4V7D8/H5GzygiFAqVMLCWLP/yQXv1ujO6NX9EZkWFMRAg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -303,13 +303,13 @@ } }, "node_modules/@aws-amplify/datastore": { - "version": "5.0.79", - "resolved": "https://registry.npmjs.org/@aws-amplify/datastore/-/datastore-5.0.79.tgz", - "integrity": "sha512-88LizSmytd6pZWqCf0mET0Qj5xMb6pRPjfDrJ0hjY4zE0V25UAxb0HPmLmSFYvW+G6SFzYYkEfehWyIDfIBHUw==", + "version": "5.0.80", + "resolved": "https://registry.npmjs.org/@aws-amplify/datastore/-/datastore-5.0.80.tgz", + "integrity": "sha512-/5yETkqAbDpcBK/iuBI7rNoPA6DxjYtWEbvEuCgOy9/fesCe0HVWm9IiTY2oCe2SdsflWztrVrJnaH+UAzvO1g==", "license": "Apache-2.0", "dependencies": { - "@aws-amplify/api": "6.3.8", - "@aws-amplify/api-graphql": "4.7.12", + "@aws-amplify/api": "6.3.9", + "@aws-amplify/api-graphql": "4.7.13", "buffer": "4.9.2", "idb": "5.0.6", "immer": "9.0.6", @@ -348,9 +348,9 @@ "license": "MIT" }, "node_modules/@aws-amplify/notifications": { - "version": "2.0.77", - "resolved": "https://registry.npmjs.org/@aws-amplify/notifications/-/notifications-2.0.77.tgz", - "integrity": "sha512-mI48rwrJyQSihIGDWfcA3YP7LVHZPtUjWahatyySsJhcS+aHjrw7L4mrfyQ2SHw9NIeHx3o4KvkFgDi/yLplqQ==", + "version": "2.0.78", + "resolved": "https://registry.npmjs.org/@aws-amplify/notifications/-/notifications-2.0.78.tgz", + "integrity": "sha512-i06CDxl+ZOZliwZnc+NqWagvz5rbX7ApeHGrvqIDptBXsfbpqo/QWPciz6NS/BLooBX7mvB3fPuBLS48jmEB5g==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "3.398.0", @@ -362,9 +362,9 @@ } }, "node_modules/@aws-amplify/storage": { - "version": "6.8.1", - "resolved": "https://registry.npmjs.org/@aws-amplify/storage/-/storage-6.8.1.tgz", - "integrity": "sha512-ZMrbsFRokxANSHwdXcBAFoOwsHhWvuweQUdMbuJXxelRM0Hk3v44j0BVFGggWZ3VaqHPJJxfPNU1LfufqIAKdg==", + "version": "6.8.2", + "resolved": "https://registry.npmjs.org/@aws-amplify/storage/-/storage-6.8.2.tgz", + "integrity": "sha512-Oh4c70zBtCez1kzSBzjabAWB1bszjktFBqKA5gS92JKmXbggsz9/pgxtxH8PzGZ66EE9c9UkH2qV6L5F3XkQSA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "3.398.0", @@ -419,9 +419,9 @@ "license": "MIT" }, "node_modules/@aws-amplify/ui": { - "version": "6.10.0", - "resolved": "https://registry.npmjs.org/@aws-amplify/ui/-/ui-6.10.0.tgz", - "integrity": "sha512-sbMteKSESv9PEC9DmCyyb85kjra361vw22WQd5UZpCbRVT3jKVZpy34hzTQRZjGtT/Q5Ej4Vuq3B8YU46tf+RQ==", + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/@aws-amplify/ui/-/ui-6.10.1.tgz", + "integrity": "sha512-xvPm09U67TcGAKfYGs8i4CUIU22TxNsfZuPTltYrcm34zNJSUBNcovd5x1XALZxPkmqpWTfxAwr6yWx6d7Yycw==", "license": "Apache-2.0", "dependencies": { "csstype": "^3.1.1", @@ -440,13 +440,13 @@ } }, "node_modules/@aws-amplify/ui-react": { - "version": "6.10.0", - "resolved": "https://registry.npmjs.org/@aws-amplify/ui-react/-/ui-react-6.10.0.tgz", - "integrity": "sha512-RCw1Xc9AfUDaXQriKkMqdrD97w6GX2CEwqa16zbDkwgTav5OLuJuVVm1xBEZ9Om5pTMMSmL0KR4II9hRhwsi0Q==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/@aws-amplify/ui-react/-/ui-react-6.11.0.tgz", + "integrity": "sha512-e8E6QNL/Y1UaSAi7eClQPqxA0879mNVbByGuRvFiLZFvKfyEcRCH2AG0+/xkungIGE19qI1cDBVTEoHu7EZ1hA==", "license": "Apache-2.0", "dependencies": { - "@aws-amplify/ui": "6.10.0", - "@aws-amplify/ui-react-core": "3.4.0", + "@aws-amplify/ui": "6.10.1", + "@aws-amplify/ui-react-core": "3.4.1", "@radix-ui/react-direction": "^1.1.0", "@radix-ui/react-dropdown-menu": "^2.1.4", "@radix-ui/react-slider": "^1.2.2", @@ -468,12 +468,12 @@ } }, "node_modules/@aws-amplify/ui-react-core": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/@aws-amplify/ui-react-core/-/ui-react-core-3.4.0.tgz", - "integrity": "sha512-tVOBGaZh+g+ToUoNZ5Q58priwEveqxjp3yIWTSOawSuoEf68ML4vWfyTiYDZi80mcEswzpD3rDtDfkjnlXTIKA==", + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@aws-amplify/ui-react-core/-/ui-react-core-3.4.1.tgz", + "integrity": "sha512-7HrssOXGjDhLzwBdI4OD7iKshpIJH5zMpaeUy0Ai95OiQsUe8NF19eI/zasRO4jpOD/i5hluAboVECAd6SVKHw==", "license": "Apache-2.0", "dependencies": { - "@aws-amplify/ui": "6.10.0", + "@aws-amplify/ui": "6.10.1", "@xstate/react": "^3.2.2", "lodash": "4.17.21", "react-hook-form": "^7.53.2", @@ -485,9 +485,9 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.230", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.230.tgz", - "integrity": "sha512-kUnhKIYu42hqBa6a8x2/7o29ObpJgjYGQy28lZDq9awXyvpR62I2bRxrNKNR3uFUQz3ySuT9JXhGHhuZPdbnFw==", + "version": "2.2.231", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.231.tgz", + "integrity": "sha512-vPqD/K2pK/ALhU5r5Nafdc2nLB+LJKxNyxUmQnLsazU6AWDJfkqjHQx8m3J4Cjl2C3chQkIRMdzSDuXIlo43GA==", "license": "Apache-2.0" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { @@ -648,24 +648,24 @@ } }, "node_modules/@aws-sdk/client-bedrock-agent": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-agent/-/client-bedrock-agent-3.777.0.tgz", - "integrity": "sha512-hCvO4OvmOelJbRpU4S4iGUK0sX4ujusxfWCVRsu/H6LGWq6P/gfRoIjBNJ+Ra1icrlqd0ytcAFEqsXGnF2LW4g==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-agent/-/client-bedrock-agent-3.787.0.tgz", + "integrity": "sha512-GqeGtmxJVfoh9zteQ+vAvWrklQEf7FB+w8ILMVB+Mq3eLccDOOuTHY+/fj25AsOGcP8DjuBZ1k3rtpSMy3JTrw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -700,24 +700,24 @@ } }, "node_modules/@aws-sdk/client-bedrock-agent-runtime": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-agent-runtime/-/client-bedrock-agent-runtime-3.777.0.tgz", - "integrity": "sha512-JTzUSEUougEHgvqQPyXgiU8QbDSIY7W/mM31AjGauw+6BV4MozhNuuVi6RLlKJuBVwmBpSNsj/bh9W4ADoeX0Q==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-agent-runtime/-/client-bedrock-agent-runtime-3.787.0.tgz", + "integrity": "sha512-iTms8QyAdvILO8IsrP91ETHDe29LERQOYuBvu7Z5JO3NuCpDSWBDTcGNXfTNCPKgBeTwXxzPWBSJ05679zieow==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/eventstream-serde-browser": "^4.0.2", @@ -805,24 +805,26 @@ } }, "node_modules/@aws-sdk/client-bedrock-runtime": { - "version": "3.779.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.779.0.tgz", - "integrity": "sha512-MyzZks8XxWwdsA4VlyPW4IekUjpgDI91VwMvEtOITHD8w+9nTGJtD32HcCKNQQCHWYfSoOj7yIoHRisVatR8yw==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.787.0.tgz", + "integrity": "sha512-aGxGNsv366rewmz0w10C6Epo9iClxdL9kY+uOEo4OO7gRchRwSHOj1AYK7Tqa0zB5vGLYa1KGCrvzvthCWt4AQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", + "@aws-sdk/eventstream-handler-node": "3.775.0", + "@aws-sdk/middleware-eventstream": "3.775.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/eventstream-serde-browser": "^4.0.2", @@ -887,24 +889,24 @@ } }, "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.777.0.tgz", - "integrity": "sha512-VGtFI3SH+jKfPln+9CM16F9zKieIqSxUSZNzQ6WZahPDVC79VmlG6QkXCqgm9Y4qZf4ebcdMhO23+FkR4s9vhA==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.787.0.tgz", + "integrity": "sha512-7v6nywZ5wcQxX7qdZ5M1ld15QdkzLU6fAKiEqbvJKu4dM8cFW6As+DbS990Mg46pp1xM/yvme+51xZDTfTfJZA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -963,25 +965,25 @@ } }, "node_modules/@aws-sdk/client-dynamodb": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.777.0.tgz", - "integrity": "sha512-LgFgdyGh8ZuBnbY1QNc+7ahNy5HFKK9OA6knEKr7UpWYZ0ct41EdPkGrsH9Ea/TMSu/C9p3W4rawwdUkfLepUQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.787.0.tgz", + "integrity": "sha512-mqR+MrO/Jef9fHU0QH4jDW5zJ3RtHUnyt6OVwO0ezQD5hWajIwT8fl2Bp0zqakhkhUAiW7FpGAiF+G+kEqb/rQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-endpoint-discovery": "3.775.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -2024,24 +2026,24 @@ } }, "node_modules/@aws-sdk/client-kendra": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kendra/-/client-kendra-3.777.0.tgz", - "integrity": "sha512-kpy2jeKev+pwrgWeInxTiuDLrmD+bUU/nV144VYI0huxnFdG0HPiJg0Xs8r6fNpPFbHOkVuJpUA/Krj9YTgQ4Q==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kendra/-/client-kendra-3.787.0.tgz", + "integrity": "sha512-BdCYVFIKBZtFS8xENqUzDjX++d+3h5OFtuGpWZ4XWajDpVh57bRx13dxMMqlCXDI8CBQ3Xld8fIMAeykYY2Wqw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -3168,24 +3170,24 @@ } }, "node_modules/@aws-sdk/client-lambda": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.777.0.tgz", - "integrity": "sha512-UXwqwS5U+kYFnqqrDSWwXddEiWPaX6sHHkiKZYslc+dM/bl1DyPmX17eTkde/8V+bsZvw9az2NuYlZy1N12DVQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.787.0.tgz", + "integrity": "sha512-aPSg7YL7IpEaijsunAYtws/3dZl+VjyQ1wbv6RxdIfzww/35x31GSc6vD6paq8KC6lcns8wlli/0qCOl8Z9wZg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/eventstream-serde-browser": "^4.0.2", @@ -4230,24 +4232,24 @@ } }, "node_modules/@aws-sdk/client-polly": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.777.0.tgz", - "integrity": "sha512-CKfU9ozw/xoBTMlPjn5MxoU01THbS7IOlNa94vmKdyx4vFckIDrpUcBD0eOaIZfzTkkQIApSbMmyKDj0auZrkg==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.787.0.tgz", + "integrity": "sha512-BD/4J/sABW4G9rECH3+AjGnm3cysia/jTFpp6XxksPoOqadYz8uaCxF8avO4zUjcrYxoVajzALKXQFk9PuirAw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -4307,32 +4309,32 @@ } }, "node_modules/@aws-sdk/client-s3": { - "version": "3.779.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.779.0.tgz", - "integrity": "sha512-Lagz+ersQaLNYkpOU9V12PYspT//lGvhPXlKU3OXDj3whDchdqUdtRKY8rmV+jli4KXe+udx/hj2yqrFRfKGvQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.787.0.tgz", + "integrity": "sha512-eGLCWkN0NlntJ9yPU6OKUggVS4cFvuZJog+cFg1KD5hniLqz7Y0YRtB4uBxW212fK3XCfddgyscEOEeHaTQQTw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-bucket-endpoint": "3.775.0", "@aws-sdk/middleware-expect-continue": "3.775.0", - "@aws-sdk/middleware-flexible-checksums": "3.775.0", + "@aws-sdk/middleware-flexible-checksums": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-location-constraint": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", "@aws-sdk/middleware-sdk-s3": "3.775.0", "@aws-sdk/middleware-ssec": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/signature-v4-multi-region": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@aws-sdk/xml-builder": "3.775.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", @@ -4400,24 +4402,24 @@ } }, "node_modules/@aws-sdk/client-sagemaker-runtime": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker-runtime/-/client-sagemaker-runtime-3.777.0.tgz", - "integrity": "sha512-b5IE89QOt7pvaImtDiOmUlKVGK7pcb3rW7nIKEqY65pJboIT/RNkehfA/XrlfgDjgGhVebojpUm/paqG+2xlDg==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker-runtime/-/client-sagemaker-runtime-3.787.0.tgz", + "integrity": "sha512-hf8le7HHruDsRWFzITyz3Isv32LdkMucnmQrPf4lhisIXJJcZJpp4m7Cf6bDopZ5q9mnsiARyEipsVEpzkUHfQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/eventstream-serde-browser": "^4.0.2", @@ -4480,9 +4482,9 @@ } }, "node_modules/@aws-sdk/client-sso": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.777.0.tgz", - "integrity": "sha512-0+z6CiAYIQa7s6FJ+dpBYPi9zr9yY5jBg/4/FGcwYbmqWPXwL9Thdtr0FearYRZgKl7bhL3m3dILCCfWqr3teQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.787.0.tgz", + "integrity": "sha512-L8R+Mh258G0DC73ktpSVrG4TT9i2vmDLecARTDR/4q5sRivdDQSL5bUp3LKcK80Bx+FRw3UETIlX6mYMLL9PJQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -4491,12 +4493,12 @@ "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -6517,24 +6519,24 @@ } }, "node_modules/@aws-sdk/client-transcribe": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.777.0.tgz", - "integrity": "sha512-8jhuZf6AXd3cq/lrE3GCq9xOkMGcDK4rhE76pvZQirlVKrShOoyLBLeKA4/YVStufZHhas/4/WieOV7O7tH0BA==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.787.0.tgz", + "integrity": "sha512-+0TNPuHydcJFyG3tyN4XmjkSQ8X/VTlISKragOqAEI05z27tM1Bx43jH6BuihC8brSgNmiEKvzNjrl7F+PPNcA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -6567,28 +6569,28 @@ } }, "node_modules/@aws-sdk/client-transcribe-streaming": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe-streaming/-/client-transcribe-streaming-3.777.0.tgz", - "integrity": "sha512-uH1BwGn4PeXTHcT0DArfnsafRmq8380qeMuc3OTEIx/G0LhK3f2DQ30bQ9JnXQq8TZv8C+3taoMPgNCw9atiTQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe-streaming/-/client-transcribe-streaming-3.787.0.tgz", + "integrity": "sha512-6ODQsLqAsujgH3KpKs8/fwN2eWspv3o0e8zFrzH6hW5ztdOq7k6H1CWBpFl88kdN3/EUHXNak+DkJanAq3rNTA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/credential-provider-node": "3.777.0", + "@aws-sdk/credential-provider-node": "3.787.0", "@aws-sdk/eventstream-handler-node": "3.775.0", "@aws-sdk/middleware-eventstream": "3.775.0", "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", "@aws-sdk/middleware-sdk-transcribe-streaming": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/middleware-websocket": "3.775.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/eventstream-serde-browser": "^4.0.2", @@ -6711,12 +6713,12 @@ } }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.777.0.tgz", - "integrity": "sha512-lNvz3v94TvEcBvQqVUyg+c/aL3Max+8wUMXvehWoQPv9y9cJAHciZqvA/G+yFo/JB+1Y4IBpMu09W2lfpT6Euw==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.787.0.tgz", + "integrity": "sha512-nF5XjgvZHFuyttOeTjMgfEsg6slZPQ6uI34yzq12Kq4icFgcD4bQsijnQClMN7A0u5qR8Ad8kume4b7+I2++Ig==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.777.0", + "@aws-sdk/client-cognito-identity": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/property-provider": "^4.0.2", "@smithy/types": "^4.2.0", @@ -6803,18 +6805,18 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.777.0.tgz", - "integrity": "sha512-1X9mCuM9JSQPmQ+D2TODt4THy6aJWCNiURkmKmTIPRdno7EIKgAqrr/LLN++K5mBf54DZVKpqcJutXU2jwo01A==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.787.0.tgz", + "integrity": "sha512-hc2taRoDlXn2uuNuHWDJljVWYrp3r9JF1a/8XmOAZhVUNY+ImeeStylHXhXXKEA4JOjW+5PdJj0f1UDkVCHJiQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.775.0", "@aws-sdk/credential-provider-env": "3.775.0", "@aws-sdk/credential-provider-http": "3.775.0", "@aws-sdk/credential-provider-process": "3.775.0", - "@aws-sdk/credential-provider-sso": "3.777.0", - "@aws-sdk/credential-provider-web-identity": "3.777.0", - "@aws-sdk/nested-clients": "3.777.0", + "@aws-sdk/credential-provider-sso": "3.787.0", + "@aws-sdk/credential-provider-web-identity": "3.787.0", + "@aws-sdk/nested-clients": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/credential-provider-imds": "^4.0.2", "@smithy/property-provider": "^4.0.2", @@ -6840,17 +6842,17 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.777.0.tgz", - "integrity": "sha512-ZD66ywx1Q0KyUSuBXZIQzBe3Q7MzX8lNwsrCU43H3Fww+Y+HB3Ncws9grhSdNhKQNeGmZ+MgKybuZYaaeLwJEQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.787.0.tgz", + "integrity": "sha512-JioVi44B1vDMaK2CdzqimwvJD3uzvzbQhaEWXsGMBcMcNHajXAXf08EF50JG3ZhLrhhUsT1ObXpbTaPINOhh+g==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-provider-env": "3.775.0", "@aws-sdk/credential-provider-http": "3.775.0", - "@aws-sdk/credential-provider-ini": "3.777.0", + "@aws-sdk/credential-provider-ini": "3.787.0", "@aws-sdk/credential-provider-process": "3.775.0", - "@aws-sdk/credential-provider-sso": "3.777.0", - "@aws-sdk/credential-provider-web-identity": "3.777.0", + "@aws-sdk/credential-provider-sso": "3.787.0", + "@aws-sdk/credential-provider-web-identity": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/credential-provider-imds": "^4.0.2", "@smithy/property-provider": "^4.0.2", @@ -6906,14 +6908,14 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.777.0.tgz", - "integrity": "sha512-9mPz7vk9uE4PBVprfINv4tlTkyq1OonNevx2DiXC1LY4mCUCNN3RdBwAY0BTLzj0uyc3k5KxFFNbn3/8ZDQP7w==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.787.0.tgz", + "integrity": "sha512-fHc08bsvwm4+dEMEQKnQ7c1irEQmmxbgS+Fq41y09pPvPh31nAhoMcjBSTWAaPHvvsRbTYvmP4Mf12ZGr8/nfg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.777.0", + "@aws-sdk/client-sso": "3.787.0", "@aws-sdk/core": "3.775.0", - "@aws-sdk/token-providers": "3.777.0", + "@aws-sdk/token-providers": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/property-provider": "^4.0.2", "@smithy/shared-ini-file-loader": "^4.0.2", @@ -6938,13 +6940,13 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.777.0.tgz", - "integrity": "sha512-uGCqr47fnthkqwq5luNl2dksgcpHHjSXz2jUra7TXtFOpqvnhOW8qXjoa1ivlkq8qhqlaZwCzPdbcN0lXpmLzQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.787.0.tgz", + "integrity": "sha512-SobmCwNbk6TfEsF283mZPQEI5vV2j6eY5tOCj8Er4Lzraxu9fBPADV+Bib2A8F6jlB1lMPJzOuDCbEasSt/RIw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.775.0", - "@aws-sdk/nested-clients": "3.777.0", + "@aws-sdk/nested-clients": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/property-provider": "^4.0.2", "@smithy/types": "^4.2.0", @@ -6967,6 +6969,49 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/credential-providers": { + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.787.0.tgz", + "integrity": "sha512-kR3RtI7drOc9pho13vWbUC2Bvrx9A0G4iizBDGmTs08NOdg4w3c1I4kdLG9tyPiIMeVnH+wYrsli5CM7xIfqiA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.787.0", + "@aws-sdk/core": "3.775.0", + "@aws-sdk/credential-provider-cognito-identity": "3.787.0", + "@aws-sdk/credential-provider-env": "3.775.0", + "@aws-sdk/credential-provider-http": "3.775.0", + "@aws-sdk/credential-provider-ini": "3.787.0", + "@aws-sdk/credential-provider-node": "3.787.0", + "@aws-sdk/credential-provider-process": "3.775.0", + "@aws-sdk/credential-provider-sso": "3.787.0", + "@aws-sdk/credential-provider-web-identity": "3.787.0", + "@aws-sdk/nested-clients": "3.787.0", + "@aws-sdk/types": "3.775.0", + "@smithy/config-resolver": "^4.1.0", + "@smithy/core": "^3.2.0", + "@smithy/credential-provider-imds": "^4.0.2", + "@smithy/node-config-provider": "^4.0.2", + "@smithy/property-provider": "^4.0.2", + "@smithy/types": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-providers/node_modules/@aws-sdk/types": { + "version": "3.775.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.775.0.tgz", + "integrity": "sha512-ZoGKwa4C9fC9Av6bdfqcW6Ix5ot05F/S4VxWR2nHuMv7hzfmAjTOcUiWT7UR4hM/U0whf84VhDtXN/DWAk52KA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/endpoint-cache": { "version": "3.723.0", "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.723.0.tgz", @@ -7009,13 +7054,13 @@ } }, "node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.778.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.778.0.tgz", - "integrity": "sha512-zyFo58UldyLOhVdnhvx4u4YkprIcrpMAYX3cI14JIH5KcwebreGrUA4lzko2sn7r/uqEArqFJWgGO3AJLBUW8w==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.787.0.tgz", + "integrity": "sha512-961/DbHJMh9EC+4nI1l6/81WkRoUtHcFa5T7PbJ41hWPge5IMy+JwDZwpF/e77kvws2loqATEZ0FRa0APgaN4A==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.775.0", - "@aws-sdk/util-dynamodb": "3.777.0", + "@aws-sdk/util-dynamodb": "3.787.0", "@smithy/core": "^3.2.0", "@smithy/smithy-client": "^4.2.0", "@smithy/types": "^4.2.0", @@ -7025,7 +7070,7 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.777.0" + "@aws-sdk/client-dynamodb": "^3.787.0" } }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { @@ -7146,9 +7191,9 @@ } }, "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.775.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.775.0.tgz", - "integrity": "sha512-OmHLfRIb7IIXsf9/X/pMOlcSV3gzW/MmtPSZTkrz5jCTKzWXd7eRoyOJqewjsaC6KMAxIpNU77FoAd16jOZ21A==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.787.0.tgz", + "integrity": "sha512-X71qEwWoixFmwowWzlPoZUR3u1CWJ7iAzU0EzIxqmPhQpQJLFmdL1+SRjqATynDPZQzLs1a5HBtPT++EnZ+Quw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", @@ -7416,14 +7461,14 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.775.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.775.0.tgz", - "integrity": "sha512-7Lffpr1ptOEDE1ZYH1T78pheEY1YmeXWBfFt/amZ6AGsKSLG+JPXvof3ltporTGR2bhH/eJPo7UHCglIuXfzYg==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.787.0.tgz", + "integrity": "sha512-Lnfj8SmPLYtrDFthNIaNj66zZsBCam+E4XiUDr55DIHTGstH6qZ/q6vg0GfbukxwSmUcGMwSR4Qbn8rb8yd77g==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@smithy/core": "^3.2.0", "@smithy/protocol-http": "^5.1.0", "@smithy/types": "^4.2.0", @@ -7493,9 +7538,9 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.777.0.tgz", - "integrity": "sha512-bmmVRsCjuYlStYPt06hr+f8iEyWg7+AklKCA8ZLDEJujXhXIowgUIqXmqpTkXwkVvDQ9tzU7hxaONjyaQCGybA==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.787.0.tgz", + "integrity": "sha512-xk03q1xpKNHgbuo+trEf1dFrI239kuMmjKKsqLEsHlAZbuFq4yRGMlHBrVMnKYOPBhVFDS/VineM991XI52fKg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -7504,12 +7549,12 @@ "@aws-sdk/middleware-host-header": "3.775.0", "@aws-sdk/middleware-logger": "3.775.0", "@aws-sdk/middleware-recursion-detection": "3.775.0", - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/region-config-resolver": "3.775.0", "@aws-sdk/types": "3.775.0", - "@aws-sdk/util-endpoints": "3.775.0", + "@aws-sdk/util-endpoints": "3.787.0", "@aws-sdk/util-user-agent-browser": "3.775.0", - "@aws-sdk/util-user-agent-node": "3.775.0", + "@aws-sdk/util-user-agent-node": "3.787.0", "@smithy/config-resolver": "^4.1.0", "@smithy/core": "^3.2.0", "@smithy/fetch-http-handler": "^5.0.2", @@ -7598,9 +7643,9 @@ } }, "node_modules/@aws-sdk/s3-request-presigner": { - "version": "3.779.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.779.0.tgz", - "integrity": "sha512-L3mGSh6/9gf3FBVrQziCkuLbaRJMeNbLr6tg9ZSymJcDRzRqAiCWnHrenAavTnAAnm+Lu62Fg/A4g3T+YT+gEg==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.787.0.tgz", + "integrity": "sha512-WBm0AS3RRURNN0yjYXHaiI692boVwWXGt3RLdI7tYBX58E1Zb5nzC8rlk81O9Xe7ZTgTC1KCr8y4+jcBD+zwJg==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/signature-v4-multi-region": "3.775.0", @@ -7660,12 +7705,12 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.777.0.tgz", - "integrity": "sha512-Yc2cDONsHOa4dTSGOev6Ng2QgTKQUEjaUnsyKd13pc/nLLz/WLqHiQ/o7PcnKERJxXGs1g1C6l3sNXiX+kbnFQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.787.0.tgz", + "integrity": "sha512-d7/NIqxq308Zg0RPMNrmn0QvzniL4Hx8Qdwzr6YZWLYAbUSvZYS2ppLR3BFWSkV6SsTJUx8BuDaj3P8vttkrog==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/nested-clients": "3.777.0", + "@aws-sdk/nested-clients": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/property-provider": "^4.0.2", "@smithy/shared-ini-file-loader": "^4.0.2", @@ -7727,9 +7772,9 @@ } }, "node_modules/@aws-sdk/util-dynamodb": { - "version": "3.777.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.777.0.tgz", - "integrity": "sha512-KNYZPcMM/37uOIDLw/5V31b4Q2ThFYr8woD0Y8hvkSX3kP1F8x9/0XlqUWzt0ZfDyG29yqHxnGPLN8kSUcV7gQ==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.787.0.tgz", + "integrity": "sha512-wPccKwwNs36sGoE3GuyJebQentnMnQqf5EaTonqFuC+sfUqp49SwF0waExYKQihp64i0mYpl7UjLL7ALUn9vWw==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -7738,13 +7783,13 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.777.0" + "@aws-sdk/client-dynamodb": "^3.787.0" } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.775.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.775.0.tgz", - "integrity": "sha512-yjWmUgZC9tUxAo8Uaplqmq0eUh0zrbZJdwxGRKdYxfm4RG6fMw1tj52+KkatH7o+mNZvg1GDcVp/INktxonJLw==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.787.0.tgz", + "integrity": "sha512-fd3zkiOkwnbdbN0Xp9TsP5SWrmv0SpT70YEdbb8wAj2DWQwiCmFszaSs+YCvhoCdmlR3Wl9Spu0pGpSAGKeYvQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "3.775.0", @@ -7835,12 +7880,12 @@ } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.775.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.775.0.tgz", - "integrity": "sha512-N9yhTevbizTOMo3drH7Eoy6OkJ3iVPxhV7dwb6CMAObbLneS36CSfA6xQXupmHWcRvZPTz8rd1JGG3HzFOau+g==", + "version": "3.787.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.787.0.tgz", + "integrity": "sha512-mG7Lz8ydfG4SF9e8WSXiPQ/Lsn3n8A5B5jtPROidafi06I3ckV2WxyMLdwG14m919NoS6IOfWHyRGSqWIwbVKA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "3.775.0", + "@aws-sdk/middleware-user-agent": "3.787.0", "@aws-sdk/types": "3.775.0", "@smithy/node-config-provider": "^4.0.2", "@smithy/types": "^4.2.0", @@ -7885,26 +7930,26 @@ } }, "node_modules/@aws-solutions-constructs/aws-cloudfront-s3": { - "version": "2.81.0", - "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/aws-cloudfront-s3/-/aws-cloudfront-s3-2.81.0.tgz", - "integrity": "sha512-5rFNANFYGzwHSAxHjHGqjtqcfJuQTJC4WSfB7kWXkRWxykxb6iFnf6x7jSJ/IgfmMDcOMy+cOAVjEorj4Oelrw==", + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/aws-cloudfront-s3/-/aws-cloudfront-s3-2.83.0.tgz", + "integrity": "sha512-qW6gVmwpi6jqbtMNyT0EW/S9P9FWRg7HssOqaVSrzRdZToqOm7G8cDO0i6gQQ8XiNZsN0BZcWV77HSIRTpWBFA==", "license": "Apache-2.0", "dependencies": { - "@aws-solutions-constructs/core": "2.81.0", - "@aws-solutions-constructs/resources": "2.81.0", + "@aws-solutions-constructs/core": "2.83.0", + "@aws-solutions-constructs/resources": "2.83.0", "constructs": "^10.0.0" }, "peerDependencies": { - "@aws-solutions-constructs/core": "2.81.0", - "@aws-solutions-constructs/resources": "2.81.0", - "aws-cdk-lib": "^2.186.0", + "@aws-solutions-constructs/core": "2.83.0", + "@aws-solutions-constructs/resources": "2.83.0", + "aws-cdk-lib": "^2.187.0", "constructs": "^10.0.0" } }, "node_modules/@aws-solutions-constructs/core": { - "version": "2.81.0", - "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/core/-/core-2.81.0.tgz", - "integrity": "sha512-XkUnBQvuDUsM7XVAYzpdqYuxUHumP6EJFsgfhxS2W+VIT7hjzzCFjbPjnqE8N2EVDnTFQ3QmMgvTIp7pZcEGSA==", + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/core/-/core-2.83.0.tgz", + "integrity": "sha512-afFWkJkS3bQx5zIOT7Zm4UwGWkE8EU90+PB1MLtNhDeg1A3Tsq7DLNaeW6zDytPDnJaohdVRnemJraxwIBXK5Q==", "bundleDependencies": [ "deepmerge", "npmlog", @@ -7918,7 +7963,7 @@ "npmlog": "^7.0.0" }, "peerDependencies": { - "aws-cdk-lib": "^2.186.0", + "aws-cdk-lib": "^2.187.0", "constructs": "^10.0.0" } }, @@ -7950,9 +7995,9 @@ } }, "node_modules/@aws-solutions-constructs/resources": { - "version": "2.81.0", - "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/resources/-/resources-2.81.0.tgz", - "integrity": "sha512-OBnLjR39OrTvaIyYNrTdICzKJ8MIg3J0/m7ub8xibCGZ6hUwVTGWnxt0eOsY68SSV6FS7O1Gr4VlFk8NF3aavA==", + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/@aws-solutions-constructs/resources/-/resources-2.83.0.tgz", + "integrity": "sha512-Uy6adJEMO5rYIlfuZKperfUt6PhE3ZL6N4xPPURGixVyUyHQDmI8/fiSbWY/zr9403UXfL+dZL25GZDsTwEuKw==", "bundleDependencies": [ "@aws-sdk/client-kms", "@aws-sdk/client-s3" @@ -7961,12 +8006,12 @@ "dependencies": { "@aws-sdk/client-kms": "^3.478.0", "@aws-sdk/client-s3": "^3.478.0", - "@aws-solutions-constructs/core": "2.81.0", + "@aws-solutions-constructs/core": "2.83.0", "constructs": "^10.0.0" }, "peerDependencies": { - "@aws-solutions-constructs/core": "2.81.0", - "aws-cdk-lib": "^2.186.0", + "@aws-solutions-constructs/core": "2.83.0", + "aws-cdk-lib": "^2.187.0", "constructs": "^10.0.0" } }, @@ -11463,9 +11508,9 @@ } }, "node_modules/@pkgr/core": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.0.tgz", - "integrity": "sha512-vsJDAkYR6qCPu+ioGScGiMYR7LvZYIXh/dlQeviqoTWNCVfKTLYD/LkNWH4Mxsv2a5vpIRc77FN5DnmK1eBggQ==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.2.tgz", + "integrity": "sha512-25L86MyPvnlQoX2MTIV2OiUcb6vJ6aRbFa9pbwByn95INKD5mFH2smgjDhq+fwJoqAgvgbdJLj6Tz7V9X5CFAQ==", "dev": true, "license": "MIT", "engines": { @@ -11486,24 +11531,24 @@ } }, "node_modules/@radix-ui/number": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.0.tgz", - "integrity": "sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", + "integrity": "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==", "license": "MIT" }, "node_modules/@radix-ui/primitive": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.1.tgz", - "integrity": "sha512-SJ31y+Q/zAyShtXJc8x83i9TYdbAfHZ++tUZnvjJJqFjzsdUnKsxPL6IEtBlxKkU7yzer//GQtZSV4GbldL3YA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.2.tgz", + "integrity": "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA==", "license": "MIT" }, "node_modules/@radix-ui/react-arrow": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.2.tgz", - "integrity": "sha512-G+KcpzXHq24iH0uGG/pF8LyzpFJYGD4RfLjCIBfGdSLXvjLHST31RUiRVrupIBMvIppMgSzQ6l66iAxl03tdlg==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.3.tgz", + "integrity": "sha512-2dvVU4jva0qkNZH6HHWuSz5FN5GeU5tymvCgutF8WaXz9WnD1NgUhy73cqzkjkN4Zkn8lfTPv5JIfrC221W+Nw==", "license": "MIT", "dependencies": { - "@radix-ui/react-primitive": "2.0.2" + "@radix-ui/react-primitive": "2.0.3" }, "peerDependencies": { "@types/react": "*", @@ -11521,15 +11566,15 @@ } }, "node_modules/@radix-ui/react-collection": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.2.tgz", - "integrity": "sha512-9z54IEKRxIa9VityapoEYMuByaG42iSy1ZXlY2KcuLSEtq8x4987/N6m15ppoMffgZX72gER2uHe1D9Y6Unlcw==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.3.tgz", + "integrity": "sha512-mM2pxoQw5HJ49rkzwOs7Y6J4oYH22wS8BfK2/bBxROlI4xuR0c4jEenQP63LlTlDkO6Buj2Vt+QYAYcOgqtrXA==", "license": "MIT", "dependencies": { - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-slot": "1.1.2" + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-slot": "1.2.0" }, "peerDependencies": { "@types/react": "*", @@ -11547,9 +11592,9 @@ } }, "node_modules/@radix-ui/react-compose-refs": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.1.tgz", - "integrity": "sha512-Y9VzoRDSJtgFMUCoiZBDVo084VQ5hfpXxVE+NgkdNsjiDBByiImMZKKhxMwCbdHvhlENG6a833CbFkOQvTricw==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.2.tgz", + "integrity": "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -11562,9 +11607,9 @@ } }, "node_modules/@radix-ui/react-context": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz", - "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.2.tgz", + "integrity": "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -11577,23 +11622,23 @@ } }, "node_modules/@radix-ui/react-dialog": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.6.tgz", - "integrity": "sha512-/IVhJV5AceX620DUJ4uYVMymzsipdKBzo3edo+omeskCKGm9FRHM0ebIdbPnlQVJqyuHbuBltQUOG2mOTq2IYw==", - "license": "MIT", - "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-dismissable-layer": "1.1.5", - "@radix-ui/react-focus-guards": "1.1.1", - "@radix-ui/react-focus-scope": "1.1.2", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-portal": "1.1.4", - "@radix-ui/react-presence": "1.1.2", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-slot": "1.1.2", - "@radix-ui/react-use-controllable-state": "1.1.0", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.7.tgz", + "integrity": "sha512-EIdma8C0C/I6kL6sO02avaCRqi3fmWJpxH6mqbVScorW6nNktzKJT/le7VPho3o/7wCsyRg3z0+Q+Obr0Gy/VQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-dismissable-layer": "1.1.6", + "@radix-ui/react-focus-guards": "1.1.2", + "@radix-ui/react-focus-scope": "1.1.3", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-portal": "1.1.5", + "@radix-ui/react-presence": "1.1.3", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-slot": "1.2.0", + "@radix-ui/react-use-controllable-state": "1.1.1", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, @@ -11613,9 +11658,9 @@ } }, "node_modules/@radix-ui/react-direction": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.0.tgz", - "integrity": "sha512-BUuBvgThEiAXh2DWu93XsT+a3aWrGqolGlqqw5VU1kG7p/ZH2cuDlM1sRLNnY3QcBS69UIz2mcKhMxDsdewhjg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.1.tgz", + "integrity": "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -11628,16 +11673,16 @@ } }, "node_modules/@radix-ui/react-dismissable-layer": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.5.tgz", - "integrity": "sha512-E4TywXY6UsXNRhFrECa5HAvE5/4BFcGyfTyK36gP+pAW1ed7UTK4vKwdr53gAJYwqbfCWC6ATvJa3J3R/9+Qrg==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.6.tgz", + "integrity": "sha512-7gpgMT2gyKym9Jz2ZhlRXSg2y6cNQIK8d/cqBZ0RBCaps8pFryCWXiUKI+uHGFrhMrbGUP7U6PWgiXzIxoyF3Q==", "license": "MIT", "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-callback-ref": "1.1.0", - "@radix-ui/react-use-escape-keydown": "1.1.0" + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11655,18 +11700,18 @@ } }, "node_modules/@radix-ui/react-dropdown-menu": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.6.tgz", - "integrity": "sha512-no3X7V5fD487wab/ZYSHXq3H37u4NVeLDKI/Ks724X/eEFSSEFYZxWgsIlr1UBeEyDaM29HM5x9p1Nv8DuTYPA==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.7.tgz", + "integrity": "sha512-7/1LiuNZuCQE3IzdicGoHdQOHkS2Q08+7p8w6TXZ6ZjgAULaCI85ZY15yPl4o4FVgoKLRT43/rsfNVN8osClQQ==", "license": "MIT", "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-menu": "2.1.6", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-controllable-state": "1.1.0" + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-menu": "2.1.7", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-controllable-state": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11684,9 +11729,9 @@ } }, "node_modules/@radix-ui/react-focus-guards": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz", - "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.2.tgz", + "integrity": "sha512-fyjAACV62oPV925xFCrH8DR5xWhg9KYtJT4s3u54jxp+L/hbpTY2kIeEFFbFe+a/HCE94zGQMZLIpVTPVZDhaA==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -11699,14 +11744,14 @@ } }, "node_modules/@radix-ui/react-focus-scope": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.2.tgz", - "integrity": "sha512-zxwE80FCU7lcXUGWkdt6XpTTCKPitG1XKOwViTxHVKIJhZl9MvIl2dVHeZENCWD9+EdWv05wlaEkRXUykU27RA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.3.tgz", + "integrity": "sha512-4XaDlq0bPt7oJwR+0k0clCiCO/7lO7NKZTAaJBYxDNQT/vj4ig0/UvctrRscZaFREpRvUTkpKR96ov1e6jptQg==", "license": "MIT", "dependencies": { - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-callback-ref": "1.1.0" + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11724,12 +11769,12 @@ } }, "node_modules/@radix-ui/react-id": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz", - "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.1.tgz", + "integrity": "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg==", "license": "MIT", "dependencies": { - "@radix-ui/react-use-layout-effect": "1.1.0" + "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11742,27 +11787,27 @@ } }, "node_modules/@radix-ui/react-menu": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.6.tgz", - "integrity": "sha512-tBBb5CXDJW3t2mo9WlO7r6GTmWV0F0uzHZVFmlRmYpiSK1CDU5IKojP1pm7oknpBOrFZx/YgBRW9oorPO2S/Lg==", - "license": "MIT", - "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-collection": "1.1.2", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-direction": "1.1.0", - "@radix-ui/react-dismissable-layer": "1.1.5", - "@radix-ui/react-focus-guards": "1.1.1", - "@radix-ui/react-focus-scope": "1.1.2", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-popper": "1.2.2", - "@radix-ui/react-portal": "1.1.4", - "@radix-ui/react-presence": "1.1.2", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-roving-focus": "1.1.2", - "@radix-ui/react-slot": "1.1.2", - "@radix-ui/react-use-callback-ref": "1.1.0", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.7.tgz", + "integrity": "sha512-tBODsrk68rOi1/iQzbM54toFF+gSw/y+eQgttFflqlGekuSebNqvFNHjJgjqPhiMb4Fw9A0zNFly1QT6ZFdQ+Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-collection": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-dismissable-layer": "1.1.6", + "@radix-ui/react-focus-guards": "1.1.2", + "@radix-ui/react-focus-scope": "1.1.3", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-popper": "1.2.3", + "@radix-ui/react-portal": "1.1.5", + "@radix-ui/react-presence": "1.1.3", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-roving-focus": "1.1.3", + "@radix-ui/react-slot": "1.2.0", + "@radix-ui/react-use-callback-ref": "1.1.1", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, @@ -11782,24 +11827,24 @@ } }, "node_modules/@radix-ui/react-popover": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.6.tgz", - "integrity": "sha512-NQouW0x4/GnkFJ/pRqsIS3rM/k97VzKnVb2jB7Gq7VEGPy5g7uNV1ykySFt7eWSp3i2uSGFwaJcvIRJBAHmmFg==", - "license": "MIT", - "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-dismissable-layer": "1.1.5", - "@radix-ui/react-focus-guards": "1.1.1", - "@radix-ui/react-focus-scope": "1.1.2", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-popper": "1.2.2", - "@radix-ui/react-portal": "1.1.4", - "@radix-ui/react-presence": "1.1.2", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-slot": "1.1.2", - "@radix-ui/react-use-controllable-state": "1.1.0", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.7.tgz", + "integrity": "sha512-I38OYWDmJF2kbO74LX8UsFydSHWOJuQ7LxPnTefjxxvdvPLempvAnmsyX9UsBlywcbSGpRH7oMLfkUf+ij4nrw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-dismissable-layer": "1.1.6", + "@radix-ui/react-focus-guards": "1.1.2", + "@radix-ui/react-focus-scope": "1.1.3", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-popper": "1.2.3", + "@radix-ui/react-portal": "1.1.5", + "@radix-ui/react-presence": "1.1.3", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-slot": "1.2.0", + "@radix-ui/react-use-controllable-state": "1.1.1", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, @@ -11819,21 +11864,21 @@ } }, "node_modules/@radix-ui/react-popper": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.2.tgz", - "integrity": "sha512-Rvqc3nOpwseCyj/rgjlJDYAgyfw7OC1tTkKn2ivhaMGcYt8FSBlahHOZak2i3QwkRXUXgGgzeEe2RuqeEHuHgA==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.3.tgz", + "integrity": "sha512-iNb9LYUMkne9zIahukgQmHlSBp9XWGeQQ7FvUGNk45ywzOb6kQa+Ca38OphXlWDiKvyneo9S+KSJsLfLt8812A==", "license": "MIT", "dependencies": { "@floating-ui/react-dom": "^2.0.0", - "@radix-ui/react-arrow": "1.1.2", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-callback-ref": "1.1.0", - "@radix-ui/react-use-layout-effect": "1.1.0", - "@radix-ui/react-use-rect": "1.1.0", - "@radix-ui/react-use-size": "1.1.0", - "@radix-ui/rect": "1.1.0" + "@radix-ui/react-arrow": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-layout-effect": "1.1.1", + "@radix-ui/react-use-rect": "1.1.1", + "@radix-ui/react-use-size": "1.1.1", + "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11851,13 +11896,13 @@ } }, "node_modules/@radix-ui/react-portal": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.4.tgz", - "integrity": "sha512-sn2O9k1rPFYVyKd5LAJfo96JlSGVFpa1fS6UuBJfrZadudiw5tAmru+n1x7aMRQ84qDM71Zh1+SzK5QwU0tJfA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.5.tgz", + "integrity": "sha512-ps/67ZqsFm+Mb6lSPJpfhRLrVL2i2fntgCmGMqqth4eaGUf+knAuuRtWVJrNjUhExgmdRqftSgzpf0DF0n6yXA==", "license": "MIT", "dependencies": { - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-layout-effect": "1.1.0" + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11875,13 +11920,13 @@ } }, "node_modules/@radix-ui/react-presence": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.2.tgz", - "integrity": "sha512-18TFr80t5EVgL9x1SwF/YGtfG+l0BS0PRAlCWBDoBEiDQjeKgnNZRVJp/oVBl24sr3Gbfwc/Qpj4OcWTQMsAEg==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.3.tgz", + "integrity": "sha512-IrVLIhskYhH3nLvtcBLQFZr61tBG7wx7O3kEmdzcYwRGAEBmBicGGL7ATzNgruYJ3xBTbuzEEq9OXJM3PAX3tA==", "license": "MIT", "dependencies": { - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-use-layout-effect": "1.1.0" + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11899,12 +11944,12 @@ } }, "node_modules/@radix-ui/react-primitive": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.2.tgz", - "integrity": "sha512-Ec/0d38EIuvDF+GZjcMU/Ze6MxntVJYO/fRlCPhCaVUyPY9WTalHJw54tp9sXeJo3tlShWpy41vQRgLRGOuz+w==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.3.tgz", + "integrity": "sha512-Pf/t/GkndH7CQ8wE2hbkXA+WyZ83fhQQn5DDmwDiDo6AwN/fhaH8oqZ0jRjMrO2iaMhDi6P1HRx6AZwyMinY1g==", "license": "MIT", "dependencies": { - "@radix-ui/react-slot": "1.1.2" + "@radix-ui/react-slot": "1.2.0" }, "peerDependencies": { "@types/react": "*", @@ -11922,20 +11967,20 @@ } }, "node_modules/@radix-ui/react-roving-focus": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.2.tgz", - "integrity": "sha512-zgMQWkNO169GtGqRvYrzb0Zf8NhMHS2DuEB/TiEmVnpr5OqPU3i8lfbxaAmC2J/KYuIQxyoQQ6DxepyXp61/xw==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.3.tgz", + "integrity": "sha512-ufbpLUjZiOg4iYgb2hQrWXEPYX6jOLBbR27bDyAff5GYMRrCzcze8lukjuXVUQvJ6HZe8+oL+hhswDcjmcgVyg==", "license": "MIT", "dependencies": { - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-collection": "1.1.2", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-direction": "1.1.0", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-callback-ref": "1.1.0", - "@radix-ui/react-use-controllable-state": "1.1.0" + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-collection": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-controllable-state": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11953,20 +11998,20 @@ } }, "node_modules/@radix-ui/react-scroll-area": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.3.tgz", - "integrity": "sha512-l7+NNBfBYYJa9tNqVcP2AGvxdE3lmE6kFTBXdvHgUaZuy+4wGCL1Cl2AfaR7RKyimj7lZURGLwFO59k4eBnDJQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.4.tgz", + "integrity": "sha512-G9rdWTQjOR4sk76HwSdROhPU0jZWpfozn9skU1v4N0/g9k7TmswrJn8W8WMU+aYktnLLpk5LX6fofj2bGe5NFQ==", "license": "MIT", "dependencies": { - "@radix-ui/number": "1.1.0", - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-direction": "1.1.0", - "@radix-ui/react-presence": "1.1.2", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-callback-ref": "1.1.0", - "@radix-ui/react-use-layout-effect": "1.1.0" + "@radix-ui/number": "1.1.1", + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-presence": "1.1.3", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -11984,30 +12029,30 @@ } }, "node_modules/@radix-ui/react-select": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.1.6.tgz", - "integrity": "sha512-T6ajELxRvTuAMWH0YmRJ1qez+x4/7Nq7QIx7zJ0VK3qaEWdnWpNbEDnmWldG1zBDwqrLy5aLMUWcoGirVj5kMg==", - "license": "MIT", - "dependencies": { - "@radix-ui/number": "1.1.0", - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-collection": "1.1.2", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-direction": "1.1.0", - "@radix-ui/react-dismissable-layer": "1.1.5", - "@radix-ui/react-focus-guards": "1.1.1", - "@radix-ui/react-focus-scope": "1.1.2", - "@radix-ui/react-id": "1.1.0", - "@radix-ui/react-popper": "1.2.2", - "@radix-ui/react-portal": "1.1.4", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-slot": "1.1.2", - "@radix-ui/react-use-callback-ref": "1.1.0", - "@radix-ui/react-use-controllable-state": "1.1.0", - "@radix-ui/react-use-layout-effect": "1.1.0", - "@radix-ui/react-use-previous": "1.1.0", - "@radix-ui/react-visually-hidden": "1.1.2", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.1.7.tgz", + "integrity": "sha512-exzGIRtc7S8EIM2KjFg+7lJZsH7O7tpaBaJbBNVDnOZNhtoQ2iV+iSNfi2Wth0m6h3trJkMVvzAehB3c6xj/3Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.1", + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-collection": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-dismissable-layer": "1.1.6", + "@radix-ui/react-focus-guards": "1.1.2", + "@radix-ui/react-focus-scope": "1.1.3", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-popper": "1.2.3", + "@radix-ui/react-portal": "1.1.5", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-slot": "1.2.0", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-controllable-state": "1.1.1", + "@radix-ui/react-use-layout-effect": "1.1.1", + "@radix-ui/react-use-previous": "1.1.1", + "@radix-ui/react-visually-hidden": "1.1.3", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, @@ -12027,12 +12072,12 @@ } }, "node_modules/@radix-ui/react-separator": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.2.tgz", - "integrity": "sha512-oZfHcaAp2Y6KFBX6I5P1u7CQoy4lheCGiYj+pGFrHy8E/VNRb5E39TkTr3JrV520csPBTZjkuKFdEsjS5EUNKQ==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.3.tgz", + "integrity": "sha512-2omrWKJvxR0U/tkIXezcc1nFMwtLU0+b/rDK40gnzJqTLWQ/TD/D5IYVefp9sC3QWfeQbpSbEA6op9MQKyaALQ==", "license": "MIT", "dependencies": { - "@radix-ui/react-primitive": "2.0.2" + "@radix-ui/react-primitive": "2.0.3" }, "peerDependencies": { "@types/react": "*", @@ -12050,22 +12095,22 @@ } }, "node_modules/@radix-ui/react-slider": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.2.3.tgz", - "integrity": "sha512-nNrLAWLjGESnhqBqcCNW4w2nn7LxudyMzeB6VgdyAnFLC6kfQgnAjSL2v6UkQTnDctJBlxrmxfplWS4iYjdUTw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.2.4.tgz", + "integrity": "sha512-Vr/OgNejNJPAghIhjS7Mf/2F/EXGDT0qgtiHf2BHz71+KqgN+jndFLKq5xAB9JOGejGzejfJLIvT04Do+yzhcg==", "license": "MIT", "dependencies": { - "@radix-ui/number": "1.1.0", - "@radix-ui/primitive": "1.1.1", - "@radix-ui/react-collection": "1.1.2", - "@radix-ui/react-compose-refs": "1.1.1", - "@radix-ui/react-context": "1.1.1", - "@radix-ui/react-direction": "1.1.0", - "@radix-ui/react-primitive": "2.0.2", - "@radix-ui/react-use-controllable-state": "1.1.0", - "@radix-ui/react-use-layout-effect": "1.1.0", - "@radix-ui/react-use-previous": "1.1.0", - "@radix-ui/react-use-size": "1.1.0" + "@radix-ui/number": "1.1.1", + "@radix-ui/primitive": "1.1.2", + "@radix-ui/react-collection": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-primitive": "2.0.3", + "@radix-ui/react-use-controllable-state": "1.1.1", + "@radix-ui/react-use-layout-effect": "1.1.1", + "@radix-ui/react-use-previous": "1.1.1", + "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -12083,12 +12128,12 @@ } }, "node_modules/@radix-ui/react-slot": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.1.2.tgz", - "integrity": "sha512-YAKxaiGsSQJ38VzKH86/BPRC4rh+b1Jpa+JneA5LRE7skmLPNAyeG8kPJj/oo4STLvlrs8vkf/iYyc3A5stYCQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.0.tgz", + "integrity": "sha512-ujc+V6r0HNDviYqIK3rW4ffgYiZ8g5DEHrGJVk4x7kTlLXRDILnKX9vAUYeIsLOoDpDJ0ujpqMkjH4w2ofuo6w==", "license": "MIT", "dependencies": { - "@radix-ui/react-compose-refs": "1.1.1" + "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", @@ -12101,9 +12146,9 @@ } }, "node_modules/@radix-ui/react-use-callback-ref": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz", - "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.1.tgz", + "integrity": "sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -12116,12 +12161,12 @@ } }, "node_modules/@radix-ui/react-use-controllable-state": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz", - "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.1.tgz", + "integrity": "sha512-YnEXIy8/ga01Y1PN0VfaNH//MhA91JlEGVBDxDzROqwrAtG5Yr2QGEPz8A/rJA3C7ZAHryOYGaUv8fLSW2H/mg==", "license": "MIT", "dependencies": { - "@radix-ui/react-use-callback-ref": "1.1.0" + "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -12134,12 +12179,12 @@ } }, "node_modules/@radix-ui/react-use-escape-keydown": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz", - "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.1.tgz", + "integrity": "sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g==", "license": "MIT", "dependencies": { - "@radix-ui/react-use-callback-ref": "1.1.0" + "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -12152,9 +12197,9 @@ } }, "node_modules/@radix-ui/react-use-layout-effect": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz", - "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.1.tgz", + "integrity": "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -12167,9 +12212,9 @@ } }, "node_modules/@radix-ui/react-use-previous": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.0.tgz", - "integrity": "sha512-Z/e78qg2YFnnXcW88A4JmTtm4ADckLno6F7OXotmkQfeuCVaKuYzqAATPhVzl3delXE7CxIV8shofPn3jPc5Og==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.1.tgz", + "integrity": "sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ==", "license": "MIT", "peerDependencies": { "@types/react": "*", @@ -12182,12 +12227,12 @@ } }, "node_modules/@radix-ui/react-use-rect": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz", - "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.1.tgz", + "integrity": "sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w==", "license": "MIT", "dependencies": { - "@radix-ui/rect": "1.1.0" + "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -12200,12 +12245,12 @@ } }, "node_modules/@radix-ui/react-use-size": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz", - "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.1.tgz", + "integrity": "sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ==", "license": "MIT", "dependencies": { - "@radix-ui/react-use-layout-effect": "1.1.0" + "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", @@ -12218,12 +12263,12 @@ } }, "node_modules/@radix-ui/react-visually-hidden": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.1.2.tgz", - "integrity": "sha512-1SzA4ns2M1aRlvxErqhLHsBHoS5eI5UUcI2awAMgGUp4LoaoWOKYmvqDY2s/tltuPkh3Yk77YF/r3IRj+Amx4Q==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.1.3.tgz", + "integrity": "sha512-oXSF3ZQRd5fvomd9hmUCb2EHSZbPp3ZSHAHJJU/DlF9XoFkJBBW8RHU/E8WEH+RbSfJd/QFA0sl8ClJXknBwHQ==", "license": "MIT", "dependencies": { - "@radix-ui/react-primitive": "2.0.2" + "@radix-ui/react-primitive": "2.0.3" }, "peerDependencies": { "@types/react": "*", @@ -12241,9 +12286,9 @@ } }, "node_modules/@radix-ui/rect": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz", - "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.1.tgz", + "integrity": "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==", "license": "MIT" }, "node_modules/@react-icons/all-files": { @@ -14579,9 +14624,9 @@ "license": "MIT" }, "node_modules/@types/aws-lambda": { - "version": "8.10.148", - "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.148.tgz", - "integrity": "sha512-JL+2cfkY9ODQeE06hOxSFNkafjNk4JRBgY837kpoq1GHDttq2U3BA9IzKOWxS4DLjKoymGB4i9uBrlCkjUl1yg==", + "version": "8.10.149", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.149.tgz", + "integrity": "sha512-NXSZIhfJjnXqJgtS7IwutqIF/SOy1Wz5Px4gUY1RWITp3AYTyuJS4xaXr/bIJY1v15XMzrJ5soGnPM+7uigZjA==", "license": "MIT" }, "node_modules/@types/babel__core": { @@ -14599,9 +14644,9 @@ } }, "node_modules/@types/babel__generator": { - "version": "7.6.8", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", - "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", "devOptional": true, "license": "MIT", "dependencies": { @@ -15491,6 +15536,70 @@ "react-dom": ">=16.8.0" } }, + "node_modules/@uiw/react-markdown-preview/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/@uiw/react-markdown-preview/node_modules/hast-util-parse-selector": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz", + "integrity": "sha512-jdlwBjEexy1oGz0aJ2f4GKMaVKkA9jwjr4MjAAI22E5fM/TXVZHuS5OpONtdeIkRKqAaryQ2E9xNQxijoThSZA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/hast-util-parse-selector/node_modules/@types/hast": { + "version": "2.3.10", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.10.tgz", + "integrity": "sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/hastscript": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-7.2.0.tgz", + "integrity": "sha512-TtYPq24IldU8iKoJQqvZOuhi5CyCQRAbvDOX0x1eW6rsHSxa/1i2CCiptNTotGHJ3VoHRGmqiv6/D3q113ikkw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^3.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/hastscript/node_modules/@types/hast": { + "version": "2.3.10", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.10.tgz", + "integrity": "sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/property-information": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", + "integrity": "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/@uiw/react-markdown-preview/node_modules/react-markdown": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.3.tgz", @@ -15517,6 +15626,45 @@ "react": ">=18" } }, + "node_modules/@uiw/react-markdown-preview/node_modules/refractor": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-4.9.0.tgz", + "integrity": "sha512-nEG1SPXFoGGx+dcjftjv8cAjEusIh6ED1xhf5DG3C0x/k+rmZ2duKnc3QLpt6qeHv5fPb8uwN3VWN2BT7fr3Og==", + "license": "MIT", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/prismjs": "^1.0.0", + "hastscript": "^7.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/refractor/node_modules/@types/hast": { + "version": "2.3.10", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.10.tgz", + "integrity": "sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2" + } + }, + "node_modules/@uiw/react-markdown-preview/node_modules/rehype-prism-plus": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/rehype-prism-plus/-/rehype-prism-plus-2.0.0.tgz", + "integrity": "sha512-FeM/9V2N7EvDZVdR2dqhAzlw5YI49m9Tgn7ZrYJeYHIahM6gcXpH0K1y2gNnKanZCydOMluJvX2cB9z3lhY8XQ==", + "license": "MIT", + "dependencies": { + "hast-util-to-string": "^3.0.0", + "parse-numeric-range": "^1.3.0", + "refractor": "^4.8.0", + "rehype-parse": "^9.0.0", + "unist-util-filter": "^5.0.0", + "unist-util-visit": "^5.0.0" + } + }, "node_modules/@uiw/react-md-editor": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@uiw/react-md-editor/-/react-md-editor-4.0.5.tgz", @@ -16151,18 +16299,18 @@ } }, "node_modules/aws-amplify": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/aws-amplify/-/aws-amplify-6.14.1.tgz", - "integrity": "sha512-O/hGulvMjHHdf+VqIpRdfM9fHzE0eOG7zxnBl7yEvZpAejj2/jnSSzUf2sXoQM2Kdvqw7KuaTERZD3U3v6Z74g==", - "license": "Apache-2.0", - "dependencies": { - "@aws-amplify/analytics": "7.0.77", - "@aws-amplify/api": "6.3.8", - "@aws-amplify/auth": "6.12.1", - "@aws-amplify/core": "6.11.1", - "@aws-amplify/datastore": "5.0.79", - "@aws-amplify/notifications": "2.0.77", - "@aws-amplify/storage": "6.8.1", + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/aws-amplify/-/aws-amplify-6.14.2.tgz", + "integrity": "sha512-kBPTb211EdGz6TG3Syamd9h9wPSujVuTq5/iYzcSL2gEOp2XO2aPeHFuiCA5ey9GspcCcNAHvOb/3vsc6lS4hw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-amplify/analytics": "7.0.78", + "@aws-amplify/api": "6.3.9", + "@aws-amplify/auth": "6.12.2", + "@aws-amplify/core": "6.11.2", + "@aws-amplify/datastore": "5.0.80", + "@aws-amplify/notifications": "2.0.78", + "@aws-amplify/storage": "6.8.2", "tslib": "^2.5.0" } }, @@ -16183,9 +16331,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.187.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.187.0.tgz", - "integrity": "sha512-OrAWin+LD5sZhRF5cWuEYEkmC/sxxlgcAasCpfzeRsj6yDImwmeQsaKhM7xqzZQBInog6ZbN6oFZYiWEGJMSIA==", + "version": "2.189.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.189.0.tgz", + "integrity": "sha512-B5Uha7uRntOAyuKfU0eFtxij3ZVTzGAbetw5qaXlURa68wsWpKlU72/OyKugB6JYkhjCZkSTVVBxd1pVTosxEw==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -17268,9 +17416,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001709", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001709.tgz", - "integrity": "sha512-NgL3vUTnDrPCZ3zTahp4fsugQ4dc7EKTSzwQDPEel6DMoMnfH2jhry9n2Zm8onbSR+f/QtKHFOA+iAQu4kbtWA==", + "version": "1.0.30001713", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001713.tgz", + "integrity": "sha512-wCIWIg+A4Xr7NfhTuHdX+/FKh3+Op3LBbSp2N5Pfx6T/LhdQy3GTyoTg48BReaW/MyMNZAkTadsBtai3ldWK0Q==", "funding": [ { "type": "opencollective", @@ -17653,9 +17801,9 @@ "license": "MIT" }, "node_modules/confbox": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.1.tgz", - "integrity": "sha512-hkT3yDPFbs95mNCy1+7qNKC6Pro+/ibzYxtM2iqEigpf0sVw+bg4Zh9/snjsBcf990vfIsg5+1U7VyiyBb3etg==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.2.tgz", + "integrity": "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ==", "license": "MIT" }, "node_modules/console-browserify": { @@ -19042,9 +19190,9 @@ } }, "node_modules/dompurify": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.4.tgz", - "integrity": "sha512-ysFSFEDVduQpyhzAob/kkuJjf5zWkZD8/A9ywSp1byueyuCfHamrCBa14/Oc2iiB0e51B+NpxSl5gmzn+Ms/mg==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.5.tgz", + "integrity": "sha512-mLPd29uoRe9HpvwP2TxClGQBzGXeEC/we/q+bFlmPPmj2p2Ugl3r6ATu/UU1v77DXNcehiBg9zsr1dREyA/dJQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -19111,9 +19259,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.130", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.130.tgz", - "integrity": "sha512-Ou2u7L9j2XLZbhqzyX0jWDj6gA8D3jIfVzt4rikLf3cGBa0VdReuFimBKS9tQJA4+XpeCxj1NoWlfBXzbMa9IA==", + "version": "1.5.136", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.136.tgz", + "integrity": "sha512-kL4+wUTD7RSA5FHx5YwWtjDnEEkIIikFgWHR4P6fqjw1PPLlqYkxeOb++wAauAssat0YClCy8Y3C5SxgSkjibQ==", "license": "ISC" }, "node_modules/elliptic": { @@ -19868,9 +20016,9 @@ } }, "node_modules/eslint-plugin-react": { - "version": "7.37.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.4.tgz", - "integrity": "sha512-BGP0jRmfYyvOyvMoRX/uoUeW+GqNj9y16bPQzqAHf3AYII/tDs+jMN0dBVkl88/OZwNGwrVFxE7riHsXVfy/LQ==", + "version": "7.37.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.5.tgz", + "integrity": "sha512-Qteup0SqU15kdocexFNAJMvCJEfa2xUKNV4CC1xsVMrIIqEy3SQ/rqyxCWNzfrd3/ldy6HMlD2e0JDVpDg2qIA==", "dev": true, "license": "MIT", "dependencies": { @@ -19884,7 +20032,7 @@ "hasown": "^2.0.2", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", - "object.entries": "^1.1.8", + "object.entries": "^1.1.9", "object.fromentries": "^2.0.8", "object.values": "^1.2.1", "prop-types": "^15.8.1", @@ -21966,6 +22114,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -24836,9 +24985,9 @@ } }, "node_modules/katex": { - "version": "0.16.21", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.21.tgz", - "integrity": "sha512-XvqR7FgOHtWupfMiigNzmh+MgUVmDGU2kXZm899ZkPfcuoPuFxyHmXsgATDpFZDAXCI8tvinaVcDo8PIIJSo4A==", + "version": "0.16.22", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.22.tgz", + "integrity": "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==", "funding": [ "https://opencollective.com/katex", "https://github.com/sponsors/katex" @@ -25261,9 +25410,9 @@ } }, "node_modules/marked": { - "version": "15.0.7", - "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.7.tgz", - "integrity": "sha512-dgLIeKGLx5FwziAnsk4ONoGwHwGPJzselimvlVskE9XLN4Orv9u2VA3GWw/lYUqjfA0rUT/6fqKwfZJapP9BEg==", + "version": "15.0.8", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.8.tgz", + "integrity": "sha512-rli4l2LyZqpQuRve5C0rkn6pj3hT8EWPC+zkAxFTAJLxRbENfTAhEQq9itrmf1Y81QtAX5D/MYlGlIomNgj9lA==", "license": "MIT", "bin": { "marked": "bin/marked.js" @@ -28011,9 +28160,9 @@ } }, "node_modules/prosemirror-view": { - "version": "1.38.1", - "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.38.1.tgz", - "integrity": "sha512-4FH/uM1A4PNyrxXbD+RAbAsf0d/mM0D/wAKSVVWK7o0A9Q/oOXJBrw786mBf2Vnrs/Edly6dH6Z2gsb7zWwaUw==", + "version": "1.39.1", + "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.39.1.tgz", + "integrity": "sha512-GhLxH1xwnqa5VjhJ29LfcQITNDp+f1jzmMPXQfGW9oNrF0lfjPzKvV5y/bjIQkyKpwCX3Fp+GA4dBpMMk8g+ZQ==", "license": "MIT", "dependencies": { "prosemirror-model": "^1.20.0", @@ -29070,9 +29219,9 @@ } }, "node_modules/rehype-prism-plus": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/rehype-prism-plus/-/rehype-prism-plus-2.0.0.tgz", - "integrity": "sha512-FeM/9V2N7EvDZVdR2dqhAzlw5YI49m9Tgn7ZrYJeYHIahM6gcXpH0K1y2gNnKanZCydOMluJvX2cB9z3lhY8XQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/rehype-prism-plus/-/rehype-prism-plus-2.0.1.tgz", + "integrity": "sha512-Wglct0OW12tksTUseAPyWPo3srjBOY7xKlql/DPKi7HbsdZTyaLCAoO58QBKSczFQxElTsQlOY3JDOFzB/K++Q==", "license": "MIT", "dependencies": { "hast-util-to-string": "^3.0.0", @@ -29739,6 +29888,7 @@ "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver" @@ -30141,9 +30291,9 @@ "license": "MIT" }, "node_modules/std-env": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.8.1.tgz", - "integrity": "sha512-vj5lIj3Mwf9D79hBkltk5qmkFI+biIKWS2IBxEyEU3AX1tUf7AoL8nSazCOiiqQsGKIq01SClsKEzweu34uwvA==", + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.9.0.tgz", + "integrity": "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==", "dev": true, "license": "MIT" }, @@ -30632,13 +30782,13 @@ } }, "node_modules/synckit": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.1.tgz", - "integrity": "sha512-fWZqNBZNNFp/7mTUy1fSsydhKsAKJ+u90Nk7kOK5Gcq9vObaqLBLjWFDBkyVU9Vvc6Y71VbOevMuGhqv02bT+Q==", + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.3.tgz", + "integrity": "sha512-szhWDqNNI9etJUvbZ1/cx1StnZx8yMmFxme48SwR4dty4ioSY50KEZlpv0qAfgc1fpRzuh9hBXEzoCpJ779dLg==", "dev": true, "license": "MIT", "dependencies": { - "@pkgr/core": "^0.2.0", + "@pkgr/core": "^0.2.1", "tslib": "^2.8.1" }, "engines": { @@ -31436,15 +31586,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.29.0.tgz", - "integrity": "sha512-ep9rVd9B4kQsZ7ZnWCVxUE/xDLUUUsRzE0poAeNu+4CkFErLfuvPt/qtm2EpnSyfvsR0S6QzDFSrPCFBwf64fg==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.29.1.tgz", + "integrity": "sha512-f8cDkvndhbQMPcysk6CUSGBWV+g1utqdn71P5YKwMumVMOG/5k7cHq0KyG4O52nB0oKS4aN2Tp5+wB4APJGC+w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.29.0", - "@typescript-eslint/parser": "8.29.0", - "@typescript-eslint/utils": "8.29.0" + "@typescript-eslint/eslint-plugin": "8.29.1", + "@typescript-eslint/parser": "8.29.1", + "@typescript-eslint/utils": "8.29.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31459,17 +31609,17 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.29.0.tgz", - "integrity": "sha512-PAIpk/U7NIS6H7TEtN45SPGLQaHNgB7wSjsQV/8+KYokAb2T/gloOA/Bee2yd4/yKVhPKe5LlaUGhAZk5zmSaQ==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.29.1.tgz", + "integrity": "sha512-ba0rr4Wfvg23vERs3eB+P3lfj2E+2g3lhWcCVukUuhtcdUx5lSIFZlGFEBHKr+3zizDa/TvZTptdNHVZWAkSBg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.29.0", - "@typescript-eslint/type-utils": "8.29.0", - "@typescript-eslint/utils": "8.29.0", - "@typescript-eslint/visitor-keys": "8.29.0", + "@typescript-eslint/scope-manager": "8.29.1", + "@typescript-eslint/type-utils": "8.29.1", + "@typescript-eslint/utils": "8.29.1", + "@typescript-eslint/visitor-keys": "8.29.1", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -31489,16 +31639,16 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/parser": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.29.0.tgz", - "integrity": "sha512-8C0+jlNJOwQso2GapCVWWfW/rzaq7Lbme+vGUFKE31djwNncIpgXD7Cd4weEsDdkoZDjH0lwwr3QDQFuyrMg9g==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.29.1.tgz", + "integrity": "sha512-zczrHVEqEaTwh12gWBIJWj8nx+ayDcCJs06yoNMY0kwjMWDM6+kppljY+BxWI06d2Ja+h4+WdufDcwMnnMEWmg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.29.0", - "@typescript-eslint/types": "8.29.0", - "@typescript-eslint/typescript-estree": "8.29.0", - "@typescript-eslint/visitor-keys": "8.29.0", + "@typescript-eslint/scope-manager": "8.29.1", + "@typescript-eslint/types": "8.29.1", + "@typescript-eslint/typescript-estree": "8.29.1", + "@typescript-eslint/visitor-keys": "8.29.1", "debug": "^4.3.4" }, "engines": { @@ -31514,14 +31664,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/scope-manager": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.29.0.tgz", - "integrity": "sha512-aO1PVsq7Gm+tcghabUpzEnVSFMCU4/nYIgC2GOatJcllvWfnhrgW0ZEbnTxm36QsikmCN1K/6ZgM7fok2I7xNw==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.29.1.tgz", + "integrity": "sha512-2nggXGX5F3YrsGN08pw4XpMLO1Rgtnn4AzTegC2MDesv6q3QaTU5yU7IbS1tf1IwCR0Hv/1EFygLn9ms6LIpDA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.0", - "@typescript-eslint/visitor-keys": "8.29.0" + "@typescript-eslint/types": "8.29.1", + "@typescript-eslint/visitor-keys": "8.29.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31532,14 +31682,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/type-utils": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.29.0.tgz", - "integrity": "sha512-ahaWQ42JAOx+NKEf5++WC/ua17q5l+j1GFrbbpVKzFL/tKVc0aYY8rVSYUpUvt2hUP1YBr7mwXzx+E/DfUWI9Q==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.29.1.tgz", + "integrity": "sha512-DkDUSDwZVCYN71xA4wzySqqcZsHKic53A4BLqmrWFFpOpNSoxX233lwGu/2135ymTCR04PoKiEEEvN1gFYg4Tw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.29.0", - "@typescript-eslint/utils": "8.29.0", + "@typescript-eslint/typescript-estree": "8.29.1", + "@typescript-eslint/utils": "8.29.1", "debug": "^4.3.4", "ts-api-utils": "^2.0.1" }, @@ -31556,9 +31706,9 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/types": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.29.0.tgz", - "integrity": "sha512-wcJL/+cOXV+RE3gjCyl/V2G877+2faqvlgtso/ZRbTCnZazh0gXhe+7gbAnfubzN2bNsBtZjDvlh7ero8uIbzg==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.29.1.tgz", + "integrity": "sha512-VT7T1PuJF1hpYC3AGm2rCgJBjHL3nc+A/bhOp9sGMKfi5v0WufsX/sHCFBfNTx2F+zA6qBc/PD0/kLRLjdt8mQ==", "dev": true, "license": "MIT", "engines": { @@ -31570,14 +31720,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/typescript-estree": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.29.0.tgz", - "integrity": "sha512-yOfen3jE9ISZR/hHpU/bmNvTtBW1NjRbkSFdZOksL1N+ybPEE7UVGMwqvS6CP022Rp00Sb0tdiIkhSCe6NI8ow==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.29.1.tgz", + "integrity": "sha512-l1enRoSaUkQxOQnbi0KPUtqeZkSiFlqrx9/3ns2rEDhGKfTa+88RmXqedC1zmVTOWrLc2e6DEJrTA51C9iLH5g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.0", - "@typescript-eslint/visitor-keys": "8.29.0", + "@typescript-eslint/types": "8.29.1", + "@typescript-eslint/visitor-keys": "8.29.1", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -31597,16 +31747,16 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/utils": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.29.0.tgz", - "integrity": "sha512-gX/A0Mz9Bskm8avSWFcK0gP7cZpbY4AIo6B0hWYFCaIsz750oaiWR4Jr2CI+PQhfW1CpcQr9OlfPS+kMFegjXA==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.29.1.tgz", + "integrity": "sha512-QAkFEbytSaB8wnmB+DflhUPz6CLbFWE2SnSCrRMEa+KnXIzDYbpsn++1HGvnfAsUY44doDXmvRkO5shlM/3UfA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.29.0", - "@typescript-eslint/types": "8.29.0", - "@typescript-eslint/typescript-estree": "8.29.0" + "@typescript-eslint/scope-manager": "8.29.1", + "@typescript-eslint/types": "8.29.1", + "@typescript-eslint/typescript-estree": "8.29.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31621,13 +31771,13 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/visitor-keys": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.29.0.tgz", - "integrity": "sha512-Sne/pVz8ryR03NFK21VpN88dZ2FdQXOlq3VIklbrTYEt8yXtRFr9tvUhqvCeKjqYk5FSim37sHbooT6vzBTZcg==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.29.1.tgz", + "integrity": "sha512-RGLh5CRaUEf02viP5c1Vh1cMGffQscyHe7HPAzGpfmfflFg1wUz2rYxd+OZqwpeypYvZ8UxSxuIpF++fmOzEcg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.0", + "@typescript-eslint/types": "8.29.1", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -31710,9 +31860,9 @@ "license": "MIT" }, "node_modules/ufo": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.5.4.tgz", - "integrity": "sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.1.tgz", + "integrity": "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==", "license": "MIT" }, "node_modules/uint8array-extras": { @@ -32208,9 +32358,9 @@ } }, "node_modules/vite": { - "version": "6.2.4", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.4.tgz", - "integrity": "sha512-veHMSew8CcRzhL5o8ONjy8gkfmFJAd5Ac16oxBUjlwgX3Gq2Wqr+qNC3TjPIpy7TPV/KporLga5GT9HqdrCizw==", + "version": "6.2.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", + "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", "license": "MIT", "dependencies": { "esbuild": "^0.25.0", @@ -33231,6 +33381,27 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xstate": { "version": "4.38.3", "resolved": "https://registry.npmjs.org/xstate/-/xstate-4.38.3.tgz", @@ -33408,9 +33579,12 @@ "@aws-sdk/client-sagemaker-runtime": "^3.755.0", "@aws-sdk/client-transcribe": "^3.755.0", "@aws-sdk/client-transcribe-streaming": "^3.755.0", + "@aws-sdk/credential-providers": "^3.755.0", "@aws-sdk/lib-dynamodb": "^3.755.0", "@aws-sdk/s3-request-presigner": "^3.755.0", "@aws-solutions-constructs/aws-cloudfront-s3": "^2.68.0", + "@smithy/node-http-handler": "^4.0.4", + "aws-amplify": "^6.14.2", "aws-cdk-lib": "^2.154.1", "aws-jwt-verify": "^4.0.0", "constructs": "^10.3.0", @@ -33419,6 +33593,7 @@ "sanitize-html": "^2.13.0", "source-map-support": "^0.5.21", "upsert-slr": "^1.0.4", + "ws": "^8.18.0", "zod": "^3.24.1" }, "devDependencies": { @@ -33437,6 +33612,7 @@ } }, "packages/common": { + "name": "@generative-ai-use-cases/common", "dependencies": { "aws-jwt-verify": "^4.0.0" } @@ -33458,6 +33634,7 @@ } }, "packages/types": { + "name": "@types/generative-ai-use-cases", "dependencies": { "@aws-sdk/client-bedrock-agent-runtime": "^3.755.0", "@aws-sdk/client-bedrock-runtime": "^3.755.0", diff --git a/packages/cdk/lambda/nova-sonic-lambda.ts b/packages/cdk/lambda/nova-sonic-lambda.ts new file mode 100644 index 000000000..5ad4e401a --- /dev/null +++ b/packages/cdk/lambda/nova-sonic-lambda.ts @@ -0,0 +1,538 @@ +import { Amplify } from 'aws-amplify'; +import { events } from 'aws-amplify/data'; +import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import { + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamCommand, + InvokeModelWithBidirectionalStreamInput, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttp2Handler } from "@smithy/node-http-handler"; +import { randomUUID } from "crypto"; + +// WebSocketをグローバルに設定（Lambda環境用） +Object.assign(global, { WebSocket: require('ws') }); + +// AppSync Events のチャンネル名 +const CHANNEL_NAME = '/default/nova-sonic'; + +// Nova Sonic の設定 +const DEFAULT_INFERENCE_CONFIG = { + maxTokens: 1024, + topP: 0.9, + temperature: 0.7, +}; + +const DEFAULT_AUDIO_INPUT_CONFIG = { + audioType: "SPEECH", + encoding: "base64", + mediaType: "audio/lpcm", + sampleRateHertz: 16000, + sampleSizeBits: 16, + channelCount: 1, +}; + +const DEFAULT_AUDIO_OUTPUT_CONFIG = { + ...DEFAULT_AUDIO_INPUT_CONFIG, + sampleRateHertz: 24000, + voiceId: "tiffany", +}; + +const DEFAULT_TEXT_CONFIG = { + mediaType: "text/plain" +}; + +const DEFAULT_SYSTEM_PROMPT = "あなたは親切なAIアシスタントです。ユーザーの質問に簡潔に答えてください。"; + +// セッション管理用のマップ +const activeSessions = new Map(); + +export const handler = async (event: any) => { + try { + console.log('Lambda function started'); + console.log('Event:', JSON.stringify(event, null, 2)); + + // AppSync Events の設定 + Amplify.configure( + { + API: { + Events: { + endpoint: `${process.env.EVENT_API_ENDPOINT!}/event`, + region: process.env.AWS_DEFAULT_REGION!, + defaultAuthMode: 'iam', + }, + }, + }, + { + Auth: { + credentialsProvider: { + getCredentialsAndIdentityId: async () => { + const provider = fromNodeProviderChain(); + const credentials = await provider(); + return { + credentials, + }; + }, + clearCredentialsAndIdentityId: async () => {}, + }, + }, + } + ); + + // Bedrock クライアントの初期化 + const bedrockClient = new BedrockRuntimeClient({ + region: 'us-east-1', // TODO + requestHandler: new NodeHttp2Handler({ + requestTimeout: 300000, + sessionTimeout: 300000, + }) + }); + + // AppSync Events に接続 + console.log(`Connecting to AppSync Events channel: ${CHANNEL_NAME}`); + const appSyncChannel = await events.connect(CHANNEL_NAME); + console.log('Connected to AppSync Events!'); + + // AppSync Events からのメッセージ受信 + appSyncChannel.subscribe({ + next: async (data: any) => { + if (data.event?.type === 'ClientToAppSync') { + console.log('Received data from client:', data.event.data); + + const clientData = data.event.data; + const sessionId = clientData.sessionId; + + if (!sessionId) { + console.error('No sessionId provided in client data'); + return; + } + + // セッション開始アクション + if (clientData.action === 'startSession') { + console.log(`Starting new session: ${sessionId}`); + await startNovaSession(sessionId, bedrockClient, appSyncChannel); + return; + } + + // 録音停止アクション + if (clientData.action === 'stopRecording') { + console.log(`Stopping recording for session: ${sessionId}`); + const session = activeSessions.get(sessionId); + if (session) { + await endAudioContent(session); + } + return; + } + + // 音声データの処理 + if (clientData.audioData) { + let session = activeSessions.get(sessionId); + + // セッションがなければ新規作成 + if (!session) { + console.log(`Creating new session for: ${sessionId}`); + session = await startNovaSession(sessionId, bedrockClient, appSyncChannel); + } + + // 音声データをNova Sonicに送信 + await streamAudioToNova(session, clientData.audioData); + } + } + }, + error: (e: any) => { + console.error('Error in AppSync Events subscription:', e); + }, + }); + + // 実際は Bedrock のデータ受信待ちで止まるが、ここでは仮の処理として sleep で止める + await new Promise(s => setTimeout(s, 15 * 60 * 1000)); // 15分待機 + + // クリーンアップ + console.log('Cleaning up sessions before exit'); + for (const [sessionId, session] of activeSessions.entries()) { + try { + await closeSession(session); + console.log(`Closed session: ${sessionId}`); + } catch (err) { + console.error(`Error closing session ${sessionId}:`, err); + } + } + + return { statusCode: 200, body: 'Lambda execution completed' }; + } catch (error) { + console.error('Error in Lambda handler:', error); + return { statusCode: 500, body: 'Error in Lambda handler' }; + } +}; + +// Nova Sonicセッションを開始する関数 +async function startNovaSession(sessionId: string, bedrockClient: BedrockRuntimeClient, appSyncChannel: any) { + console.log(`Initializing Nova Sonic session: ${sessionId}`); + + // セッション情報を作成 + const session = { + sessionId, + bedrockClient, + appSyncChannel, + promptName: randomUUID(), + audioContentId: randomUUID(), + queue: [], + isActive: true, + isPromptStartSent: false, + isAudioContentStartSent: false, + responseProcessor: null as any + }; + + // セッションを保存 + activeSessions.set(sessionId, session); + + try { + // Nova Sonicとの双方向ストリームを開始 + await startNovaStream(session); + return session; + } catch (error) { + console.error(`Error in startNovaSession: ${error}`); + activeSessions.delete(sessionId); + throw error; + } +} + +// Nova Sonicとの双方向ストリームを開始する関数 +async function startNovaStream(session: any) { + try { + console.log(`Starting bidirectional stream for session: ${session.sessionId}`); + + // 非同期イテレータを作成 + const asyncIterable = createSessionAsyncIterable(session); + + // セッション開始イベントをキューに追加 + addEventToQueue(session, { + event: { + sessionStart: { + inferenceConfiguration: DEFAULT_INFERENCE_CONFIG + } + } + }); + + // プロンプト開始イベントをキューに追加 + addEventToQueue(session, { + event: { + promptStart: { + promptName: session.promptName, + textOutputConfiguration: { + mediaType: "text/plain", + }, + audioOutputConfiguration: DEFAULT_AUDIO_OUTPUT_CONFIG + } + } + }); + session.isPromptStartSent = true; + + // システムプロンプトを設定 + const textPromptID = randomUUID(); + addEventToQueue(session, { + event: { + contentStart: { + promptName: session.promptName, + contentName: textPromptID, + type: "TEXT", + interactive: true, + role: "SYSTEM", + textInputConfiguration: DEFAULT_TEXT_CONFIG, + }, + } + }); + + addEventToQueue(session, { + event: { + textInput: { + promptName: session.promptName, + contentName: textPromptID, + content: DEFAULT_SYSTEM_PROMPT, + }, + } + }); + + addEventToQueue(session, { + event: { + contentEnd: { + promptName: session.promptName, + contentName: textPromptID, + }, + } + }); + + // 音声入力開始イベントをキューに追加 + addEventToQueue(session, { + event: { + contentStart: { + promptName: session.promptName, + contentName: session.audioContentId, + type: "AUDIO", + interactive: true, + role: "USER", + audioInputConfiguration: DEFAULT_AUDIO_INPUT_CONFIG, + }, + } + }); + session.isAudioContentStartSent = true; + + // Bedrock Nova Sonicとの双方向ストリームを開始 + console.log(`Invoking Bedrock model for session ${session.sessionId}`); + const response = await session.bedrockClient.send( + new InvokeModelWithBidirectionalStreamCommand({ + modelId: "amazon.nova-sonic-v1:0", + body: asyncIterable, + }) + ); + + console.log(`Stream established for session ${session.sessionId}, processing responses...`); + + // レスポンスの処理 + session.responseProcessor = processResponseStream(session, response); + } catch (error) { + console.error(`Error starting Nova stream for session ${session.sessionId}:`, error); + await closeSession(session); + throw error; + } +} + +// Nova Sonicからのレスポンスを処理する関数 +async function processResponseStream(session: any, response: any) { + try { + console.log(`Starting to process response stream for session ${session.sessionId}`); + for await (const event of response.body) { + if (!session.isActive) { + console.log(`Session ${session.sessionId} is no longer active, stopping response processing`); + break; + } + + if (event.chunk?.bytes) { + try { + const textResponse = new TextDecoder().decode(event.chunk.bytes); + console.log(`Received response from Nova Sonic for session ${session.sessionId}: ${textResponse.substring(0, 100)}...`); + + try { + const jsonResponse = JSON.parse(textResponse); + + // 音声出力イベントの処理 + if (jsonResponse.event?.audioOutput) { + console.log(`Received audio output for session ${session.sessionId}, length: ${jsonResponse.event.audioOutput.content.length}`); + + // AppSync Eventsを通じてクライアントに音声データを送信 + try { + await session.appSyncChannel.publish({ + type: 'BedrockToAppSync', + data: { + sessionId: session.sessionId, + audioData: jsonResponse.event.audioOutput.content, + timestamp: Date.now() + } + }); + console.log(`Sent audio data to client for session ${session.sessionId}`); + } catch (error) { + console.error(`Error sending audio data to client for session ${session.sessionId}:`, error); + } + } + + // テキスト出力イベントの処理 + if (jsonResponse.event?.textOutput) { + console.log(`Received text output for session ${session.sessionId}: ${jsonResponse.event.textOutput.content}`); + } + + // その他のイベントの処理 + if (jsonResponse.event?.contentStart) { + console.log(`Received contentStart event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentStart)}`); + } else if (jsonResponse.event?.contentEnd) { + console.log(`Received contentEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentEnd)}`); + } else if (jsonResponse.event?.promptEnd) { + console.log(`Received promptEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.promptEnd)}`); + } else if (jsonResponse.event?.sessionEnd) { + console.log(`Received sessionEnd event for session ${session.sessionId}`); + // セッションが終了した場合は、セッションを閉じる + await closeSession(session); + } else { + // その他のイベント + const eventKeys = Object.keys(jsonResponse.event || {}); + if (eventKeys.length > 0) { + console.log(`Received other event for session ${session.sessionId}: ${eventKeys.join(', ')}`); + } + } + } catch (e) { + console.error(`Error parsing response for session ${session.sessionId}:`, e); + } + } catch (e) { + console.error(`Error processing response chunk for session ${session.sessionId}:`, e); + } + } else if (event.modelStreamErrorException) { + console.error(`Model stream error for session ${session.sessionId}:`, event.modelStreamErrorException); + } else if (event.internalServerException) { + console.error(`Internal server error for session ${session.sessionId}:`, event.internalServerException); + } + } + + console.log(`Response stream processing complete for session ${session.sessionId}`); + } catch (error) { + console.error(`Error processing response stream for session ${session.sessionId}:`, error); + } +} + +// 音声データをNova Sonicに送信する関数 +async function streamAudioToNova(session: any, base64AudioData: string) { + if (!session.isActive || !session.isAudioContentStartSent) { + console.log(`Cannot stream audio: Session ${session.sessionId} not active or audio content not started`); + return; + } + + console.log(`Streaming audio chunk to Nova Sonic for session ${session.sessionId}`); + + // 音声データイベントをキューに追加 + addEventToQueue(session, { + event: { + audioInput: { + promptName: session.promptName, + contentName: session.audioContentId, + content: base64AudioData, + }, + } + }); +} + +// 音声入力を終了する関数 +async function endAudioContent(session: any) { + if (!session.isActive || !session.isAudioContentStartSent) { + console.log(`Cannot end audio content: Session ${session.sessionId} not active or audio content not started`); + return; + } + + console.log(`Ending audio content for session ${session.sessionId}`); + + try { + // 音声コンテンツ終了イベントをキューに追加 + addEventToQueue(session, { + event: { + contentEnd: { + promptName: session.promptName, + contentName: session.audioContentId, + } + } + }); + + // 少し待機して、音声コンテンツ終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + + // プロンプト終了イベントをキューに追加 + addEventToQueue(session, { + event: { + promptEnd: { + promptName: session.promptName + } + } + }); + + // 少し待機して、プロンプト終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + } catch (error) { + console.error(`Error ending audio content for session ${session.sessionId}:`, error); + } +} + +// セッションを閉じる関数 +async function closeSession(session: any) { + if (!session || !session.isActive) { + if (session) { + console.log(`Session ${session.sessionId} already closed`); + } + return; + } + + console.log(`Closing session ${session.sessionId}`); + + try { + // 音声コンテンツが開始されていれば終了 + if (session.isAudioContentStartSent) { + await endAudioContent(session); + } + + // セッション終了イベントをキューに追加 + addEventToQueue(session, { + event: { + sessionEnd: {} + } + }); + + // 少し待機して、セッション終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + + // セッションを非アクティブにする + session.isActive = false; + + // アクティブセッションから削除 + activeSessions.delete(session.sessionId); + + console.log(`Session ${session.sessionId} closed`); + } catch (error) { + console.error(`Error closing session ${session.sessionId}:`, error); + + // エラーが発生しても確実にセッションを閉じる + session.isActive = false; + activeSessions.delete(session.sessionId); + } +} + +// イベントをセッションのキューに追加する関数 +function addEventToQueue(session: any, event: any) { + if (!session || !session.isActive) return; + + session.queue.push(event); + console.log(`Added event to queue for session ${session.sessionId}: ${JSON.stringify(event).substring(0, 100)}...`); +} + +// セッション用の非同期イテレータを作成する関数 +function createSessionAsyncIterable(session: any): AsyncIterable { + return { + [Symbol.asyncIterator]: () => { + console.log(`Creating async iterator for session ${session.sessionId}`); + + return { + next: async (): Promise> => { + try { + // セッションがアクティブでなければ終了 + if (!session || !session.isActive) { + console.log(`Session ${session?.sessionId} is not active, iterator done`); + return { value: undefined, done: true }; + } + + // キューが空なら待機 + while (session.queue.length === 0 && session.isActive) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + + // セッションが非アクティブになっていれば終了 + if (!session.isActive) { + return { value: undefined, done: true }; + } + + // キューからイベントを取得 + const nextEvent = session.queue.shift(); + console.log(`Sending event to Nova Sonic: ${JSON.stringify(nextEvent).substring(0, 100)}...`); + + return { + value: { + chunk: { + bytes: new TextEncoder().encode(JSON.stringify(nextEvent)) + } + }, + done: false + }; + } catch (error) { + console.error(`Error in session ${session?.sessionId} iterator:`, error); + if (session) { + session.isActive = false; + } + return { value: undefined, done: true }; + } + } + }; + } + }; +} diff --git a/packages/cdk/lib/construct/events.ts b/packages/cdk/lib/construct/events.ts new file mode 100644 index 000000000..d98325757 --- /dev/null +++ b/packages/cdk/lib/construct/events.ts @@ -0,0 +1,61 @@ +import { Duration } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as appsync from 'aws-cdk-lib/aws-appsync'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; +import { Runtime } from 'aws-cdk-lib/aws-lambda'; +import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam'; + +export interface EventsProps { + userPool: cognito.UserPool; +} + +export class Events extends Construct { + constructor(scope: Construct, id: string, props: EventsProps) { + super(scope, id); + + const eventApi = new appsync.EventApi(this, 'EventApi', { + apiName: 'GenUEvents', // TODO: add stg + authorizationConfig: { + authProviders: [{ + authorizationType: appsync.AppSyncAuthorizationType.IAM, + },{ + authorizationType: appsync.AppSyncAuthorizationType.USER_POOL, + cognitoConfig: { + userPool: props.userPool, + }, + }], + connectionAuthModeTypes: [appsync.AppSyncAuthorizationType.IAM, appsync.AppSyncAuthorizationType.USER_POOL], + defaultPublishAuthModeTypes: [appsync.AppSyncAuthorizationType.IAM, appsync.AppSyncAuthorizationType.USER_POOL], + defaultSubscribeAuthModeTypes: [appsync.AppSyncAuthorizationType.IAM, appsync.AppSyncAuthorizationType.USER_POOL], + }, + }); + + // TODO: avoid hardcoded namespace + const namespace = new appsync.ChannelNamespace(this, 'ChannelName', { + api: eventApi, + channelNamespaceName: 'default', + }); + + const lambda = new NodejsFunction(this, 'NovaSonic', { + runtime: Runtime.NODEJS_LATEST, + entry: './lambda/nova-sonic-lambda.ts', + timeout: Duration.minutes(15), + environment: { + EVENT_API_ENDPOINT: `https://${eventApi.httpDns}`, + }, + bundling: { + nodeModules: ['@aws-sdk/client-bedrock-runtime'], + }, + }); + + eventApi.grantConnect(lambda); + namespace.grantPublishAndSubscribe(lambda); + + lambda.role?.addToPrincipalPolicy(new PolicyStatement({ + effect: Effect.ALLOW, + resources: ['*'], + actions: ['bedrock:*'], + })); + } +} diff --git a/packages/cdk/lib/construct/index.ts b/packages/cdk/lib/construct/index.ts index 1504de3f2..f0d3e34f0 100644 --- a/packages/cdk/lib/construct/index.ts +++ b/packages/cdk/lib/construct/index.ts @@ -8,3 +8,4 @@ export * from './common-web-acl'; export * from './agent'; export * from './rag-knowledge-base'; export * from './guardrail'; +export * from './events'; diff --git a/packages/cdk/lib/generative-ai-use-cases-stack.ts b/packages/cdk/lib/generative-ai-use-cases-stack.ts index bb286bd7b..ab4e964f8 100644 --- a/packages/cdk/lib/generative-ai-use-cases-stack.ts +++ b/packages/cdk/lib/generative-ai-use-cases-stack.ts @@ -15,6 +15,7 @@ import * as cognito from 'aws-cdk-lib/aws-cognito'; import { ICertificate } from 'aws-cdk-lib/aws-certificatemanager'; import { Agent } from 'generative-ai-use-cases'; import { UseCaseBuilder } from './construct/use-case-builder'; +import { Events } from './construct/events'; import { ProcessedStackInput } from './stack-input'; export interface GenerativeAiUseCasesStackProps extends StackProps { @@ -199,6 +200,11 @@ export class GenerativeAiUseCasesStack extends Stack { api: api.api, }); + // Events (for bidirectional communication) + new Events(this, 'Events', { + userPool: auth.userPool, + }); + // Cfn Outputs new CfnOutput(this, 'Region', { value: this.region, diff --git a/packages/cdk/package.json b/packages/cdk/package.json index f12caea5b..bc7b7a141 100644 --- a/packages/cdk/package.json +++ b/packages/cdk/package.json @@ -36,17 +36,21 @@ "@aws-sdk/client-sagemaker-runtime": "^3.755.0", "@aws-sdk/client-transcribe": "^3.755.0", "@aws-sdk/client-transcribe-streaming": "^3.755.0", + "@aws-sdk/credential-providers": "^3.755.0", "@aws-sdk/lib-dynamodb": "^3.755.0", "@aws-sdk/s3-request-presigner": "^3.755.0", "@aws-solutions-constructs/aws-cloudfront-s3": "^2.68.0", + "aws-amplify": "^6.14.2", "aws-cdk-lib": "^2.154.1", "aws-jwt-verify": "^4.0.0", "constructs": "^10.3.0", "deploy-time-build": "^0.3.17", "node-html-parser": "^6.1.13", "sanitize-html": "^2.13.0", + "@smithy/node-http-handler": "^4.0.4", "source-map-support": "^0.5.21", "upsert-slr": "^1.0.4", + "ws": "^8.18.0", "zod": "^3.24.1" } } diff --git a/packages/web/public/audio-processor.worklet.js b/packages/web/public/audio-processor.worklet.js new file mode 100644 index 000000000..1bb076d52 --- /dev/null +++ b/packages/web/public/audio-processor.worklet.js @@ -0,0 +1,124 @@ +// Audio sample buffer to minimize reallocations +class ExpandableBuffer { + constructor() { + // Start with one second's worth of buffered audio capacity + this.buffer = new Float32Array(24000); + this.readIndex = 0; + this.writeIndex = 0; + this.underflowedSamples = 0; + this.isInitialBuffering = true; + this.initialBufferLength = 24000; // One second + this.lastWriteTime = 0; + } + + logTimeElapsedSinceLastWrite() { + const now = Date.now(); + if (this.lastWriteTime !== 0) { + const elapsed = now - this.lastWriteTime; + console.log(`[AudioWorklet] Elapsed time since last audio buffer write: ${elapsed} ms`); + } + this.lastWriteTime = now; + } + + write(samples) { + this.logTimeElapsedSinceLastWrite(); + if (this.writeIndex + samples.length <= this.buffer.length) { + // Enough space to append the new samples + } + else { + // Not enough space ... + if (samples.length <= this.readIndex) { + // ... but we can shift samples to the beginning of the buffer + const subarray = this.buffer.subarray(this.readIndex, this.writeIndex); + console.log(`[AudioWorklet] Shifting the audio buffer of length ${subarray.length} by ${this.readIndex}`); + this.buffer.set(subarray); + } + else { + // ... and we need to grow the buffer capacity to make room for more audio + const newLength = (samples.length + this.writeIndex - this.readIndex) * 2; + const newBuffer = new Float32Array(newLength); + console.log(`[AudioWorklet] Expanding the audio buffer from ${this.buffer.length} to ${newLength}`); + newBuffer.set(this.buffer.subarray(this.readIndex, this.writeIndex)); + this.buffer = newBuffer; + } + this.writeIndex -= this.readIndex; + this.readIndex = 0; + } + this.buffer.set(samples, this.writeIndex); + this.writeIndex += samples.length; + if (this.writeIndex - this.readIndex >= this.initialBufferLength) { + // Filled the initial buffer length, so we can start playback with some cushion + this.isInitialBuffering = false; + console.log("[AudioWorklet] Initial audio buffer filled"); + } + } + + read(destination) { + let copyLength = 0; + if (!this.isInitialBuffering) { + // Only start to play audio after we've built up some initial cushion + copyLength = Math.min(destination.length, this.writeIndex - this.readIndex); + } + destination.set(this.buffer.subarray(this.readIndex, this.readIndex + copyLength)); + this.readIndex += copyLength; + if (copyLength > 0 && this.underflowedSamples > 0) { + console.log(`[AudioWorklet] Detected audio buffer underflow of ${this.underflowedSamples} samples`); + this.underflowedSamples = 0; + } + if (copyLength < destination.length) { + // Not enough samples (buffer underflow). Fill the rest with silence. + destination.fill(0, copyLength); + this.underflowedSamples += destination.length - copyLength; + } + if (copyLength === 0) { + // Ran out of audio, so refill the buffer to the initial length before playing more + this.isInitialBuffering = true; + } + } + + clearBuffer() { + this.readIndex = 0; + this.writeIndex = 0; + } +} + +class AudioPlayerProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.playbackBuffer = new ExpandableBuffer(); + console.log('[AudioWorklet] AudioPlayerProcessor initialized'); + + this.port.onmessage = (event) => { + console.log('[AudioWorklet] Player received message:', event.data); + + if (event.data.type === "audio") { + console.log('[AudioWorklet] Received audio data for playback, length:', event.data.audioData.length); + this.playbackBuffer.write(event.data.audioData); + } + else if (event.data.type === "initial-buffer-length") { + // Override the current playback initial buffer length + const newLength = event.data.bufferLength; + this.playbackBuffer.initialBufferLength = newLength; + console.log(`[AudioWorklet] Changed initial audio buffer length to: ${newLength}`) + } + else if (event.data.type === "barge-in") { + console.log('[AudioWorklet] Barge-in received, clearing buffer'); + this.playbackBuffer.clearBuffer(); + } + }; + + // 初期化完了を通知 + this.port.postMessage({ + type: 'init', + message: 'AudioPlayerProcessor initialized' + }); + } + + process(inputs, outputs, parameters) { + const output = outputs[0][0]; // Assume one output with one channel + this.playbackBuffer.read(output); + return true; // True to continue processing + } +} + +registerProcessor("audio-player-processor", AudioPlayerProcessor); diff --git a/packages/web/public/audio-recorder.worklet.js b/packages/web/public/audio-recorder.worklet.js new file mode 100644 index 000000000..815065a77 --- /dev/null +++ b/packages/web/public/audio-recorder.worklet.js @@ -0,0 +1,127 @@ +class AudioRecorderProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.isRecording = false; + this.bufferSize = 512; + this.recordingBuffer = new Float32Array(this.bufferSize); + this.bufferIndex = 0; + this.debugCounter = 0; + + console.log('[AudioWorklet] AudioRecorderProcessor initialized'); + + // メッセージハンドラを追加 + this.port.onmessage = (event) => { + console.log('[AudioWorklet] Message received:', event.data); + + if (event.data.command === 'start') { + console.log('[AudioWorklet] Recording started'); + this.isRecording = true; + } else if (event.data.command === 'stop') { + console.log('[AudioWorklet] Recording stopped'); + this.isRecording = false; + } else if (event.data.command === 'test') { + console.log('[AudioWorklet] Test message received'); + // テストメッセージに応答 + this.port.postMessage({ + eventType: 'test', + message: 'Test response from AudioWorklet' + }); + + // テスト用にダミーデータを送信 + this.sendTestAudioData(); + } + }; + + // 初期化完了を通知 + this.port.postMessage({ + eventType: 'init', + message: 'AudioRecorderProcessor initialized' + }); + } + + // テスト用にダミーの音声データを送信 + sendTestAudioData() { + console.log('[AudioWorklet] Sending test audio data'); + const testBuffer = new Float32Array(this.bufferSize); + // サイン波を生成 + for (let i = 0; i < this.bufferSize; i++) { + testBuffer[i] = Math.sin(i * 0.01) * 0.5; + } + + this.port.postMessage({ + eventType: 'audioData', + audioData: testBuffer + }); + } + + process(inputs, outputs, parameters) { + // パラメータから録音状態を更新 + const isRecordingParam = parameters.isRecording; + if (isRecordingParam && isRecordingParam.length > 0) { + // パラメータ値が0より大きければ録音中 + const newIsRecording = isRecordingParam[0] > 0; + if (this.isRecording !== newIsRecording) { + console.log(`[AudioWorklet] Recording state changed from ${this.isRecording} to ${newIsRecording}`); + this.isRecording = newIsRecording; + } + } + + // デバッグカウンター + this.debugCounter++; + if (this.debugCounter % 100 === 0) { + console.log('[AudioWorklet] Process called, isRecording:', this.isRecording); + console.log('[AudioWorklet] Input channels:', inputs[0]?.length); + + // 定期的にテストデータを送信（デバッグ用） + if (this.isRecording && this.debugCounter % 500 === 0) { + this.sendTestAudioData(); + } + } + + const input = inputs[0]; + if (!input || !input.length) { + return true; + } + + const inputChannel = input[0]; + + // 入力データの処理（録音中のみ） + if (this.isRecording) { + // 音声データがあるか確認 + const hasAudioData = inputChannel.some(sample => Math.abs(sample) > 0.01); + if (hasAudioData && this.debugCounter % 100 === 0) { + console.log('[AudioWorklet] Receiving audio data with signal'); + } + + for (let i = 0; i < inputChannel.length; i++) { + this.recordingBuffer[this.bufferIndex] = inputChannel[i]; + this.bufferIndex++; + + // バッファがいっぱいになったらメインスレッドに送信 + if (this.bufferIndex >= this.bufferSize) { + console.log('[AudioWorklet] Buffer full, sending data'); + this.port.postMessage({ + eventType: 'audioData', + audioData: this.recordingBuffer.slice(0) + }); + this.bufferIndex = 0; + } + } + } + + return true; + } + + // パラメータ定義 + static get parameterDescriptors() { + return [{ + name: 'isRecording', + defaultValue: 0, + minValue: 0, + maxValue: 1, + automationRate: 'k-rate' + }]; + } +} + +registerProcessor('audio-recorder-processor', AudioRecorderProcessor); diff --git a/packages/web/src/components/AuthWithSAML.tsx b/packages/web/src/components/AuthWithSAML.tsx index f580fc0e7..d7840813c 100644 --- a/packages/web/src/components/AuthWithSAML.tsx +++ b/packages/web/src/components/AuthWithSAML.tsx @@ -61,6 +61,14 @@ const AuthWithSAML: React.FC = (props) => { }, }, }, + API: { + Events: { + // TODO + endpoint: `https://jiinfzwujrblvdthn7svyonbt4.appsync-api.ap-northeast-1.amazonaws.com/event`, + region: process.env.AWS_DEFAULT_REGION!, + defaultAuthMode: 'userPool', + }, + }, }); return ( diff --git a/packages/web/src/components/AuthWithUserpool.tsx b/packages/web/src/components/AuthWithUserpool.tsx index 3d39ead25..528021292 100644 --- a/packages/web/src/components/AuthWithUserpool.tsx +++ b/packages/web/src/components/AuthWithUserpool.tsx @@ -21,6 +21,14 @@ const AuthWithUserpool: React.FC = (props) => { identityPoolId: import.meta.env.VITE_APP_IDENTITY_POOL_ID, }, }, + API: { + Events: { + // TODO + endpoint: `https://jiinfzwujrblvdthn7svyonbt4.appsync-api.ap-northeast-1.amazonaws.com/event`, + region: process.env.AWS_DEFAULT_REGION!, + defaultAuthMode: 'userPool', + }, + }, }); I18n.putVocabularies(translations); diff --git a/packages/web/src/hooks/useNovaSonic.ts b/packages/web/src/hooks/useNovaSonic.ts new file mode 100644 index 000000000..9f3b2417b --- /dev/null +++ b/packages/web/src/hooks/useNovaSonic.ts @@ -0,0 +1,552 @@ +import { useEffect, useRef, useCallback, useState } from 'react'; +import { events } from 'aws-amplify/data'; + +// AppSync Events のチャンネル名 +const CHANNEL_NAME = '/default/nova-sonic'; + +// バッファリング設定 +const MAX_QUEUE_SIZE = 200; // 最大キューサイズを小さくする +const MAX_CHUNKS_PER_BATCH = 5; // 一度に処理する最大チャンク数を減らす + +// 音声データの変換ユーティリティ +const convertFloat32ToInt16 = (float32Array: Float32Array): Int16Array => { + const int16Array = new Int16Array(float32Array.length); + for (let i = 0; i < float32Array.length; i++) { + // Float32を-32768から32767の範囲にスケーリング + const s = Math.max(-1, Math.min(1, float32Array[i])); + int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + return int16Array; +}; + +export const useNovaSonic = () => { + const [isConnected, setIsConnected] = useState(false); + const [isRecording, setIsRecording] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [error, setError] = useState(null); + + // refs + const channelRef = useRef(null); + // 録音用と再生用に別々のAudioContextを使用 + const recordingContextRef = useRef(null); // 16000Hz + const playbackContextRef = useRef(null); // 24000Hz + const audioWorkletNodeRef = useRef(null); + const recorderWorkletNodeRef = useRef(null); + const micStreamRef = useRef(null); + const sourceNodeRef = useRef(null); + const analyserNodeRef = useRef(null); + const sessionIdRef = useRef(crypto.randomUUID()); + const recordingParamRef = useRef(null); + + // バッファリング用のrefs + const audioBufferQueueRef = useRef([]); + const isProcessingAudioRef = useRef(false); + + // AudioWorkletのURLを生成する関数 + const getWorkletUrl = (filename: string): string => { + // 現在のURLを基準にパスを構築 + const baseUrl = window.location.origin; + return `${baseUrl}/${filename}`; + }; + + // 音声データをキューに追加する関数 + const queueAudioData = useCallback((audioData: Float32Array) => { + console.log('Queueing audio data, length:', audioData.length); + + // キューサイズをチェック + if (audioBufferQueueRef.current.length >= MAX_QUEUE_SIZE) { + // キューがいっぱいなら古いチャンクを破棄 + audioBufferQueueRef.current.shift(); + console.log('Audio queue full, dropping oldest chunk'); + } + + // キューに音声データを追加 + audioBufferQueueRef.current.push(audioData); + + // キュー処理を開始 + processAudioQueue(); + }, [isConnected]); + + // キューを処理する関数 + const processAudioQueue = useCallback(async () => { + console.log('Process audio queue', audioBufferQueueRef.current?.length || -1, isConnected, channelRef.current, isProcessingAudioRef.current); + // 既に処理中か、キューが空か、接続されていない場合は何もしない + if (isProcessingAudioRef.current || + audioBufferQueueRef.current.length === 0 || + !isConnected || + !channelRef.current) { + return; + } + + isProcessingAudioRef.current = true; + try { + // キューから一定数のチャンクを処理 + let processedChunks = 0; + let combinedLength = 0; + const chunksToProcess: Float32Array[] = []; + + // 処理するチャンクを集める + while (audioBufferQueueRef.current.length > 0 && processedChunks < MAX_CHUNKS_PER_BATCH) { + const chunk = audioBufferQueueRef.current.shift(); + if (chunk) { + chunksToProcess.push(chunk); + combinedLength += chunk.length; + processedChunks++; + } + } + + if (chunksToProcess.length > 0) { + // チャンクを結合 + const combinedBuffer = new Float32Array(combinedLength); + let offset = 0; + for (const chunk of chunksToProcess) { + combinedBuffer.set(chunk, offset); + offset += chunk.length; + } + + // Float32Array から Int16Array に変換 + const int16Data = convertFloat32ToInt16(combinedBuffer); + + // Int16Array を Base64 エンコード + const buffer = int16Data.buffer; + const bytes = new Uint8Array(buffer); + let binary = ''; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + const base64Data = btoa(binary); + + console.log(`Sending buffered audio data, chunks: ${chunksToProcess.length}, total samples: ${combinedLength}`); + + // AppSync Events経由でLambdaに送信 + try { + await channelRef.current.publish({ + type: 'ClientToAppSync', + data: { + sessionId: sessionIdRef.current, + audioData: base64Data, + timestamp: Date.now() + } + }); + + console.log('Buffered audio data sent successfully'); + } catch (publishError) { + console.error('Failed to publish audio data:', publishError); + // 送信に失敗した場合は、チャンクをキューに戻す + for (let i = chunksToProcess.length - 1; i >= 0; i--) { + audioBufferQueueRef.current.unshift(chunksToProcess[i]); + } + console.log(`Returned ${chunksToProcess.length} chunks to queue after publish failure`); + } + } + } catch (err) { + console.error('Failed to process audio data:', err); + } finally { + isProcessingAudioRef.current = false; + + // キューにまだデータがある場合は、次の処理をスケジュール + if (audioBufferQueueRef.current.length > 0 && isConnected) { + setTimeout(() => processAudioQueue(), 100); // 100msの遅延を入れる + } + } + }, [isConnected]); + + // AppSync Events に接続 + useEffect(() => { + console.log('Initializing Nova Sonic hook'); + + // バッファをクリア + audioBufferQueueRef.current = []; + isProcessingAudioRef.current = false; + + const initializeAudio = async () => { + try { + // 録音用AudioContextの初期化（16000Hz） + recordingContextRef.current = new AudioContext({ + sampleRate: 16000, + latencyHint: 'interactive' + }); + + // 再生用AudioContextの初期化（24000Hz） + playbackContextRef.current = new AudioContext({ + sampleRate: 24000, + latencyHint: 'interactive' + }); + + console.log('Loading audio worklets...'); + + // Audio Workletの登録 - 完全なURLを使用 + const processorUrl = getWorkletUrl('audio-processor.worklet.js'); + const recorderUrl = getWorkletUrl('audio-recorder.worklet.js'); + + // 再生用Workletの登録 + console.log(`Loading processor worklet from: ${processorUrl} for playback context`); + await playbackContextRef.current.audioWorklet.addModule(processorUrl); + + // 録音用Workletの登録 + console.log(`Loading recorder worklet from: ${recorderUrl} for recording context`); + await recordingContextRef.current.audioWorklet.addModule(recorderUrl); + + console.log('Audio worklets loaded successfully'); + + // 再生用のWorkletNodeを作成（24000Hz用） + audioWorkletNodeRef.current = new AudioWorkletNode( + playbackContextRef.current, + 'audio-player-processor' + ); + + // 出力に接続 + audioWorkletNodeRef.current.connect(playbackContextRef.current.destination); + + console.log('Audio player initialized with separate contexts: recording=16000Hz, playback=24000Hz'); + } catch (err) { + console.error('Failed to initialize audio:', err); + setError(err instanceof Error ? err : new Error('Failed to initialize audio')); + } + }; + + const connectToAppSync = async () => { + try { + console.log('Connecting to AppSync Events...'); + const channel = await events.connect(CHANNEL_NAME); + channelRef.current = channel; + setIsConnected(true); + console.log('Connected to AppSync Events'); + + // バッファをクリア + audioBufferQueueRef.current = []; + isProcessingAudioRef.current = false; + + // AppSync Eventsからのメッセージ受信 + channel.subscribe({ + next: (data: any) => { + if (data.event?.type === 'BedrockToAppSync' && data.event?.data) { + console.log('Received audio data from Bedrock'); + + // 受信した音声データを再生 + if (data.event.data.audioData && audioWorkletNodeRef.current) { + try { + // Base64エンコードされた音声データをデコード + const binaryData = atob(data.event.data.audioData); + const bytes = new Uint8Array(binaryData.length); + for (let i = 0; i < binaryData.length; i++) { + bytes[i] = binaryData.charCodeAt(i); + } + + // PCM音声データをFloat32Arrayに変換 + const int16Data = new Int16Array(bytes.buffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + // 再生用AudioContextの状態を確認して再開 + if (playbackContextRef.current && playbackContextRef.current.state === 'suspended') { + playbackContextRef.current.resume(); + console.log('Playback AudioContext resumed, state:', playbackContextRef.current.state); + } + + // 音声データをWorkletに送信 + audioWorkletNodeRef.current.port.postMessage({ + type: 'audio', + audioData: float32Data + }); + + setIsPlaying(true); + + // 再生状態を一定時間後に更新 + setTimeout(() => { + setIsPlaying(false); + }, 500); + } catch (err) { + console.error('Error processing received audio data:', err); + } + } + } + }, + error: (err: any) => { + console.error('Error in AppSync Events subscription:', err); + setError(err instanceof Error ? err : new Error('AppSync Events subscription error')); + setIsConnected(false); + }, + }); + } catch (err) { + console.error('Failed to connect to AppSync Events:', err); + setError(err instanceof Error ? err : new Error('Failed to connect to AppSync Events')); + setIsConnected(false); + } + }; + + // 初期化処理を実行 + initializeAudio().then(connectToAppSync); + + // クリーンアップ関数 + return () => { + console.log('Cleaning up Nova Sonic hook'); + + // バッファをクリア + if (audioBufferQueueRef.current.length > 0) { + console.log(`Clearing audio buffer queue with ${audioBufferQueueRef.current.length} items`); + audioBufferQueueRef.current = []; + } + + // マイクストリームの停止 + if (micStreamRef.current) { + micStreamRef.current.getTracks().forEach(track => track.stop()); + micStreamRef.current = null; + } + + // AudioWorkletNodeの切断 + if (audioWorkletNodeRef.current) { + audioWorkletNodeRef.current.disconnect(); + audioWorkletNodeRef.current = null; + } + + if (recorderWorkletNodeRef.current) { + recorderWorkletNodeRef.current.disconnect(); + recorderWorkletNodeRef.current = null; + } + + if (analyserNodeRef.current) { + analyserNodeRef.current.disconnect(); + analyserNodeRef.current = null; + } + + // AudioContextの閉じる + if (recordingContextRef.current) { + recordingContextRef.current.close(); + recordingContextRef.current = null; + } + + if (playbackContextRef.current) { + playbackContextRef.current.close(); + playbackContextRef.current = null; + } + + // AppSync Eventsの切断 + if (channelRef.current) { + channelRef.current.close(); + channelRef.current = null; + setIsConnected(false); + console.log('Disconnected from AppSync Events'); + } + }; + }, []); + + // 録音を開始する関数 + const startRecording = useCallback(async () => { + if (isRecording) { + console.log('Already recording'); + return; + } + + // バッファをクリア + audioBufferQueueRef.current = []; + isProcessingAudioRef.current = false; + + try { + console.log('Starting recording...'); + + // AudioContext の状態を確認して再開 + if (recordingContextRef.current && recordingContextRef.current.state === 'suspended') { + await recordingContextRef.current.resume(); + console.log('Recording AudioContext resumed, state:', recordingContextRef.current.state); + } + + if (!recordingContextRef.current) { + console.error('Recording AudioContext is not initialized'); + return; + } + + // マイクへのアクセス許可を取得 + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + sampleRate: 16000 + } + }); + + console.log('Microphone access granted, tracks:', stream.getAudioTracks().length); + micStreamRef.current = stream; + + // マイク入力のソースノードを作成 + sourceNodeRef.current = recordingContextRef.current.createMediaStreamSource(stream); + console.log('MediaStreamSource created'); + + // 録音用のWorkletNodeを作成 + console.log('Creating AudioWorkletNode...'); + recorderWorkletNodeRef.current = new AudioWorkletNode( + recordingContextRef.current, + 'audio-recorder-processor' + ); + console.log('AudioWorkletNode created'); + + // 録音パラメータを取得 + recordingParamRef.current = recorderWorkletNodeRef.current.parameters.get('isRecording')!; + console.log('Got recording parameter:', recordingParamRef.current ? 'yes' : 'no'); + + // デバッグ用のAnalyserNodeを作成 + analyserNodeRef.current = recordingContextRef.current.createAnalyser(); + + // マイク入力を録音用Workletに接続 + sourceNodeRef.current.connect(recorderWorkletNodeRef.current); + // WorkletをAnalyserに接続（音声を出力しないようにする） + recorderWorkletNodeRef.current.connect(analyserNodeRef.current); + + console.log('Audio nodes connected'); + + // 録音開始コマンドを送信 + console.log('Sending start command to recorder worklet'); + recorderWorkletNodeRef.current.port.postMessage({ command: 'start' }); + + // パラメータで録音状態を設定 + if (recordingParamRef.current) { + recordingParamRef.current.setValueAtTime(1, recordingContextRef.current.currentTime); + console.log('Set recording parameter to 1'); + } + + // 録音データの受信ハンドラを設定 + console.log('Setting up message handler'); + recorderWorkletNodeRef.current.port.onmessage = (event) => { + if (event.data.eventType === 'audioData') { + console.log('Audio data received from worklet, length:', event.data.audioData.length); + + // 音声データをキューに追加（直接送信せず） + queueAudioData(event.data.audioData); + } else { + console.log('Other message from worklet:', event.data); + } + }; + + // テスト用のメッセージを送信 + setTimeout(() => { + if (recorderWorkletNodeRef.current) { + console.log('Sending test message to worklet'); + recorderWorkletNodeRef.current.port.postMessage({ command: 'test' }); + } + }, 1000); + + setIsRecording(true); + console.log('Recording started'); + + } catch (err) { + console.error('Failed to start recording:', err); + setError(err instanceof Error ? err : new Error('Failed to start recording')); + } + }, [isRecording, isConnected]); + + // 録音を停止する関数 + const stopRecording = useCallback(() => { + if (!isRecording) { + console.log('Not recording'); + return; + } + + console.log('Stopping recording...'); + + // 残りのバッファを処理 + if (audioBufferQueueRef.current.length > 0) { + console.log(`Processing remaining ${audioBufferQueueRef.current.length} audio chunks before stopping`); + processAudioQueue(); + } + + // パラメータで録音状態を設定 + if (recordingParamRef.current && recordingContextRef.current) { + recordingParamRef.current.setValueAtTime(0, recordingContextRef.current.currentTime); + console.log('Set recording parameter to 0'); + } + + // 録音停止コマンドを送信 + if (recorderWorkletNodeRef.current) { + recorderWorkletNodeRef.current.port.postMessage({ command: 'stop' }); + + // 残りのバッファを処理 + if (audioBufferQueueRef.current.length > 0) { + console.log(`Processing remaining ${audioBufferQueueRef.current.length} audio chunks before stopping`); + processAudioQueue(); + } + + recorderWorkletNodeRef.current.disconnect(); + recorderWorkletNodeRef.current = null; + } + + // マイクストリームの停止 + if (micStreamRef.current) { + micStreamRef.current.getTracks().forEach(track => track.stop()); + micStreamRef.current = null; + } + + // ソースノードの切断 + if (sourceNodeRef.current) { + sourceNodeRef.current.disconnect(); + sourceNodeRef.current = null; + } + + // アナライザーノードの切断 + if (analyserNodeRef.current) { + analyserNodeRef.current.disconnect(); + analyserNodeRef.current = null; + } + + setIsRecording(false); + console.log('Recording stopped'); + + // 録音停止を通知 + if (channelRef.current && isConnected) { + events.post(CHANNEL_NAME, { + type: 'ClientToAppSync', + data: { + sessionId: sessionIdRef.current, + action: 'stopRecording', + timestamp: Date.now() + } + }).then(() => { + console.log('Stop recording notification sent successfully'); + }).catch(err => { + console.error('Failed to send stop recording notification:', err); + }); + } + }, [isRecording, isConnected]); + + // 新しいセッションを開始する関数 + const startNewSession = useCallback(() => { + if (!channelRef.current || !isConnected) { + console.error('Cannot start new session: Not connected to AppSync Events'); + return; + } + + // バッファをクリア + audioBufferQueueRef.current = []; + isProcessingAudioRef.current = false; + + sessionIdRef.current = crypto.randomUUID(); + console.log(`Started new session with ID: ${sessionIdRef.current}`); + + // 新しいセッション開始を通知 + channelRef.current.publish({ + type: 'ClientToAppSync', + data: { + sessionId: sessionIdRef.current, + action: 'startSession', + timestamp: Date.now() + } + }).then(() => { + console.log('Start session notification sent successfully'); + }).catch((err: any) => { + console.error('Failed to send start session notification:', err); + }); + }, [isConnected]); + + return { + isConnected, + isRecording, + isPlaying, + error, + startRecording, + stopRecording, + startNewSession, + sessionId: sessionIdRef.current + }; +}; diff --git a/packages/web/src/main.tsx b/packages/web/src/main.tsx index 0bdcad680..a9a03472b 100644 --- a/packages/web/src/main.tsx +++ b/packages/web/src/main.tsx @@ -27,6 +27,7 @@ import OptimizePromptPage from './pages/OptimizePromptPage'; import TranscribePage from './pages/TranscribePage'; import AgentChatPage from './pages/AgentChatPage.tsx'; import FlowChatPage from './pages/FlowChatPage'; +import SpeechToSpeechPage from './pages/SpeechToSpeechPage'; import { MODELS } from './hooks/useModel'; import { Authenticator } from '@aws-amplify/ui-react'; import UseCaseBuilderEditPage from './pages/useCaseBuilder/UseCaseBuilderEditPage.tsx'; @@ -166,7 +167,11 @@ const routes: RouteObject[] = [ path: '/agent/:agentName', element: , } - : null, + : null, + { + path: '/speech-to-speech', + element: , + }, { path: '*', element: , diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx new file mode 100644 index 000000000..dbba96dae --- /dev/null +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { useNovaSonic } from '../hooks/useNovaSonic'; + +const SpeechToSpeech: React.FC = () => { + const { + isConnected, + startNewSession, + startRecording, + stopRecording, + } = useNovaSonic(); + + return ( +
+

Speech To Speech

+
+ isConnected: {isConnected.toString()} +
+
+ +
+
+ +
+
+ +
+
+ ); +}; + +export default SpeechToSpeech; From fee81a471fa1c6c851092dee032875cf232c9460 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Mon, 14 Apr 2025 01:27:47 +0900 Subject: [PATCH 02/20] fix --- packages/cdk/lambda/nova-sonic-lambda.ts | 704 ++++++------------ packages/cdk/lambda/nova-sonic-lambdaold | 538 +++++++++++++ packages/cdk/lib/construct/events.ts | 2 + .../web/public/audio-processor.worklet.js | 12 +- .../web/src/hooks/useNovaSonic/AudioPlayer.js | 128 ++++ .../AudioPlayerProcessor.worklet.js | 114 +++ .../web/src/hooks/useNovaSonic/ObjectsExt.js | 17 + packages/web/src/hooks/useNovaSonic/index.ts | 226 ++++++ .../{useNovaSonic.ts => useNovaSonicOld} | 113 ++- packages/web/src/pages/SpeechToSpeechPage.tsx | 12 +- packages/web/tsconfig.json | 1 + 11 files changed, 1349 insertions(+), 518 deletions(-) create mode 100644 packages/cdk/lambda/nova-sonic-lambdaold create mode 100644 packages/web/src/hooks/useNovaSonic/AudioPlayer.js create mode 100644 packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js create mode 100644 packages/web/src/hooks/useNovaSonic/ObjectsExt.js create mode 100644 packages/web/src/hooks/useNovaSonic/index.ts rename packages/web/src/hooks/{useNovaSonic.ts => useNovaSonicOld} (89%) diff --git a/packages/cdk/lambda/nova-sonic-lambda.ts b/packages/cdk/lambda/nova-sonic-lambda.ts index 5ad4e401a..1d65fc033 100644 --- a/packages/cdk/lambda/nova-sonic-lambda.ts +++ b/packages/cdk/lambda/nova-sonic-lambda.ts @@ -1,57 +1,231 @@ import { Amplify } from 'aws-amplify'; import { events } from 'aws-amplify/data'; import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import { randomUUID } from "crypto"; import { BedrockRuntimeClient, InvokeModelWithBidirectionalStreamCommand, - InvokeModelWithBidirectionalStreamInput, + // InvokeModelWithBidirectionalStreamInput, } from "@aws-sdk/client-bedrock-runtime"; -import { NodeHttp2Handler } from "@smithy/node-http-handler"; -import { randomUUID } from "crypto"; +import { NodeHttp2Handler } from '@smithy/node-http-handler'; -// WebSocketをグローバルに設定（Lambda環境用） Object.assign(global, { WebSocket: require('ws') }); -// AppSync Events のチャンネル名 -const CHANNEL_NAME = '/default/nova-sonic'; +// Event queue +const queue: Array = []; + +// Array of base64 input data +const audioInputQueue: string[] = []; -// Nova Sonic の設定 -const DEFAULT_INFERENCE_CONFIG = { - maxTokens: 1024, - topP: 0.9, - temperature: 0.7, +const promptName = randomUUID(); +let audioContentId = randomUUID(); +let audioStarted = false; + +const enqueueSessionStart = () => { + queue.push({ + event: { + sessionStart: { + inferenceConfiguration: { + maxTokens: 1024, + topP: 0.9, + temperature: 0.7, + } + } + } + }) }; -const DEFAULT_AUDIO_INPUT_CONFIG = { - audioType: "SPEECH", - encoding: "base64", - mediaType: "audio/lpcm", - sampleRateHertz: 16000, - sampleSizeBits: 16, - channelCount: 1, +const enqueuePromptStart = () => { + queue.push({ + event: { + promptStart: { + promptName, + textOutputConfiguration: { + mediaType: "text/plain", + }, + audioOutputConfiguration: { + audioType: "SPEECH", + encoding: "base64", + mediaType: "audio/lpcm", + sampleRateHertz: 24000, + sampleSizeBits: 16, + channelCount: 1, + voiceId: "tiffany", + } + } + } + }); }; -const DEFAULT_AUDIO_OUTPUT_CONFIG = { - ...DEFAULT_AUDIO_INPUT_CONFIG, - sampleRateHertz: 24000, - voiceId: "tiffany", +const enqueueSystemPrompt = () => { + const contentName = randomUUID(); + + queue.push({ + event: { + contentStart: { + promptName, + contentName, + type: "TEXT", + interactive: true, + role: "SYSTEM", + textInputConfiguration: { + mediaType: "text/plain", + }, + }, + }, + }); + + queue.push({ + event: { + textInput: { + promptName, + contentName, + content: 'You are the AI assistant', + }, + } + }); + + queue.push({ + event: { + contentEnd: { + promptName, + contentName, + }, + } + }) +}; + +const enqueueAudioStart = () => { + audioContentId = randomUUID(); + + queue.push({ + event: { + contentStart: { + promptName, + contentName: audioContentId, + type: 'AUDIO', + interactive: true, + role: 'USER', + audioInputConfiguration: { + audioType: "SPEECH", + encoding: "base64", + mediaType: "audio/lpcm", + sampleRateHertz: 16000, + sampleSizeBits: 16, + channelCount: 1, + }, + }, + }, + }); + + audioStarted = true; }; -const DEFAULT_TEXT_CONFIG = { - mediaType: "text/plain" +const enqueueAudioStop = () => { + queue.push({ + event: { + contentEnd: { + promptName, + contentName: audioContentId, + }, + }, + }); + + audioStarted = false; }; -const DEFAULT_SYSTEM_PROMPT = "あなたは親切なAIアシスタントです。ユーザーの質問に簡潔に答えてください。"; +const enqueueAudioInput = (audioInput: string) => { + audioInputQueue.push(audioInput); +}; -// セッション管理用のマップ -const activeSessions = new Map(); +const createAsyncIterator = () => { + return { + [Symbol.asyncIterator]: () => { + return { + next: async () => { + while (queue.length === 0) { + // TODO: close signal + await new Promise(s => setTimeout(s, 100)); + } + + const nextEvent = queue.shift(); + console.log(`Consume event ${JSON.stringify(nextEvent)}`); + return { + value: { + chunk: { + bytes: new TextEncoder().encode(JSON.stringify(nextEvent)), + }, + }, + done: false, + }; + }, + }; + }, + return: async () => { + return { value: undefined, done: true} + }, + throw: async (error: any) => { + console.error(error) + throw error; + }, + } +} + +const processAudioQueue = async () => { + while (audioInputQueue.length > 0 && audioStarted) { + const audioChunk = audioInputQueue.shift(); + + queue.push({ + event: { + audioInput: { + promptName, + contentName: audioContentId, + content: audioChunk, + }, + }, + }); + } + + setTimeout(() => processAudioQueue(), 0); +}; + +const processResponseStream = async (channel: any, response: any) => { + try { + for await (const event of response.body) { + const textResponse = new TextDecoder().decode(event.chunk.bytes); + + if (event.chunk?.bytes) { + const jsonResponse = JSON.parse(textResponse); + console.log('JSON Response', jsonResponse); + + if (jsonResponse.event?.audioOutput) { + await channel.publish({ + direction: 'btoc', + event: 'audioOutput', + data: jsonResponse.event.audioOutput, + }); + } + } + } + } catch (e) { + console.error(e); + } +}; export const handler = async (event: any) => { try { - console.log('Lambda function started'); - console.log('Event:', JSON.stringify(event, null, 2)); + console.log('event', event); + + const bedrock = new BedrockRuntimeClient({ + region: 'us-east-1', // TODO + requestHandler: new NodeHttp2Handler({ + requestTimeout: 300000, + sessionTimeout: 300000, + disableConcurrentStreams: false, + maxConcurrentStreams: 1, + }), + }); - // AppSync Events の設定 Amplify.configure( { API: { @@ -78,461 +252,43 @@ export const handler = async (event: any) => { } ); - // Bedrock クライアントの初期化 - const bedrockClient = new BedrockRuntimeClient({ - region: 'us-east-1', // TODO - requestHandler: new NodeHttp2Handler({ - requestTimeout: 300000, - sessionTimeout: 300000, - }) - }); - - // AppSync Events に接続 - console.log(`Connecting to AppSync Events channel: ${CHANNEL_NAME}`); - const appSyncChannel = await events.connect(CHANNEL_NAME); - console.log('Connected to AppSync Events!'); + const channel = await events.connect('/default/dummy-session'); // TODO - // AppSync Events からのメッセージ受信 - appSyncChannel.subscribe({ + channel.subscribe({ next: async (data: any) => { - if (data.event?.type === 'ClientToAppSync') { - console.log('Received data from client:', data.event.data); - - const clientData = data.event.data; - const sessionId = clientData.sessionId; - - if (!sessionId) { - console.error('No sessionId provided in client data'); - return; - } - - // セッション開始アクション - if (clientData.action === 'startSession') { - console.log(`Starting new session: ${sessionId}`); - await startNovaSession(sessionId, bedrockClient, appSyncChannel); - return; - } - - // 録音停止アクション - if (clientData.action === 'stopRecording') { - console.log(`Stopping recording for session: ${sessionId}`); - const session = activeSessions.get(sessionId); - if (session) { - await endAudioContent(session); - } - return; - } - - // 音声データの処理 - if (clientData.audioData) { - let session = activeSessions.get(sessionId); - - // セッションがなければ新規作成 - if (!session) { - console.log(`Creating new session for: ${sessionId}`); - session = await startNovaSession(sessionId, bedrockClient, appSyncChannel); - } - - // 音声データをNova Sonicに送信 - await streamAudioToNova(session, clientData.audioData); + const event = data?.event; + if (event && event.direction === 'ctob') { + if (event.event === 'audioStart') { + enqueueAudioStart(); + } else if (event.event === 'audioStop') { + enqueueAudioStop(); + } else if (event.event === 'audioInput') { + enqueueAudioInput(event.data); } } }, - error: (e: any) => { - console.error('Error in AppSync Events subscription:', e); - }, + error: console.error, }); - // 実際は Bedrock のデータ受信待ちで止まるが、ここでは仮の処理として sleep で止める - await new Promise(s => setTimeout(s, 15 * 60 * 1000)); // 15分待機 - - // クリーンアップ - console.log('Cleaning up sessions before exit'); - for (const [sessionId, session] of activeSessions.entries()) { - try { - await closeSession(session); - console.log(`Closed session: ${sessionId}`); - } catch (err) { - console.error(`Error closing session ${sessionId}:`, err); - } - } - - return { statusCode: 200, body: 'Lambda execution completed' }; - } catch (error) { - console.error('Error in Lambda handler:', error); - return { statusCode: 500, body: 'Error in Lambda handler' }; - } -}; + enqueueSessionStart(); + enqueuePromptStart(); + enqueueSystemPrompt(); -// Nova Sonicセッションを開始する関数 -async function startNovaSession(sessionId: string, bedrockClient: BedrockRuntimeClient, appSyncChannel: any) { - console.log(`Initializing Nova Sonic session: ${sessionId}`); - - // セッション情報を作成 - const session = { - sessionId, - bedrockClient, - appSyncChannel, - promptName: randomUUID(), - audioContentId: randomUUID(), - queue: [], - isActive: true, - isPromptStartSent: false, - isAudioContentStartSent: false, - responseProcessor: null as any - }; - - // セッションを保存 - activeSessions.set(sessionId, session); - - try { - // Nova Sonicとの双方向ストリームを開始 - await startNovaStream(session); - return session; - } catch (error) { - console.error(`Error in startNovaSession: ${error}`); - activeSessions.delete(sessionId); - throw error; - } -} + const asyncIterator = createAsyncIterator(); -// Nova Sonicとの双方向ストリームを開始する関数 -async function startNovaStream(session: any) { - try { - console.log(`Starting bidirectional stream for session: ${session.sessionId}`); - - // 非同期イテレータを作成 - const asyncIterable = createSessionAsyncIterable(session); - - // セッション開始イベントをキューに追加 - addEventToQueue(session, { - event: { - sessionStart: { - inferenceConfiguration: DEFAULT_INFERENCE_CONFIG - } - } - }); - - // プロンプト開始イベントをキューに追加 - addEventToQueue(session, { - event: { - promptStart: { - promptName: session.promptName, - textOutputConfiguration: { - mediaType: "text/plain", - }, - audioOutputConfiguration: DEFAULT_AUDIO_OUTPUT_CONFIG - } - } - }); - session.isPromptStartSent = true; - - // システムプロンプトを設定 - const textPromptID = randomUUID(); - addEventToQueue(session, { - event: { - contentStart: { - promptName: session.promptName, - contentName: textPromptID, - type: "TEXT", - interactive: true, - role: "SYSTEM", - textInputConfiguration: DEFAULT_TEXT_CONFIG, - }, - } - }); - - addEventToQueue(session, { - event: { - textInput: { - promptName: session.promptName, - contentName: textPromptID, - content: DEFAULT_SYSTEM_PROMPT, - }, - } - }); - - addEventToQueue(session, { - event: { - contentEnd: { - promptName: session.promptName, - contentName: textPromptID, - }, - } - }); - - // 音声入力開始イベントをキューに追加 - addEventToQueue(session, { - event: { - contentStart: { - promptName: session.promptName, - contentName: session.audioContentId, - type: "AUDIO", - interactive: true, - role: "USER", - audioInputConfiguration: DEFAULT_AUDIO_INPUT_CONFIG, - }, - } - }); - session.isAudioContentStartSent = true; - - // Bedrock Nova Sonicとの双方向ストリームを開始 - console.log(`Invoking Bedrock model for session ${session.sessionId}`); - const response = await session.bedrockClient.send( + const response = await bedrock.send( new InvokeModelWithBidirectionalStreamCommand({ - modelId: "amazon.nova-sonic-v1:0", - body: asyncIterable, - }) + modelId: 'amazon.nova-sonic-v1:0', + body: asyncIterator, + }), ); - console.log(`Stream established for session ${session.sessionId}, processing responses...`); - - // レスポンスの処理 - session.responseProcessor = processResponseStream(session, response); - } catch (error) { - console.error(`Error starting Nova stream for session ${session.sessionId}:`, error); - await closeSession(session); - throw error; - } -} - -// Nova Sonicからのレスポンスを処理する関数 -async function processResponseStream(session: any, response: any) { - try { - console.log(`Starting to process response stream for session ${session.sessionId}`); - for await (const event of response.body) { - if (!session.isActive) { - console.log(`Session ${session.sessionId} is no longer active, stopping response processing`); - break; - } - - if (event.chunk?.bytes) { - try { - const textResponse = new TextDecoder().decode(event.chunk.bytes); - console.log(`Received response from Nova Sonic for session ${session.sessionId}: ${textResponse.substring(0, 100)}...`); - - try { - const jsonResponse = JSON.parse(textResponse); - - // 音声出力イベントの処理 - if (jsonResponse.event?.audioOutput) { - console.log(`Received audio output for session ${session.sessionId}, length: ${jsonResponse.event.audioOutput.content.length}`); - - // AppSync Eventsを通じてクライアントに音声データを送信 - try { - await session.appSyncChannel.publish({ - type: 'BedrockToAppSync', - data: { - sessionId: session.sessionId, - audioData: jsonResponse.event.audioOutput.content, - timestamp: Date.now() - } - }); - console.log(`Sent audio data to client for session ${session.sessionId}`); - } catch (error) { - console.error(`Error sending audio data to client for session ${session.sessionId}:`, error); - } - } - - // テキスト出力イベントの処理 - if (jsonResponse.event?.textOutput) { - console.log(`Received text output for session ${session.sessionId}: ${jsonResponse.event.textOutput.content}`); - } - - // その他のイベントの処理 - if (jsonResponse.event?.contentStart) { - console.log(`Received contentStart event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentStart)}`); - } else if (jsonResponse.event?.contentEnd) { - console.log(`Received contentEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentEnd)}`); - } else if (jsonResponse.event?.promptEnd) { - console.log(`Received promptEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.promptEnd)}`); - } else if (jsonResponse.event?.sessionEnd) { - console.log(`Received sessionEnd event for session ${session.sessionId}`); - // セッションが終了した場合は、セッションを閉じる - await closeSession(session); - } else { - // その他のイベント - const eventKeys = Object.keys(jsonResponse.event || {}); - if (eventKeys.length > 0) { - console.log(`Received other event for session ${session.sessionId}: ${eventKeys.join(', ')}`); - } - } - } catch (e) { - console.error(`Error parsing response for session ${session.sessionId}:`, e); - } - } catch (e) { - console.error(`Error processing response chunk for session ${session.sessionId}:`, e); - } - } else if (event.modelStreamErrorException) { - console.error(`Model stream error for session ${session.sessionId}:`, event.modelStreamErrorException); - } else if (event.internalServerException) { - console.error(`Internal server error for session ${session.sessionId}:`, event.internalServerException); - } - } - - console.log(`Response stream processing complete for session ${session.sessionId}`); - } catch (error) { - console.error(`Error processing response stream for session ${session.sessionId}:`, error); - } -} - -// 音声データをNova Sonicに送信する関数 -async function streamAudioToNova(session: any, base64AudioData: string) { - if (!session.isActive || !session.isAudioContentStartSent) { - console.log(`Cannot stream audio: Session ${session.sessionId} not active or audio content not started`); - return; - } - - console.log(`Streaming audio chunk to Nova Sonic for session ${session.sessionId}`); - - // 音声データイベントをキューに追加 - addEventToQueue(session, { - event: { - audioInput: { - promptName: session.promptName, - contentName: session.audioContentId, - content: base64AudioData, - }, - } - }); -} - -// 音声入力を終了する関数 -async function endAudioContent(session: any) { - if (!session.isActive || !session.isAudioContentStartSent) { - console.log(`Cannot end audio content: Session ${session.sessionId} not active or audio content not started`); - return; - } - - console.log(`Ending audio content for session ${session.sessionId}`); - - try { - // 音声コンテンツ終了イベントをキューに追加 - addEventToQueue(session, { - event: { - contentEnd: { - promptName: session.promptName, - contentName: session.audioContentId, - } - } - }); - - // 少し待機して、音声コンテンツ終了イベントが処理されるのを待つ - await new Promise(resolve => setTimeout(resolve, 500)); - - // プロンプト終了イベントをキューに追加 - addEventToQueue(session, { - event: { - promptEnd: { - promptName: session.promptName - } - } - }); - - // 少し待機して、プロンプト終了イベントが処理されるのを待つ - await new Promise(resolve => setTimeout(resolve, 500)); - } catch (error) { - console.error(`Error ending audio content for session ${session.sessionId}:`, error); - } -} + // Start audio event loop + processAudioQueue(); -// セッションを閉じる関数 -async function closeSession(session: any) { - if (!session || !session.isActive) { - if (session) { - console.log(`Session ${session.sessionId} already closed`); - } - return; + // Start response stream + await processResponseStream(channel, response); + } catch (e) { + console.error(e); } - - console.log(`Closing session ${session.sessionId}`); - - try { - // 音声コンテンツが開始されていれば終了 - if (session.isAudioContentStartSent) { - await endAudioContent(session); - } - - // セッション終了イベントをキューに追加 - addEventToQueue(session, { - event: { - sessionEnd: {} - } - }); - - // 少し待機して、セッション終了イベントが処理されるのを待つ - await new Promise(resolve => setTimeout(resolve, 500)); - - // セッションを非アクティブにする - session.isActive = false; - - // アクティブセッションから削除 - activeSessions.delete(session.sessionId); - - console.log(`Session ${session.sessionId} closed`); - } catch (error) { - console.error(`Error closing session ${session.sessionId}:`, error); - - // エラーが発生しても確実にセッションを閉じる - session.isActive = false; - activeSessions.delete(session.sessionId); - } -} - -// イベントをセッションのキューに追加する関数 -function addEventToQueue(session: any, event: any) { - if (!session || !session.isActive) return; - - session.queue.push(event); - console.log(`Added event to queue for session ${session.sessionId}: ${JSON.stringify(event).substring(0, 100)}...`); -} - -// セッション用の非同期イテレータを作成する関数 -function createSessionAsyncIterable(session: any): AsyncIterable { - return { - [Symbol.asyncIterator]: () => { - console.log(`Creating async iterator for session ${session.sessionId}`); - - return { - next: async (): Promise> => { - try { - // セッションがアクティブでなければ終了 - if (!session || !session.isActive) { - console.log(`Session ${session?.sessionId} is not active, iterator done`); - return { value: undefined, done: true }; - } - - // キューが空なら待機 - while (session.queue.length === 0 && session.isActive) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - - // セッションが非アクティブになっていれば終了 - if (!session.isActive) { - return { value: undefined, done: true }; - } - - // キューからイベントを取得 - const nextEvent = session.queue.shift(); - console.log(`Sending event to Nova Sonic: ${JSON.stringify(nextEvent).substring(0, 100)}...`); - - return { - value: { - chunk: { - bytes: new TextEncoder().encode(JSON.stringify(nextEvent)) - } - }, - done: false - }; - } catch (error) { - console.error(`Error in session ${session?.sessionId} iterator:`, error); - if (session) { - session.isActive = false; - } - return { value: undefined, done: true }; - } - } - }; - } - }; -} +}; diff --git a/packages/cdk/lambda/nova-sonic-lambdaold b/packages/cdk/lambda/nova-sonic-lambdaold new file mode 100644 index 000000000..700f4091c --- /dev/null +++ b/packages/cdk/lambda/nova-sonic-lambdaold @@ -0,0 +1,538 @@ +import { Amplify } from 'aws-amplify'; +import { events } from 'aws-amplify/data'; +import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import { + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamCommand, + InvokeModelWithBidirectionalStreamInput, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttp2Handler } from "@smithy/node-http-handler"; +import { randomUUID } from "crypto"; + +// WebSocketをグローバルに設定（Lambda環境用） +Object.assign(global, { WebSocket: require('ws') }); + +// AppSync Events のチャンネル名 +const CHANNEL_NAME = '/default/nova-sonic'; + +// Nova Sonic の設定 +const DEFAULT_INFERENCE_CONFIG = { + maxTokens: 1024, + topP: 0.9, + temperature: 0.7, +}; + +const DEFAULT_AUDIO_INPUT_CONFIG = { + audioType: "SPEECH", + encoding: "base64", + mediaType: "audio/lpcm", + sampleRateHertz: 16000, + sampleSizeBits: 16, + channelCount: 1, +}; + +const DEFAULT_AUDIO_OUTPUT_CONFIG = { + ...DEFAULT_AUDIO_INPUT_CONFIG, + sampleRateHertz: 24000, + voiceId: "tiffany", +}; + +const DEFAULT_TEXT_CONFIG = { + mediaType: "text/plain" +}; + +const DEFAULT_SYSTEM_PROMPT = "あなたは親切なAIアシスタントです。ユーザーの質問に簡潔に答えてください。"; + +// セッション管理用のマップ +const activeSessions = new Map(); + +export const handler = async (event: any) => { + try { + console.log('Lambda function started'); + console.log('Event:', JSON.stringify(event, null, 2)); + + // AppSync Events の設定 + Amplify.configure( + { + API: { + Events: { + endpoint: `${process.env.EVENT_API_ENDPOINT!}/event`, + region: process.env.AWS_DEFAULT_REGION!, + defaultAuthMode: 'iam', + }, + }, + }, + { + Auth: { + credentialsProvider: { + getCredentialsAndIdentityId: async () => { + const provider = fromNodeProviderChain(); + const credentials = await provider(); + return { + credentials, + }; + }, + clearCredentialsAndIdentityId: async () => {}, + }, + }, + } + ); + + // Bedrock クライアントの初期化 + const bedrockClient = new BedrockRuntimeClient({ + region: 'us-east-1', // TODO + requestHandler: new NodeHttp2Handler({ + requestTimeout: 300000, + sessionTimeout: 300000, + }) + }); + + // AppSync Events に接続 + console.log(`Connecting to AppSync Events channel: ${CHANNEL_NAME}`); + const appSyncChannel = await events.connect(CHANNEL_NAME); + console.log('Connected to AppSync Events!'); + + // AppSync Events からのメッセージ受信 + appSyncChannel.subscribe({ + next: async (data: any) => { + if (data.event?.type === 'ClientToAppSync') { + console.log('Received data from client:', data.event.data); + + const clientData = data.event.data; + const sessionId = clientData.sessionId; + + if (!sessionId) { + console.error('No sessionId provided in client data'); + return; + } + + // セッション開始アクション + if (clientData.action === 'startSession') { + console.log(`Starting new session: ${sessionId}`); + await startNovaSession(sessionId, bedrockClient, appSyncChannel); + return; + } + + // 録音停止アクション + if (clientData.action === 'stopRecording') { + console.log(`Stopping recording for session: ${sessionId}`); + const session = activeSessions.get(sessionId); + if (session) { + await endAudioContent(session); + } + return; + } + + // 音声データの処理 + if (clientData.audioData) { + let session = activeSessions.get(sessionId); + + // セッションがなければ新規作成 + if (!session) { + console.log(`Creating new session for: ${sessionId}`); + session = await startNovaSession(sessionId, bedrockClient, appSyncChannel); + } + + // 音声データをNova Sonicに送信 + await streamAudioToNova(session, clientData.audioData); + } + } + }, + error: (e: any) => { + console.error('Error in AppSync Events subscription:', e); + }, + }); + + // 実際は Bedrock のデータ受信待ちで止まるが、ここでは仮の処理として sleep で止める + await new Promise(s => setTimeout(s, 15 * 60 * 1000)); // 15分待機 + + // クリーンアップ + console.log('Cleaning up sessions before exit'); + for (const [sessionId, session] of activeSessions.entries()) { + try { + await closeSession(session); + console.log(`Closed session: ${sessionId}`); + } catch (err) { + console.error(`Error closing session ${sessionId}:`, err); + } + } + + return { statusCode: 200, body: 'Lambda execution completed' }; + } catch (error) { + console.error('Error in Lambda handler:', error); + return { statusCode: 500, body: 'Error in Lambda handler' }; + } +}; + +// Nova Sonicセッションを開始する関数 +async function startNovaSession(sessionId: string, bedrockClient: BedrockRuntimeClient, appSyncChannel: any) { + console.log(`Initializing Nova Sonic session: ${sessionId}`); + + // セッション情報を作成 + const session = { + sessionId, + bedrockClient, + appSyncChannel, + promptName: randomUUID(), + audioContentId: randomUUID(), + queue: [], + isActive: true, + isPromptStartSent: false, + isAudioContentStartSent: false, + responseProcessor: null as any + }; + + // セッションを保存 + activeSessions.set(sessionId, session); + + try { + // Nova Sonicとの双方向ストリームを開始 + await startNovaStream(session); + return session; + } catch (error) { + console.error(`Error in startNovaSession: ${error}`); + activeSessions.delete(sessionId); + throw error; + } +} + +// Nova Sonicとの双方向ストリームを開始する関数 +async function startNovaStream(session: any) { + try { + console.log(`Starting bidirectional stream for session: ${session.sessionId}`); + + // 非同期イテレータを作成 + const asyncIterable = createSessionAsyncIterable(session); + + // セッション開始イベントをキューに追加 + addEventToQueue(session, { + event: { + sessionStart: { + inferenceConfiguration: DEFAULT_INFERENCE_CONFIG + } + } + }); + + // プロンプト開始イベントをキューに追加 + addEventToQueue(session, { + event: { + promptStart: { + promptName: session.promptName, + textOutputConfiguration: { + mediaType: "text/plain", + }, + audioOutputConfiguration: DEFAULT_AUDIO_OUTPUT_CONFIG + } + } + }); + session.isPromptStartSent = true; + + // システムプロンプトを設定 + const textPromptID = randomUUID(); + addEventToQueue(session, { + event: { + contentStart: { + promptName: session.promptName, + contentName: textPromptID, + type: "TEXT", + interactive: true, + role: "SYSTEM", + textInputConfiguration: DEFAULT_TEXT_CONFIG, + }, + } + }); + + addEventToQueue(session, { + event: { + textInput: { + promptName: session.promptName, + contentName: textPromptID, + content: DEFAULT_SYSTEM_PROMPT, + }, + } + }); + + addEventToQueue(session, { + event: { + contentEnd: { + promptName: session.promptName, + contentName: textPromptID, + }, + } + }); + + // 音声入力開始イベントをキューに追加 + addEventToQueue(session, { + event: { + contentStart: { + promptName: session.promptName, + contentName: session.audioContentId, + type: "AUDIO", + interactive: true, + role: "USER", + audioInputConfiguration: DEFAULT_AUDIO_INPUT_CONFIG, + }, + } + }); + session.isAudioContentStartSent = true; + + // Bedrock Nova Sonicとの双方向ストリームを開始 + console.log(`Invoking Bedrock model for session ${session.sessionId}`); + const response = await session.bedrockClient.send( + new InvokeModelWithBidirectionalStreamCommand({ + modelId: "amazon.nova-sonic-v1:0", + body: asyncIterable, + }) + ); + + console.log(`Stream established for session ${session.sessionId}, processing responses...`); + + // レスポンスの処理 + session.responseProcessor = processResponseStream(session, response); + } catch (error) { + console.error(`Error starting Nova stream for session ${session.sessionId}:`, error); + await closeSession(session); + throw error; + } +} + +// Nova Sonicからのレスポンスを処理する関数 +async function processResponseStream(session: any, response: any) { + try { + console.log(`Starting to process response stream for session ${session.sessionId}`); + for await (const event of response.body) { + if (!session.isActive) { + console.log(`Session ${session.sessionId} is no longer active, stopping response processing`); + break; + } + + if (event.chunk?.bytes) { + try { + const textResponse = new TextDecoder().decode(event.chunk.bytes); + console.log(`Received response from Nova Sonic for session ${session.sessionId}: ${textResponse.substring(0, 100)}...`); + + try { + const jsonResponse = JSON.parse(textResponse); + + // 音声出力イベントの処理 + if (jsonResponse.event?.audioOutput) { + console.log(`Received audio output for session ${session.sessionId} ${jsonResponse.event.audioOutput.content}`); + + // AppSync Eventsを通じてクライアントに音声データを送信 + try { + await session.appSyncChannel.publish({ + type: 'BedrockToAppSync', + data: { + sessionId: session.sessionId, + audioData: jsonResponse.event.audioOutput.content, + timestamp: Date.now() + } + }); + console.log(`Sent audio data to client for session ${session.sessionId}`); + } catch (error) { + console.error(`Error sending audio data to client for session ${session.sessionId}:`, error); + } + } + + // テキスト出力イベントの処理 + if (jsonResponse.event?.textOutput) { + console.log(`Received text output for session ${session.sessionId}: ${jsonResponse.event.textOutput.content}`); + } + + // その他のイベントの処理 + if (jsonResponse.event?.contentStart) { + console.log(`Received contentStart event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentStart)}`); + } else if (jsonResponse.event?.contentEnd) { + console.log(`Received contentEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.contentEnd)}`); + } else if (jsonResponse.event?.promptEnd) { + console.log(`Received promptEnd event for session ${session.sessionId}: ${JSON.stringify(jsonResponse.event.promptEnd)}`); + } else if (jsonResponse.event?.sessionEnd) { + console.log(`Received sessionEnd event for session ${session.sessionId}`); + // セッションが終了した場合は、セッションを閉じる + await closeSession(session); + } else { + // その他のイベント + const eventKeys = Object.keys(jsonResponse.event || {}); + if (eventKeys.length > 0) { + console.log(`Received other event for session ${session.sessionId}: ${eventKeys.join(', ')}`); + } + } + } catch (e) { + console.error(`Error parsing response for session ${session.sessionId}:`, e); + } + } catch (e) { + console.error(`Error processing response chunk for session ${session.sessionId}:`, e); + } + } else if (event.modelStreamErrorException) { + console.error(`Model stream error for session ${session.sessionId}:`, event.modelStreamErrorException); + } else if (event.internalServerException) { + console.error(`Internal server error for session ${session.sessionId}:`, event.internalServerException); + } + } + + console.log(`Response stream processing complete for session ${session.sessionId}`); + } catch (error) { + console.error(`Error processing response stream for session ${session.sessionId}:`, error); + } +} + +// 音声データをNova Sonicに送信する関数 +async function streamAudioToNova(session: any, base64AudioData: string) { + if (!session.isActive || !session.isAudioContentStartSent) { + console.log(`Cannot stream audio: Session ${session.sessionId} not active or audio content not started`); + return; + } + + console.log(`Streaming audio chunk to Nova Sonic for session ${session.sessionId}`); + + // 音声データイベントをキューに追加 + addEventToQueue(session, { + event: { + audioInput: { + promptName: session.promptName, + contentName: session.audioContentId, + content: base64AudioData, + }, + } + }); +} + +// 音声入力を終了する関数 +async function endAudioContent(session: any) { + if (!session.isActive || !session.isAudioContentStartSent) { + console.log(`Cannot end audio content: Session ${session.sessionId} not active or audio content not started`); + return; + } + + console.log(`Ending audio content for session ${session.sessionId}`); + + try { + // 音声コンテンツ終了イベントをキューに追加 + addEventToQueue(session, { + event: { + contentEnd: { + promptName: session.promptName, + contentName: session.audioContentId, + } + } + }); + + // 少し待機して、音声コンテンツ終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + + // プロンプト終了イベントをキューに追加 + addEventToQueue(session, { + event: { + promptEnd: { + promptName: session.promptName + } + } + }); + + // 少し待機して、プロンプト終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + } catch (error) { + console.error(`Error ending audio content for session ${session.sessionId}:`, error); + } +} + +// セッションを閉じる関数 +async function closeSession(session: any) { + if (!session || !session.isActive) { + if (session) { + console.log(`Session ${session.sessionId} already closed`); + } + return; + } + + console.log(`Closing session ${session.sessionId}`); + + try { + // 音声コンテンツが開始されていれば終了 + if (session.isAudioContentStartSent) { + await endAudioContent(session); + } + + // セッション終了イベントをキューに追加 + addEventToQueue(session, { + event: { + sessionEnd: {} + } + }); + + // 少し待機して、セッション終了イベントが処理されるのを待つ + await new Promise(resolve => setTimeout(resolve, 500)); + + // セッションを非アクティブにする + session.isActive = false; + + // アクティブセッションから削除 + activeSessions.delete(session.sessionId); + + console.log(`Session ${session.sessionId} closed`); + } catch (error) { + console.error(`Error closing session ${session.sessionId}:`, error); + + // エラーが発生しても確実にセッションを閉じる + session.isActive = false; + activeSessions.delete(session.sessionId); + } +} + +// イベントをセッションのキューに追加する関数 +function addEventToQueue(session: any, event: any) { + if (!session || !session.isActive) return; + + session.queue.push(event); + console.log(`Added event to queue for session ${session.sessionId}: ${JSON.stringify(event).substring(0, 100)}...`); +} + +// セッション用の非同期イテレータを作成する関数 +function createSessionAsyncIterable(session: any): AsyncIterable { + return { + [Symbol.asyncIterator]: () => { + console.log(`Creating async iterator for session ${session.sessionId}`); + + return { + next: async (): Promise> => { + try { + // セッションがアクティブでなければ終了 + if (!session || !session.isActive) { + console.log(`Session ${session?.sessionId} is not active, iterator done`); + return { value: undefined, done: true }; + } + + // キューが空なら待機 + while (session.queue.length === 0 && session.isActive) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + + // セッションが非アクティブになっていれば終了 + if (!session.isActive) { + return { value: undefined, done: true }; + } + + // キューからイベントを取得 + const nextEvent = session.queue.shift(); + console.log(`Sending event to Nova Sonic: ${JSON.stringify(nextEvent).substring(0, 100)}...`); + + return { + value: { + chunk: { + bytes: new TextEncoder().encode(JSON.stringify(nextEvent)) + } + }, + done: false + }; + } catch (error) { + console.error(`Error in session ${session?.sessionId} iterator:`, error); + if (session) { + session.isActive = false; + } + return { value: undefined, done: true }; + } + } + }; + } + }; +} diff --git a/packages/cdk/lib/construct/events.ts b/packages/cdk/lib/construct/events.ts index d98325757..41a9b689f 100644 --- a/packages/cdk/lib/construct/events.ts +++ b/packages/cdk/lib/construct/events.ts @@ -39,6 +39,7 @@ export class Events extends Construct { const lambda = new NodejsFunction(this, 'NovaSonic', { runtime: Runtime.NODEJS_LATEST, + // TODO: change filename entry: './lambda/nova-sonic-lambda.ts', timeout: Duration.minutes(15), environment: { @@ -47,6 +48,7 @@ export class Events extends Construct { bundling: { nodeModules: ['@aws-sdk/client-bedrock-runtime'], }, + memorySize: 512, }); eventApi.grantConnect(lambda); diff --git a/packages/web/public/audio-processor.worklet.js b/packages/web/public/audio-processor.worklet.js index 1bb076d52..c7db419f0 100644 --- a/packages/web/public/audio-processor.worklet.js +++ b/packages/web/public/audio-processor.worklet.js @@ -1,13 +1,13 @@ // Audio sample buffer to minimize reallocations class ExpandableBuffer { constructor() { - // Start with one second's worth of buffered audio capacity + // Start with one second's worth of buffered audio capacity at 24000Hz this.buffer = new Float32Array(24000); this.readIndex = 0; this.writeIndex = 0; this.underflowedSamples = 0; this.isInitialBuffering = true; - this.initialBufferLength = 24000; // One second + this.initialBufferLength = 4800; // 200ms buffer for faster start this.lastWriteTime = 0; } @@ -86,11 +86,11 @@ class AudioPlayerProcessor extends AudioWorkletProcessor { constructor() { super(); this.playbackBuffer = new ExpandableBuffer(); - console.log('[AudioWorklet] AudioPlayerProcessor initialized'); - + console.log(`[AudioWorklet] AudioPlayerProcessor initialized with sampleRate: ${sampleRate}`); + this.port.onmessage = (event) => { console.log('[AudioWorklet] Player received message:', event.data); - + if (event.data.type === "audio") { console.log('[AudioWorklet] Received audio data for playback, length:', event.data.audioData.length); this.playbackBuffer.write(event.data.audioData); @@ -106,7 +106,7 @@ class AudioPlayerProcessor extends AudioWorkletProcessor { this.playbackBuffer.clearBuffer(); } }; - + // 初期化完了を通知 this.port.postMessage({ type: 'init', diff --git a/packages/web/src/hooks/useNovaSonic/AudioPlayer.js b/packages/web/src/hooks/useNovaSonic/AudioPlayer.js new file mode 100644 index 000000000..70ef8f443 --- /dev/null +++ b/packages/web/src/hooks/useNovaSonic/AudioPlayer.js @@ -0,0 +1,128 @@ +import { ObjectExt } from './ObjectsExt.js'; +const AudioPlayerWorkletUrl = new URL('./AudioPlayerProcessor.worklet.js', import.meta.url).toString(); + +export class AudioPlayer { + constructor() { + this.onAudioPlayedListeners = []; + this.initialized = false; + } + + addEventListener(event, callback) { + switch (event) { + case "onAudioPlayed": + this.onAudioPlayedListeners.push(callback); + break; + default: + console.error("Listener registered for event type: " + JSON.stringify(event) + " which is not supported"); + } + } + + async start() { + this.audioContext = new AudioContext({ "sampleRate": 24000 }); + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + + // Chrome caches worklet code more aggressively, so add a nocache parameter to make sure we get the latest + await this.audioContext.audioWorklet.addModule(AudioPlayerWorkletUrl); // + "?nocache=" + Date.now()); + this.workletNode = new AudioWorkletNode(this.audioContext, "audio-player-processor"); + this.workletNode.connect(this.analyser); + this.analyser.connect(this.audioContext.destination); + this.recorderNode = this.audioContext.createScriptProcessor(512, 1, 1); + this.recorderNode.onaudioprocess = (event) => { + // Pass the input along as-is + const inputData = event.inputBuffer.getChannelData(0); + const outputData = event.outputBuffer.getChannelData(0); + outputData.set(inputData); + // Notify listeners that the audio was played + const samples = new Float32Array(outputData.length); + samples.set(outputData); + this.onAudioPlayedListeners.map(listener => listener(samples)); + } + this.#maybeOverrideInitialBufferLength(); + this.initialized = true; + } + + bargeIn() { + this.workletNode.port.postMessage({ + type: "barge-in", + }) + } + + stop() { + if (ObjectExt.exists(this.audioContext)) { + this.audioContext.close(); + } + + if (ObjectExt.exists(this.analyser)) { + this.analyser.disconnect(); + } + + if (ObjectExt.exists(this.workletNode)) { + this.workletNode.disconnect(); + } + + if (ObjectExt.exists(this.recorderNode)) { + this.recorderNode.disconnect(); + } + + this.initialized = false; + this.audioContext = null; + this.analyser = null; + this.workletNode = null; + this.recorderNode = null; + } + + #maybeOverrideInitialBufferLength() { + // Read a user-specified initial buffer length from the URL parameters to help with tinkering + const params = new URLSearchParams(window.location.search); + const value = params.get("audioPlayerInitialBufferLength"); + if (value === null) { + return; // No override specified + } + const bufferLength = parseInt(value); + if (isNaN(bufferLength)) { + console.error("Invalid audioPlayerInitialBufferLength value:", JSON.stringify(value)); + return; + } + this.workletNode.port.postMessage({ + type: "initial-buffer-length", + bufferLength: bufferLength, + }); + } + + playAudio(samples) { + if (!this.initialized) { + console.error("The audio player is not initialized. Call init() before attempting to play audio."); + return; + } + this.workletNode.port.postMessage({ + type: "audio", + audioData: samples, + }); + } + + getSamples() { + if (!this.initialized) { + return null; + } + const bufferLength = this.analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + this.analyser.getByteTimeDomainData(dataArray); + return [...dataArray].map(e => e / 128 - 1); + } + + getVolume() { + if (!this.initialized) { + return 0; + } + const bufferLength = this.analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + this.analyser.getByteTimeDomainData(dataArray); + let normSamples = [...dataArray].map(e => e / 128 - 1); + let sum = 0; + for (let i = 0; i < normSamples.length; i++) { + sum += normSamples[i] * normSamples[i]; + } + return Math.sqrt(sum / normSamples.length); + } +} diff --git a/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js b/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js new file mode 100644 index 000000000..c7e27e0ab --- /dev/null +++ b/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js @@ -0,0 +1,114 @@ +// Audio sample buffer to minimize reallocations +class ExpandableBuffer { + + constructor() { + // Start with one second's worth of buffered audio capacity before needing to expand + this.buffer = new Float32Array(24000); + this.readIndex = 0; + this.writeIndex = 0; + this.underflowedSamples = 0; + this.isInitialBuffering = true; + this.initialBufferLength = 24000; // One second + this.lastWriteTime = 0; + } + + logTimeElapsedSinceLastWrite() { + const now = Date.now(); + if (this.lastWriteTime !== 0) { + const elapsed = now - this.lastWriteTime; + console.log(`Elapsed time since last audio buffer write: ${elapsed} ms`); + } + this.lastWriteTime = now; + } + + write(samples) { + this.logTimeElapsedSinceLastWrite(); + if (this.writeIndex + samples.length <= this.buffer.length) { + // Enough space to append the new samples + } + else { + // Not enough space ... + if (samples.length <= this.readIndex) { + // ... but we can shift samples to the beginning of the buffer + const subarray = this.buffer.subarray(this.readIndex, this.writeIndex); + console.log(`Shifting the audio buffer of length ${subarray.length} by ${this.readIndex}`); + this.buffer.set(subarray); + } + else { + // ... and we need to grow the buffer capacity to make room for more audio + const newLength = (samples.length + this.writeIndex - this.readIndex) * 2; + const newBuffer = new Float32Array(newLength); + console.log(`Expanding the audio buffer from ${this.buffer.length} to ${newLength}`); + newBuffer.set(this.buffer.subarray(this.readIndex, this.writeIndex)); + this.buffer = newBuffer; + } + this.writeIndex -= this.readIndex; + this.readIndex = 0; + } + this.buffer.set(samples, this.writeIndex); + this.writeIndex += samples.length; + if (this.writeIndex - this.readIndex >= this.initialBufferLength) { + // Filled the initial buffer length, so we can start playback with some cushion + this.isInitialBuffering = false; + console.log("Initial audio buffer filled"); + } + } + + read(destination) { + let copyLength = 0; + if (!this.isInitialBuffering) { + // Only start to play audio after we've built up some initial cushion + copyLength = Math.min(destination.length, this.writeIndex - this.readIndex); + } + destination.set(this.buffer.subarray(this.readIndex, this.readIndex + copyLength)); + this.readIndex += copyLength; + if (copyLength > 0 && this.underflowedSamples > 0) { + console.log(`Detected audio buffer underflow of ${this.underflowedSamples} samples`); + this.underflowedSamples = 0; + } + if (copyLength < destination.length) { + // Not enough samples (buffer underflow). Fill the rest with silence. + destination.fill(0, copyLength); + this.underflowedSamples += destination.length - copyLength; + } + if (copyLength === 0) { + // Ran out of audio, so refill the buffer to the initial length before playing more + this.isInitialBuffering = true; + } + } + + clearBuffer() { + this.readIndex = 0; + this.writeIndex = 0; + } +} + +class AudioPlayerProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.playbackBuffer = new ExpandableBuffer(); + this.port.onmessage = (event) => { + if (event.data.type === "audio") { + this.playbackBuffer.write(event.data.audioData); + } + else if (event.data.type === "initial-buffer-length") { + // Override the current playback initial buffer length + const newLength = event.data.bufferLength; + this.playbackBuffer.initialBufferLength = newLength; + // amazonq-ignore-next-line + console.log(`Changed initial audio buffer length to: ${newLength}`) + } + else if (event.data.type === "barge-in") { + this.playbackBuffer.clearBuffer(); + } + }; + } + + process(inputs, outputs, parameters) { + const output = outputs[0][0]; // Assume one output with one channel + this.playbackBuffer.read(output); + return true; // True to continue processing + } +} + +registerProcessor("audio-player-processor", AudioPlayerProcessor); diff --git a/packages/web/src/hooks/useNovaSonic/ObjectsExt.js b/packages/web/src/hooks/useNovaSonic/ObjectsExt.js new file mode 100644 index 000000000..2fceb9427 --- /dev/null +++ b/packages/web/src/hooks/useNovaSonic/ObjectsExt.js @@ -0,0 +1,17 @@ +export class ObjectExt { + static exists(obj) { + return obj !== undefined && obj !== null; + } + + static checkArgument(condition, message) { + if (!condition) { + throw TypeError(message); + } + } + + static checkExists(obj, message) { + if (ObjectsExt.exists(obj)) { + throw TypeError(message); + } + } +} diff --git a/packages/web/src/hooks/useNovaSonic/index.ts b/packages/web/src/hooks/useNovaSonic/index.ts new file mode 100644 index 000000000..d55d0c261 --- /dev/null +++ b/packages/web/src/hooks/useNovaSonic/index.ts @@ -0,0 +1,226 @@ +import { useEffect, useRef, useState } from 'react'; +import { events } from 'aws-amplify/data'; +import { AudioPlayer } from './AudioPlayer'; + +const MAX_AUDIO_CHUNKS_PER_BATCH = 5; + +const arrayBufferToBase64 = (buffer: any) => { + const binary = []; + const bytes = new Uint8Array(buffer); + for (let i = 0; i < bytes.byteLength; i++) { + binary.push(String.fromCharCode(bytes[i])); + } + return btoa(binary.join('')); +}; + +const float32ArrayToInt16Array = (float32Array: Float32Array): Int16Array => { + const int16Array = new Int16Array(float32Array.length); + for (let i = 0; i < float32Array.length; i++) { + // Float32を-32768から32767の範囲にスケーリング + const s = Math.max(-1, Math.min(1, float32Array[i])); + int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + return int16Array; +}; + +const base64ToFloat32Array = (base64String: string) => { + try { + const binaryString = atob(base64String); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + const int16Array = new Int16Array(bytes.buffer); + const float32Array = new Float32Array(int16Array.length); + for (let i = 0; i < int16Array.length; i++) { + float32Array[i] = int16Array[i] / 32768.0; + } + + return float32Array; + } catch (error) { + console.error('Error in base64ToFloat32Array:', error); + throw error; + } +}; + +export const useNovaSonic = () => { + const [isRecording, setIsRecording] = useState(false); + const audioPlayerRef = useRef(null); + const channelRef = useRef(null); + const audioContextRef = useRef(null); + const audioStreamRef = useRef(null); + const sourceNodeRef = useRef(null); + const processorRef = useRef(null); + const audioInputQueue = useRef([]); + + const dispatchEvent = async (event: string, data: any = undefined) => { + if (channelRef.current) { + await channelRef.current.publish({ + direction: 'ctob', // client to bedrock + event, + data, + }); + } + }; + + const initAudio = async () => { + const audioStream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }); + + const audioContext = new AudioContext({ + sampleRate: 16000 + }); + + audioStreamRef.current = audioStream; + audioContextRef.current = audioContext; + + const audioPlayer = new AudioPlayer(); + await audioPlayer.start(); + + audioPlayerRef.current = audioPlayer; + }; + + const processAudioInput = async () => { + if (isRecording) { + let processedChunks = 0; + let combinedLength = 0; + + const chunksToProcess: Float32Array[] = []; + + while (audioInputQueue.current.length > 0 && processedChunks < MAX_AUDIO_CHUNKS_PER_BATCH) { + const chunk = audioInputQueue.current.shift(); + + if (chunk) { + chunksToProcess.push(chunk); + combinedLength += chunk.length; + processedChunks += 1; + } + } + + if (chunksToProcess.length > 0) { + let offset = 0; + + const combinedBuffer = new Float32Array(combinedLength); + + for (const chunk of chunksToProcess) { + combinedBuffer.set(chunk, offset); + offset += chunk.length; + } + + const int16Array = float32ArrayToInt16Array(combinedBuffer); + const buffer = int16Array.buffer; + const bytes = new Uint8Array(buffer); + const base64Data = arrayBufferToBase64(bytes); + + dispatchEvent('audioInput', base64Data); + } + } + + setTimeout(() => processAudioInput(), 100); + }; + + useEffect(() => { + processAudioInput(); + }, []); + + const connectToAppSync = async () => { + audioInputQueue.current = []; + + const channel = await events.connect('/default/dummy-session'); + channelRef.current = channel; + + channel.subscribe({ + next: (data: any) => { + const event = data?.event; + if (event && event.direction === 'btoc') { + console.log(event); + if (event.event === 'audioOutput') { + const audioData = base64ToFloat32Array(event.data.content); + audioPlayerRef.current.playAudio(audioData); + } + } + }, + error: (e: any) => { + console.error(e); + }, + }); + }; + + const startSession = async () => { + await initAudio(); + await connectToAppSync(); + }; + + const startRecording = async () => { + await dispatchEvent('audioStart'); + + setIsRecording(true); + + const sourceNode = audioContextRef.current.createMediaStreamSource(audioStreamRef.current); + + if (audioContextRef.current.createScriptProcessor) { + const processor = audioContextRef.current.createScriptProcessor(512, 1, 1); + + processor.onaudioprocess = (e: any) => { + const inputData = e.inputBuffer.getChannelData(0); + // const pcmData = new Int16Array(inputData.length); + // for (let i = 0; i < inputData.length; i++) { + // pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 0x7FFF; + // } + // const base64Data = arrayBufferToBase64(pcmData.buffer); + // dispatchEvent('audioInput', base64Data); + audioInputQueue.current.push(inputData); + }; + + sourceNode.connect(processor); + processor.connect(audioContextRef.current.destination); + + sourceNodeRef.current = sourceNode; + processorRef.current = processor; + } + }; + + const stopRecording = async () => { + if (processorRef.current) { + processorRef.current.disconnect(); + processorRef.current = null; + } + + if (sourceNodeRef.current) { + sourceNodeRef.current.disconnect(); + sourceNodeRef.current = null; + } + + if (audioPlayerRef.current) { + audioPlayerRef.current.stop(); + audioPlayerRef.current = null; + } + + if (audioStreamRef.current) { + audioStreamRef.current.getTracks().forEach((track: any) => track.stop()); + audioStreamRef.current = null; + } + + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + + setIsRecording(false); + + await dispatchEvent('audioStop'); + }; + + return { + isRecording, + startSession, + startRecording, + stopRecording, + } +}; diff --git a/packages/web/src/hooks/useNovaSonic.ts b/packages/web/src/hooks/useNovaSonicOld similarity index 89% rename from packages/web/src/hooks/useNovaSonic.ts rename to packages/web/src/hooks/useNovaSonicOld index 9f3b2417b..314488def 100644 --- a/packages/web/src/hooks/useNovaSonic.ts +++ b/packages/web/src/hooks/useNovaSonicOld @@ -1,12 +1,11 @@ import { useEffect, useRef, useCallback, useState } from 'react'; import { events } from 'aws-amplify/data'; -// AppSync Events のチャンネル名 -const CHANNEL_NAME = '/default/nova-sonic'; +const CHANNEL_NAME = '/default/dummy-session'; // バッファリング設定 -const MAX_QUEUE_SIZE = 200; // 最大キューサイズを小さくする -const MAX_CHUNKS_PER_BATCH = 5; // 一度に処理する最大チャンク数を減らす +// const MAX_QUEUE_SIZE = 200; // 最大キューサイズを小さくする +// const MAX_CHUNKS_PER_BATCH = 5; // 一度に処理する最大チャンク数を減らす // 音声データの変換ユーティリティ const convertFloat32ToInt16 = (float32Array: Float32Array): Int16Array => { @@ -20,13 +19,13 @@ const convertFloat32ToInt16 = (float32Array: Float32Array): Int16Array => { }; export const useNovaSonic = () => { - const [isConnected, setIsConnected] = useState(false); - const [isRecording, setIsRecording] = useState(false); - const [isPlaying, setIsPlaying] = useState(false); - const [error, setError] = useState(null); - + // const [isConnected, setIsConnected] = useState(false); + // const [isRecording, setIsRecording] = useState(false); + // const [isPlaying, setIsPlaying] = useState(false); + // const [error, setError] = useState(null); + // refs - const channelRef = useRef(null); + // const channelRef = useRef(null); // 録音用と再生用に別々のAudioContextを使用 const recordingContextRef = useRef(null); // 16000Hz const playbackContextRef = useRef(null); // 24000Hz @@ -37,7 +36,7 @@ export const useNovaSonic = () => { const analyserNodeRef = useRef(null); const sessionIdRef = useRef(crypto.randomUUID()); const recordingParamRef = useRef(null); - + // バッファリング用のrefs const audioBufferQueueRef = useRef([]); const isProcessingAudioRef = useRef(false); @@ -115,9 +114,9 @@ export const useNovaSonic = () => { binary += String.fromCharCode(bytes[i]); } const base64Data = btoa(binary); - - console.log(`Sending buffered audio data, chunks: ${chunksToProcess.length}, total samples: ${combinedLength}`); - + + console.log(`Sending buffered audio data ${base64Data}`); + // AppSync Events経由でLambdaに送信 try { await channelRef.current.publish({ @@ -128,7 +127,7 @@ export const useNovaSonic = () => { timestamp: Date.now() } }); - + console.log('Buffered audio data sent successfully'); } catch (publishError) { console.error('Failed to publish audio data:', publishError); @@ -173,6 +172,9 @@ export const useNovaSonic = () => { latencyHint: 'interactive' }); + console.log('Recording context sample rate:', recordingContextRef.current.sampleRate); + console.log('Playback context sample rate:', playbackContextRef.current.sampleRate); + console.log('Loading audio worklets...'); // Audio Workletの登録 - 完全なURLを使用 @@ -212,48 +214,58 @@ export const useNovaSonic = () => { channelRef.current = channel; setIsConnected(true); console.log('Connected to AppSync Events'); - + // バッファをクリア audioBufferQueueRef.current = []; isProcessingAudioRef.current = false; - + // AppSync Eventsからのメッセージ受信 channel.subscribe({ next: (data: any) => { if (data.event?.type === 'BedrockToAppSync' && data.event?.data) { + // セッションIDをチェックして、自分宛のメッセージのみ処理 + if (data.event.data.sessionId !== sessionIdRef.current) { + console.log(`Ignoring message for different session: ${data.event.data.sessionId}`); + return; + } + console.log('Received audio data from Bedrock'); - + // 受信した音声データを再生 if (data.event.data.audioData && audioWorkletNodeRef.current) { try { + console.log(`Received audio data from Bedrock, processing for playback ${data.event.data.audioData}`); + // Base64エンコードされた音声データをデコード const binaryData = atob(data.event.data.audioData); const bytes = new Uint8Array(binaryData.length); for (let i = 0; i < binaryData.length; i++) { bytes[i] = binaryData.charCodeAt(i); } - + // PCM音声データをFloat32Arrayに変換 const int16Data = new Int16Array(bytes.buffer); const float32Data = new Float32Array(int16Data.length); for (let i = 0; i < int16Data.length; i++) { float32Data[i] = int16Data[i] / 32768.0; } - + + console.log(`Processing audio data: length=${float32Data.length}, sample rate=${playbackContextRef.current?.sampleRate || 'unknown'}`); + // 再生用AudioContextの状態を確認して再開 if (playbackContextRef.current && playbackContextRef.current.state === 'suspended') { playbackContextRef.current.resume(); console.log('Playback AudioContext resumed, state:', playbackContextRef.current.state); } - + // 音声データをWorkletに送信 audioWorkletNodeRef.current.port.postMessage({ type: 'audio', audioData: float32Data }); - + setIsPlaying(true); - + // 再生状態を一定時間後に更新 setTimeout(() => { setIsPlaying(false); @@ -276,26 +288,26 @@ export const useNovaSonic = () => { setIsConnected(false); } }; - + // 初期化処理を実行 initializeAudio().then(connectToAppSync); - + // クリーンアップ関数 return () => { console.log('Cleaning up Nova Sonic hook'); - + // バッファをクリア if (audioBufferQueueRef.current.length > 0) { console.log(`Clearing audio buffer queue with ${audioBufferQueueRef.current.length} items`); audioBufferQueueRef.current = []; } - + // マイクストリームの停止 if (micStreamRef.current) { micStreamRef.current.getTracks().forEach(track => track.stop()); micStreamRef.current = null; } - + // AudioWorkletNodeの切断 if (audioWorkletNodeRef.current) { audioWorkletNodeRef.current.disconnect(); @@ -516,14 +528,14 @@ export const useNovaSonic = () => { console.error('Cannot start new session: Not connected to AppSync Events'); return; } - + // バッファをクリア audioBufferQueueRef.current = []; isProcessingAudioRef.current = false; - + sessionIdRef.current = crypto.randomUUID(); console.log(`Started new session with ID: ${sessionIdRef.current}`); - + // 新しいセッション開始を通知 channelRef.current.publish({ type: 'ClientToAppSync', @@ -538,7 +550,7 @@ export const useNovaSonic = () => { console.error('Failed to send start session notification:', err); }); }, [isConnected]); - + return { isConnected, isRecording, @@ -547,6 +559,41 @@ export const useNovaSonic = () => { startRecording, stopRecording, startNewSession, - sessionId: sessionIdRef.current + sessionId: sessionIdRef.current, + // getDebugInfo }; }; + +// デバッグ情報を取得する関数 +// const getDebugInfo = useCallback(() => { +// if (audioWorkletNodeRef.current) { +// console.log('Requesting debug info from audio worklet'); +// audioWorkletNodeRef.current.port.postMessage({ +// type: 'debug' +// }); +// } +// }, []); +// +// // 定期的にデバッグ情報を取得 +// useEffect(() => { +// if (isPlaying && audioWorkletNodeRef.current) { +// const debugInterval = setInterval(() => { +// getDebugInfo(); +// }, 5000); +// +// return () => { +// clearInterval(debugInterval); +// }; +// } +// }, [isPlaying, getDebugInfo]); +// +// // AudioWorkletからのデバッグ情報を受信 +// useEffect(() => { +// if (audioWorkletNodeRef.current) { +// audioWorkletNodeRef.current.port.onmessage = (event) => { +// if (event.data.type === 'debug-info') { +// console.log('Debug info from audio worklet:', event.data); +// } +// }; +// } +// }, []); diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index dbba96dae..0211fd89b 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -3,8 +3,8 @@ import { useNovaSonic } from '../hooks/useNovaSonic'; const SpeechToSpeech: React.FC = () => { const { - isConnected, - startNewSession, + isRecording, + startSession, startRecording, stopRecording, } = useNovaSonic(); @@ -13,13 +13,15 @@ const SpeechToSpeech: React.FC = () => {

Speech To Speech

- isConnected: {isConnected.toString()}
-
+
+ isRecording: {isRecording} +
- isRecording: {isRecording} + isRecording: {isRecording.toString()}
+ isLoading: {isLoading.toString()}
- isRecording: {isRecording.toString()} -
-
-
-
diff --git a/packages/web/src/vite-env.d.ts b/packages/web/src/vite-env.d.ts index 03c332816..1c7cfb4cd 100644 --- a/packages/web/src/vite-env.d.ts +++ b/packages/web/src/vite-env.d.ts @@ -27,6 +27,8 @@ interface ImportMetaEnv { readonly VITE_APP_USE_CASE_BUILDER_ENABLED: string; readonly VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN: string; readonly VITE_APP_HIDDEN_USE_CASES: string; + readonly VITE_APP_SPEECH_TO_SPEECH_NAMESPACE: string; + readonly VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT: string; } interface ImportMeta { diff --git a/setup-env.sh b/setup-env.sh index fa4ecf20f..b79ee6667 100755 --- a/setup-env.sh +++ b/setup-env.sh @@ -47,3 +47,5 @@ export VITE_APP_INLINE_AGENTS=$(extract_value "$stack_output" InlineAgents) export VITE_APP_USE_CASE_BUILDER_ENABLED=$(extract_value "$stack_output" UseCaseBuilderEnabled) export VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN=$(extract_value "$stack_output" OptimizePromptFunctionArn) export VITE_APP_HIDDEN_USE_CASES=$(extract_value "$stack_output" HiddenUseCases) +export VITE_APP_SPEECH_TO_SPEECH_NAMESPACE=$(extract_value "$stack_output" SpeechToSpeechNamespace) +export VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT=$(extract_value "$stack_output" SpeechToSpeechEventApiEndpoint) diff --git a/web_devw_win.ps1 b/web_devw_win.ps1 index ec6fce7ee..7446da518 100644 --- a/web_devw_win.ps1 +++ b/web_devw_win.ps1 @@ -80,5 +80,7 @@ $env:VITE_APP_INLINE_AGENTS = Extract-Value $stack_output "InlineAgents" $env:VITE_APP_USE_CASE_BUILDER_ENABLED = Extract-Value $stack_output "UseCaseBuilderEnabled" $env:VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN = Extract-Value $stack_output "OptimizePromptFunctionArn" $env:VITE_APP_HIDDEN_USE_CASES = Extract-Value $stack_output "HiddenUseCases" +$env:VITE_APP_SPEECH_TO_SPEECH_NAMESPACE = Extract-Value $stack_output "SpeechToSpeechNamespace" +$env:VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT = Extract-Value $stack_output "SpeechToSpeechEventApiEndpoint" npm -w packages/web run dev From b948696a58ec148496105bd264d307aa76026003 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Mon, 14 Apr 2025 23:38:48 +0900 Subject: [PATCH 05/20] fix --- packages/cdk/lambda/speechToSpeechTask.ts | 72 +++++++++++++++---- .../cdk/lambda/startSpeechToSpeechSession.ts | 2 +- .../AudioPlayerProcessor.worklet.js | 12 ++-- packages/web/src/hooks/useNovaSonic/index.ts | 16 +++++ 4 files changed, 81 insertions(+), 21 deletions(-) diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index 171b5ae10..d3a04bb65 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -75,7 +75,7 @@ const enqueuePromptStart = () => { }); }; -const enqueueSystemPrompt = () => { +const enqueueSystemPrompt = (prompt: string) => { const contentName = randomUUID(); eventQueue.push({ @@ -98,7 +98,7 @@ const enqueueSystemPrompt = () => { textInput: { promptName, contentName, - content: 'You are the AI assistant', + content: prompt, }, } }); @@ -172,7 +172,7 @@ const enqueueAudioStop = () => { }; const enqueueAudioInput = (audioInputBase64Array: string[]) => { - if (!isAudioStarted) { + if (!isAudioStarted || !isActive) { return; } @@ -314,7 +314,9 @@ const processResponseStream = async (channel: EventsChannel, response: any) => { } }; -export const handler = async (event: { channel: string }) => { +export const handler = async (event: { channelId: string }) => { + let channel: EventsChannel | null = null; + try { console.log('event', event); @@ -324,6 +326,8 @@ export const handler = async (event: { channel: string }) => { promptName = randomUUID(); + console.log('promptName', promptName); + const bedrock = new BedrockRuntimeClient({ region: 'us-east-1', // TODO requestHandler: new NodeHttp2Handler({ @@ -334,6 +338,8 @@ export const handler = async (event: { channel: string }) => { }), }); + console.log('Bedrock client initialized'); + Amplify.configure( { API: { @@ -360,13 +366,22 @@ export const handler = async (event: { channel: string }) => { } ); - const channel = await events.connect(`/${process.env.NAMESPACE}/${event.channel}`); + console.log('Amplify configured'); + console.log(`Connect to the channel /${process.env.NAMESPACE}/${event.channelId}`) + + channel = await events.connect(`/${process.env.NAMESPACE}/${event.channelId}`); + + console.log('Connected!'); channel.subscribe({ next: async (data: any) => { const event = data?.event; if (event && event.direction === 'ctob') { - if (event.event === 'audioStart') { + if (event.event === 'promptStart') { + enqueuePromptStart(); + } else if (event.event === 'systemPrompt') { + enqueueSystemPrompt(event.data); + } else if (event.event === 'audioStart') { enqueueAudioStart(); } else if (event.event === 'audioInput') { enqueueAudioInput(event.data); @@ -376,19 +391,41 @@ export const handler = async (event: { channel: string }) => { enqueueAudioStop(); enqueuePromptEnd(); enqueueSessionEnd(); - channel.close(); + + if (channel) { + await channel.publish({ + direction: 'btoc', + event: 'end', + }); + channel.close(); + } } } }, error: console.error, }); + console.log('Subscribed to the channel'); + enqueueSessionStart(); - enqueuePromptStart(); - enqueueSystemPrompt(); + + // Without this sleep, the error below is raised + // "Subscription has not been initialized" + console.log('Sleep...'); + await new Promise(s => setTimeout(s, 1000)); + + // Notify the status to the client + await channel.publish({ + direction: 'btoc', + event: 'ready', + }); + + console.log('I\'m ready'); const asyncIterator = createAsyncIterator(); + console.log('Async iterator created'); + const response = await bedrock.send( new InvokeModelWithBidirectionalStreamCommand({ modelId: 'amazon.nova-sonic-v1:0', @@ -396,17 +433,24 @@ export const handler = async (event: { channel: string }) => { }), ); - // Notify the status to the client - await channel.publish({ - direction: 'btoc', - event: 'ready', - }); + console.log('Bidirectional stream command sent'); // Start response stream await processResponseStream(channel, response); } catch (e) { console.error(e); } finally { + if (channel) { + try { + channel.publish({ + direction: 'btoc', + event: 'end', + }); + } catch (e) { + console.error(e); + } + } + initialize(); console.log('Session ended. Every parameters are initialized.'); } diff --git a/packages/cdk/lambda/startSpeechToSpeechSession.ts b/packages/cdk/lambda/startSpeechToSpeechSession.ts index 5b7a5b2a6..16aecdb17 100644 --- a/packages/cdk/lambda/startSpeechToSpeechSession.ts +++ b/packages/cdk/lambda/startSpeechToSpeechSession.ts @@ -15,7 +15,7 @@ export const handler = async ( await lambda.send(new InvokeCommand({ FunctionName: process.env.SPEECH_TO_SPEECH_TASK_FUNCTION_ARN, InvocationType: InvocationType.Event, - Payload: JSON.stringify({ channel }), + Payload: JSON.stringify({ channelId: channel }), })); return { diff --git a/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js b/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js index c7e27e0ab..3a6c96236 100644 --- a/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js +++ b/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js @@ -16,7 +16,7 @@ class ExpandableBuffer { const now = Date.now(); if (this.lastWriteTime !== 0) { const elapsed = now - this.lastWriteTime; - console.log(`Elapsed time since last audio buffer write: ${elapsed} ms`); + // console.log(`Elapsed time since last audio buffer write: ${elapsed} ms`); } this.lastWriteTime = now; } @@ -31,14 +31,14 @@ class ExpandableBuffer { if (samples.length <= this.readIndex) { // ... but we can shift samples to the beginning of the buffer const subarray = this.buffer.subarray(this.readIndex, this.writeIndex); - console.log(`Shifting the audio buffer of length ${subarray.length} by ${this.readIndex}`); + // console.log(`Shifting the audio buffer of length ${subarray.length} by ${this.readIndex}`); this.buffer.set(subarray); } else { // ... and we need to grow the buffer capacity to make room for more audio const newLength = (samples.length + this.writeIndex - this.readIndex) * 2; const newBuffer = new Float32Array(newLength); - console.log(`Expanding the audio buffer from ${this.buffer.length} to ${newLength}`); + // console.log(`Expanding the audio buffer from ${this.buffer.length} to ${newLength}`); newBuffer.set(this.buffer.subarray(this.readIndex, this.writeIndex)); this.buffer = newBuffer; } @@ -50,7 +50,7 @@ class ExpandableBuffer { if (this.writeIndex - this.readIndex >= this.initialBufferLength) { // Filled the initial buffer length, so we can start playback with some cushion this.isInitialBuffering = false; - console.log("Initial audio buffer filled"); + // console.log("Initial audio buffer filled"); } } @@ -63,7 +63,7 @@ class ExpandableBuffer { destination.set(this.buffer.subarray(this.readIndex, this.readIndex + copyLength)); this.readIndex += copyLength; if (copyLength > 0 && this.underflowedSamples > 0) { - console.log(`Detected audio buffer underflow of ${this.underflowedSamples} samples`); + // console.log(`Detected audio buffer underflow of ${this.underflowedSamples} samples`); this.underflowedSamples = 0; } if (copyLength < destination.length) { @@ -96,7 +96,7 @@ class AudioPlayerProcessor extends AudioWorkletProcessor { const newLength = event.data.bufferLength; this.playbackBuffer.initialBufferLength = newLength; // amazonq-ignore-next-line - console.log(`Changed initial audio buffer length to: ${newLength}`) + // console.log(`Changed initial audio buffer length to: ${newLength}`) } else if (event.data.type === "barge-in") { this.playbackBuffer.clearBuffer(); diff --git a/packages/web/src/hooks/useNovaSonic/index.ts b/packages/web/src/hooks/useNovaSonic/index.ts index 26405931a..a7f5edf46 100644 --- a/packages/web/src/hooks/useNovaSonic/index.ts +++ b/packages/web/src/hooks/useNovaSonic/index.ts @@ -130,6 +130,12 @@ export const useNovaSonic = () => { startRecording().then(() => { isLoading.current = false; }); + } else if (event.event === 'end') { + console.log('Received "end" event'); + if (isActive.current) { + console.log('Close the session'); + closeSession(); + } } else if (event.event === 'audioOutput' && audioPlayerRef.current) { const chunks: string[] = event.data; @@ -153,6 +159,8 @@ export const useNovaSonic = () => { }; const startRecording = async () => { + await dispatchEvent('promptStart'); + await dispatchEvent('systemPrompt', 'You are an AI assistant'); await dispatchEvent('audioStart'); const sourceNode = audioContextRef.current.createMediaStreamSource(audioStreamRef.current); @@ -211,13 +219,21 @@ export const useNovaSonic = () => { }; const startSession = async () => { + if (isActive.current || isLoading.current) { + return; + } + isLoading.current = true; + await connectToAppSync(); await initAudio(); }; const closeSession = async () => { await stopRecording(); + + isActive.current = false; + isLoading.current = false; }; return { From f761b6dce94cd80209070eb4e0843e0927982d92 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Mon, 14 Apr 2025 23:57:19 +0900 Subject: [PATCH 06/20] fix --- packages/cdk/lambda/speechToSpeechTask.ts | 16 ++++-- packages/web/src/hooks/useNovaSonic/index.ts | 54 ++++++++++---------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index d3a04bb65..ff0c87c21 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -393,11 +393,16 @@ export const handler = async (event: { channelId: string }) => { enqueueSessionEnd(); if (channel) { - await channel.publish({ - direction: 'btoc', - event: 'end', - }); - channel.close(); + try { + await channel.publish({ + direction: 'btoc', + event: 'end', + }); + channel.close(); + } catch (e) { + console.error(e); + throw e; + } } } } @@ -446,6 +451,7 @@ export const handler = async (event: { channelId: string }) => { direction: 'btoc', event: 'end', }); + channel.close(); } catch (e) { console.error(e); } diff --git a/packages/web/src/hooks/useNovaSonic/index.ts b/packages/web/src/hooks/useNovaSonic/index.ts index a7f5edf46..1bf7b9191 100644 --- a/packages/web/src/hooks/useNovaSonic/index.ts +++ b/packages/web/src/hooks/useNovaSonic/index.ts @@ -1,4 +1,4 @@ -import { useRef, useCallback } from 'react'; +import { useRef, useState } from 'react'; import { events, EventsChannel } from 'aws-amplify/data'; import { AudioPlayer } from './AudioPlayer'; import { v4 as uuid } from 'uuid'; @@ -48,8 +48,8 @@ const base64ToFloat32Array = (base64String: string) => { export const useNovaSonic = () => { const api = useHttp(); - const isActive = useRef(false); - const isLoading = useRef(false); + const [isActive, setIsActive] = useState(false); + const [isLoading, setIsLoading] = useState(false); const audioPlayerRef = useRef(null); const channelRef = useRef(null); const audioContextRef = useRef(null); @@ -90,28 +90,26 @@ export const useNovaSonic = () => { audioPlayerRef.current = audioPlayer; }; - const processAudioInput = useCallback(async () => { - if (isActive.current) { - if (audioInputQueue.current.length > MIN_AUDIO_CHUNKS_PER_BATCH) { - const chunksToProcess: string[] = []; + const processAudioInput = async () => { + if (audioInputQueue.current.length > MIN_AUDIO_CHUNKS_PER_BATCH) { + const chunksToProcess: string[] = []; - let processedChunks = 0; + let processedChunks = 0; - while (audioInputQueue.current.length > 0 && processedChunks < MAX_AUDIO_CHUNKS_PER_BATCH) { - const chunk = audioInputQueue.current.shift(); + while (audioInputQueue.current.length > 0 && processedChunks < MAX_AUDIO_CHUNKS_PER_BATCH) { + const chunk = audioInputQueue.current.shift(); - if (chunk) { - chunksToProcess.push(chunk); - processedChunks += 1; - } + if (chunk) { + chunksToProcess.push(chunk); + processedChunks += 1; } - - dispatchEvent('audioInput', chunksToProcess); } - setTimeout(() => processAudioInput(), 0); + await dispatchEvent('audioInput', chunksToProcess); } - }, [isActive]); + + setTimeout(() => processAudioInput(), 0); + } const connectToAppSync = async () => { audioInputQueue.current = []; @@ -128,11 +126,11 @@ export const useNovaSonic = () => { if (event.event === 'ready') { console.log('Now ready to speech-to-speech!'); startRecording().then(() => { - isLoading.current = false; + setIsLoading(false); }); } else if (event.event === 'end') { console.log('Received "end" event'); - if (isActive.current) { + if (isActive) { console.log('Close the session'); closeSession(); } @@ -182,13 +180,13 @@ export const useNovaSonic = () => { processorRef.current = processor; } - isActive.current = true; + setIsActive(true); processAudioInput(); }; const stopRecording = async () => { - isActive.current = false; + setIsActive(false); if (processorRef.current) { processorRef.current.disconnect(); @@ -219,11 +217,11 @@ export const useNovaSonic = () => { }; const startSession = async () => { - if (isActive.current || isLoading.current) { + if (isActive || isLoading) { return; } - isLoading.current = true; + setIsLoading(true); await connectToAppSync(); await initAudio(); @@ -232,13 +230,13 @@ export const useNovaSonic = () => { const closeSession = async () => { await stopRecording(); - isActive.current = false; - isLoading.current = false; + setIsActive(false); + setIsLoading(false); }; return { - isActive: isActive.current, - isLoading: isLoading.current, + isActive, + isLoading, startSession, closeSession, } From f1d61f0576509a50d8e84a3175e6a50e07914de2 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Tue, 15 Apr 2025 18:43:29 +0900 Subject: [PATCH 07/20] fix --- packages/cdk/lambda/speechToSpeechTask.ts | 75 +++++++---- packages/types/src/index.d.ts | 1 + packages/types/src/speech-to-speech.d.ts | 22 ++++ packages/web/src/components/ChatMessage.tsx | 4 +- .../AudioPlayer.js | 0 .../AudioPlayerProcessor.worklet.js | 0 .../ObjectsExt.js | 0 .../index.ts | 48 +++++-- .../hooks/useSpeechToSpeech/useChatHistory.ts | 110 ++++++++++++++++ packages/web/src/pages/SpeechToSpeechPage.tsx | 120 +++++++++++++++--- 10 files changed, 322 insertions(+), 58 deletions(-) create mode 100644 packages/types/src/speech-to-speech.d.ts rename packages/web/src/hooks/{useNovaSonic => useSpeechToSpeech}/AudioPlayer.js (100%) rename packages/web/src/hooks/{useNovaSonic => useSpeechToSpeech}/AudioPlayerProcessor.worklet.js (100%) rename packages/web/src/hooks/{useNovaSonic => useSpeechToSpeech}/ObjectsExt.js (100%) rename packages/web/src/hooks/{useNovaSonic => useSpeechToSpeech}/index.ts (80%) create mode 100644 packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index ff0c87c21..57c40408e 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -6,8 +6,13 @@ import { BedrockRuntimeClient, InvokeModelWithBidirectionalStreamCommand, InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamCommandOutput, } from "@aws-sdk/client-bedrock-runtime"; import { NodeHttp2Handler } from '@smithy/node-http-handler'; +import { + SpeechToSpeechEventType, + SpeechToSpeechEvent, +} from 'generative-ai-use-cases'; Object.assign(global, { WebSocket: require('ws') }); @@ -39,6 +44,14 @@ const initialize = () => { audioOutputQueue = []; }; +const dispatchEvent = async (channel: EventsChannel, event: SpeechToSpeechEventType, data: any = undefined) => { + await channel.publish({ + direction: 'btoc', + event, + data, + } as SpeechToSpeechEvent); +}; + const enqueueSessionStart = () => { eventQueue.push({ event: { @@ -202,7 +215,6 @@ const enqueueAudioOutput = async (channel: EventsChannel, audioOutput: string) = while (audioOutputQueue.length > 0 && processedChunks < MAX_AUDIO_OUTPUT_PER_BATCH) { const chunk = audioOutputQueue.shift(); - console.log('chunk to process', chunk); if (chunk) { chunksToProcess.push(chunk); @@ -210,11 +222,7 @@ const enqueueAudioOutput = async (channel: EventsChannel, audioOutput: string) = } } - await channel.publish({ - direction: 'btoc', - event: 'audioOutput', - data: chunksToProcess, - }); + await dispatchEvent(channel, 'audioOutput', chunksToProcess); } }; @@ -228,11 +236,7 @@ const forcePublishAudioOutput = async (channel: EventsChannel) => { } } - await channel.publish({ - direction: 'btoc', - event: 'audioOutput', - data: chunksToProcess, - }); + await dispatchEvent(channel, 'audioOutput', chunksToProcess); }; const createAsyncIterator = () => { @@ -249,7 +253,7 @@ const createAsyncIterator = () => { } const nextEvent = eventQueue.shift(); - console.log(`Consume event ${JSON.stringify(nextEvent)}`); + return { value: { chunk: { @@ -262,7 +266,6 @@ const createAsyncIterator = () => { }; }, return: async () => { - console.log('It seems all done.'); return { value: undefined, done: true } }, throw: async (error: any) => { @@ -294,18 +297,40 @@ const processAudioQueue = async () => { } }; -const processResponseStream = async (channel: EventsChannel, response: any) => { +const processResponseStream = async (channel: EventsChannel, response: InvokeModelWithBidirectionalStreamCommandOutput) => { + if (!response.body) { + throw new Error('Response body is null'); + } + try { for await (const event of response.body) { if (event.chunk?.bytes) { const textResponse = new TextDecoder().decode(event.chunk.bytes); const jsonResponse = JSON.parse(textResponse); - console.log('JSON Response', jsonResponse); + + console.log('Bedrock response', jsonResponse); if (jsonResponse.event?.audioOutput) { await enqueueAudioOutput(channel, jsonResponse.event.audioOutput.content); } else if (jsonResponse.event?.contentEnd && jsonResponse.event?.contentEnd?.type === 'AUDIO') { await forcePublishAudioOutput(channel); + } else if (jsonResponse.event?.contentStart && jsonResponse.event?.contentStart?.type === 'TEXT') { + await dispatchEvent(channel, 'textStart', { + id: jsonResponse.event?.contentStart?.contentId, + role: jsonResponse.event?.contentStart?.role?.toLowerCase(), + }); + } else if (jsonResponse.event?.textOutput) { + await dispatchEvent(channel, 'textOutput', { + id: jsonResponse.event?.textOutput?.contentId, + role: jsonResponse.event?.textOutput?.role?.toLowerCase(), + content: jsonResponse.event?.textOutput?.content, + }); + } else if (jsonResponse.event?.contentEnd && jsonResponse.event?.contentEnd?.type === 'TEXT') { + await dispatchEvent(channel, 'textStop', { + id: jsonResponse.event?.contentEnd?.contentId, + role: jsonResponse.event?.contentEnd?.role?.toLowerCase(), + stopReason: jsonResponse.event?.contentEnd?.stopReason, + }); } } } @@ -318,7 +343,7 @@ export const handler = async (event: { channelId: string }) => { let channel: EventsChannel | null = null; try { - console.log('event', event); + console.log('channelId', event.channelId); initialize(); @@ -394,10 +419,7 @@ export const handler = async (event: { channelId: string }) => { if (channel) { try { - await channel.publish({ - direction: 'btoc', - event: 'end', - }); + await dispatchEvent(channel, 'end'); channel.close(); } catch (e) { console.error(e); @@ -420,10 +442,7 @@ export const handler = async (event: { channelId: string }) => { await new Promise(s => setTimeout(s, 1000)); // Notify the status to the client - await channel.publish({ - direction: 'btoc', - event: 'ready', - }); + await dispatchEvent(channel, 'ready'); console.log('I\'m ready'); @@ -447,11 +466,11 @@ export const handler = async (event: { channelId: string }) => { } finally { if (channel) { try { - channel.publish({ - direction: 'btoc', - event: 'end', + dispatchEvent(channel, 'end').then(() => { + if (channel) { + channel.close(); + } }); - channel.close(); } catch (e) { console.error(e); } diff --git a/packages/types/src/index.d.ts b/packages/types/src/index.d.ts index dab578640..80e9cff45 100644 --- a/packages/types/src/index.d.ts +++ b/packages/types/src/index.d.ts @@ -15,3 +15,4 @@ export * from './model'; export * from './rag-knowledge-base'; export * from './useCases'; export * from './share'; +export * from './speech-to-speech'; diff --git a/packages/types/src/speech-to-speech.d.ts b/packages/types/src/speech-to-speech.d.ts new file mode 100644 index 000000000..d28aa1e85 --- /dev/null +++ b/packages/types/src/speech-to-speech.d.ts @@ -0,0 +1,22 @@ +// ctob: client (web) to bedrock (api) +// btoc: bedrock (api) to client (web) +export type SpeechToSpeechEventDirection = 'ctob' | 'btoc'; + +export type SpeechToSpeechEventType = + 'ready' | + 'end' | + 'promptStart' | + 'systemPrompt' | + 'audioStart' | + 'audioInput' | + 'audioStop' | + 'audioOutput' | + 'textStart' | + 'textOutput' | + 'textStop'; + +export type SpeechToSpeechEvent = { + direction: SpeechToSpeechEventDirection; + event: SpeechToSpeechEventType; + data: any; +}; diff --git a/packages/web/src/components/ChatMessage.tsx b/packages/web/src/components/ChatMessage.tsx index bdab0f976..04138b6c7 100644 --- a/packages/web/src/components/ChatMessage.tsx +++ b/packages/web/src/components/ChatMessage.tsx @@ -27,6 +27,7 @@ type Props = BaseProps & { chatContent?: ShownMessage; loading?: boolean; hideFeedback?: boolean; + hideSaveSystemContext?: boolean; setSaveSystemContext?: (s: string) => void; setShowSystemContextModal?: (value: boolean) => void; allowRetry?: boolean; @@ -139,6 +140,7 @@ const ChatMessage: React.FC = (props) => { ? 'bg-gray-100/70' : '' }`}> +
= (props) => {
- {chatContent?.role === 'system' && ( + {chatContent?.role === 'system' && !props.hideSaveSystemContext && ( { diff --git a/packages/web/src/hooks/useNovaSonic/AudioPlayer.js b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayer.js similarity index 100% rename from packages/web/src/hooks/useNovaSonic/AudioPlayer.js rename to packages/web/src/hooks/useSpeechToSpeech/AudioPlayer.js diff --git a/packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js similarity index 100% rename from packages/web/src/hooks/useNovaSonic/AudioPlayerProcessor.worklet.js rename to packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js diff --git a/packages/web/src/hooks/useNovaSonic/ObjectsExt.js b/packages/web/src/hooks/useSpeechToSpeech/ObjectsExt.js similarity index 100% rename from packages/web/src/hooks/useNovaSonic/ObjectsExt.js rename to packages/web/src/hooks/useSpeechToSpeech/ObjectsExt.js diff --git a/packages/web/src/hooks/useNovaSonic/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts similarity index 80% rename from packages/web/src/hooks/useNovaSonic/index.ts rename to packages/web/src/hooks/useSpeechToSpeech/index.ts index 1bf7b9191..1e05327ff 100644 --- a/packages/web/src/hooks/useNovaSonic/index.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts @@ -3,6 +3,11 @@ import { events, EventsChannel } from 'aws-amplify/data'; import { AudioPlayer } from './AudioPlayer'; import { v4 as uuid } from 'uuid'; import useHttp from '../../hooks/useHttp'; +import useChatHistory from './useChatHistory'; +import { + SpeechToSpeechEventType, + SpeechToSpeechEvent, +} from 'generative-ai-use-cases'; const NAMESPACE = import.meta.env.VITE_APP_SPEECH_TO_SPEECH_NAMESPACE!; const MIN_AUDIO_CHUNKS_PER_BATCH = 10; @@ -46,25 +51,27 @@ const base64ToFloat32Array = (base64String: string) => { } }; -export const useNovaSonic = () => { +export const useSpeechToSpeech = () => { const api = useHttp(); + const { clear, messages, setupSystemPrompt, onTextOutput, onTextStop } = useChatHistory(); const [isActive, setIsActive] = useState(false); const [isLoading, setIsLoading] = useState(false); + const systemPromptRef = useRef(''); const audioPlayerRef = useRef(null); const channelRef = useRef(null); - const audioContextRef = useRef(null); - const audioStreamRef = useRef(null); - const sourceNodeRef = useRef(null); - const processorRef = useRef(null); + const audioContextRef = useRef(null); + const audioStreamRef = useRef(null); + const sourceNodeRef = useRef(null); + const processorRef = useRef(null); const audioInputQueue = useRef([]); - const dispatchEvent = async (event: string, data: any = undefined) => { + const dispatchEvent = async (event: SpeechToSpeechEventType, data: any = undefined) => { if (channelRef.current) { await channelRef.current.publish({ - direction: 'ctob', // client to bedrock + direction: 'ctob', event, data, - }); + } as SpeechToSpeechEvent); } }; @@ -145,6 +152,14 @@ export const useNovaSonic = () => { audioPlayerRef.current.playAudio(audioData); } } + } else if (event.event === 'textStart') { + console.log('textStart', event.data); + } else if (event.event === 'textOutput') { + console.log('textOutput', event.data); + onTextOutput(event.data); + } else if (event.event === 'textStop') { + console.log('textStop', event.data); + onTextStop(event.data); } } }, @@ -153,14 +168,20 @@ export const useNovaSonic = () => { }, }); - await api.post(`speech-to-speech`, { channel: channelId }); + await api.post('speech-to-speech', { channel: channelId }); }; const startRecording = async () => { + if (!audioContextRef.current || !audioStreamRef.current || !systemPromptRef.current) { + return; + } + await dispatchEvent('promptStart'); - await dispatchEvent('systemPrompt', 'You are an AI assistant'); + await dispatchEvent('systemPrompt', systemPromptRef.current); await dispatchEvent('audioStart'); + setupSystemPrompt(systemPromptRef.current); + const sourceNode = audioContextRef.current.createMediaStreamSource(audioStreamRef.current); if (audioContextRef.current.createScriptProcessor) { @@ -216,13 +237,17 @@ export const useNovaSonic = () => { await dispatchEvent('audioStop'); }; - const startSession = async () => { + const startSession = async (systemPrompt: string) => { if (isActive || isLoading) { return; } + clear(); + setIsLoading(true); + systemPromptRef.current = systemPrompt; + await connectToAppSync(); await initAudio(); }; @@ -235,6 +260,7 @@ export const useNovaSonic = () => { }; return { + messages, isActive, isLoading, startSession, diff --git a/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts new file mode 100644 index 000000000..e5e82684d --- /dev/null +++ b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts @@ -0,0 +1,110 @@ +import { useState, useRef } from 'react'; +import { UnrecordedMessage } from 'generative-ai-use-cases'; + +type SpeechToSpeechMessage = UnrecordedMessage & { isPartial: boolean }; + +// stopReason が INTERRUPTED だったら content は空 +// そこから先のそのロールの発言は無視される (基本的にアシスタントの発言と思われる) +// ユーザーの発言が到着したら、interrupted を解除 + +const useChatHistory = () => { + const [messages, setMessages] = useState([]); + const messageCache = useRef>({}); + const stopReasonCache = useRef>({}); + const interrupted = useRef(false); + + const clear = () => { + setMessages([]); + messageCache.current = {}; + } + + const setupSystemPrompt = (prompt: string) => { + setMessages([{ + role: 'system', + content: prompt, + isPartial: false, + }]); + }; + + const updateMessages = (newMessage: SpeechToSpeechMessage) => { + if (interrupted.current) { + if (newMessage.role === 'assistant') { + return; + } else { + interrupted.current = false; + } + } + + setMessages((messages) => { + const lastMessageIndex = messages.length - 1; + const lastMessage = messages[lastMessageIndex]; + const messagesWithoutLast = messages.slice(0, lastMessageIndex); + + if (lastMessage) { + if (lastMessage.role !== newMessage.role) { + return [...messagesWithoutLast, lastMessage, newMessage]; + } else { + if (lastMessage.isPartial && !newMessage.isPartial) { + return [...messagesWithoutLast, newMessage]; + } else { + const updatedLastMessage = { + ...lastMessage, + content: lastMessage.content + ' ' + newMessage.content, + isPartial: newMessage.isPartial, + }; + return [...messagesWithoutLast, updatedLastMessage]; + } + } + } + + if (lastMessage) { + return [...messagesWithoutLast, lastMessage]; + } else { + return [...messagesWithoutLast]; + } + }); + }; + + const onTextOutput = (data: { id: string, role: string, content: string}) => { + if (stopReasonCache.current[data.id]) { + const newMessage: SpeechToSpeechMessage = { + role: data.role as 'user' | 'assistant', + content: data.content, + isPartial: stopReasonCache.current[data.id] === 'PARTIAL_TURN', + }; + + updateMessages(newMessage); + } else { + messageCache.current[data.id] = { role: data.role as 'user' | 'assistant', content: data.content }; + } + }; + + const onTextStop = (data: { id: string, stopReason: string }) => { + if (data.stopReason === 'INTERRUPTED') { + interrupted.current = true; + return; + } + + if (messageCache.current[data.id]) { + const newMessage: SpeechToSpeechMessage = { + role: messageCache.current[data.id].role, + content: messageCache.current[data.id].content, + isPartial: data.stopReason === 'PARTIAL_TURN', + }; + + updateMessages(newMessage); + } else { + stopReasonCache.current[data.id] = data.stopReason; + } + }; + + return { + clear, + messages, + setupSystemPrompt, + onTextOutput, + onTextStop, + }; +} + +export default useChatHistory; diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index 241dfd903..0c8d1c471 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -1,35 +1,119 @@ -import React from 'react'; -import { useNovaSonic } from '../hooks/useNovaSonic'; +import React, { useState, useMemo } from 'react'; +import { useSpeechToSpeech } from '../hooks/useSpeechToSpeech'; +import { useTranslation } from 'react-i18next'; +import { PiArrowClockwiseBold, PiStopCircleBold, PiMicrophoneBold } from 'react-icons/pi'; +import ChatMessage from '../components/ChatMessage'; +import Switch from '../components/Switch'; +import ExpandableField from '../components/ExpandableField'; +import Button from '../components/Button'; +import InputChatContent from '../components/InputChatContent'; const SpeechToSpeech: React.FC = () => { + const { t } = useTranslation(); const { + messages, isActive, isLoading, startSession, closeSession, - } = useNovaSonic(); + } = useSpeechToSpeech(); + const [showSystemPrompt, setShowSystemPrompt] = useState(false); + // TODO: avoid hardcoding + const [systemPrompt, setSystemPrompt] = useState('You are an AI assistant.'); + const [inputSystemPrompt, setInputSystemPrompt] = useState(systemPrompt); - return ( -
-

Speech To Speech

-
- isActive: {isActive.toString()} + const messagesWithoutSystemPrompt = useMemo(() => { + return messages.filter(m => m.role !== 'system'); + }, [messages]); + + const showingMessages = useMemo(() => { + if (showSystemPrompt) { + return messages; + } else { + return messagesWithoutSystemPrompt + } + }, [messages, messagesWithoutSystemPrompt, showSystemPrompt]); + + const isEmpty = useMemo(() => { + return messagesWithoutSystemPrompt.length === 0; + }, [messagesWithoutSystemPrompt]); + + return (<> +
+
+ Speech to Speech (beta)
-
- isLoading: {isLoading.toString()} + +
+
+
- + {showingMessages.map((m, idx) => { + return ( +
+ +
+ ) + })}
-
- + +
+ {!isLoading && !isActive && ( + + <> +
+ +
+ + } + onSend={() => { + setSystemPrompt(inputSystemPrompt); + }} + /> + + + )} + + {!isActive ? ( + + ) : ( + + )}
- ); + ); }; export default SpeechToSpeech; From 50c69c2ac39ad1e008afa0dac0e7c61f2d45a1fb Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Wed, 16 Apr 2025 10:58:17 +0900 Subject: [PATCH 08/20] fix --- packages/cdk/lambda/speechToSpeechTask.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index 57c40408e..8f6702181 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -81,6 +81,7 @@ const enqueuePromptStart = () => { sampleRateHertz: 24000, sampleSizeBits: 16, channelCount: 1, + // TODO: avoid hardcoding voiceId: "tiffany", } } From 2d53b193c09e7d5c1615a17a9942d8177dc3f9c6 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Wed, 16 Apr 2025 15:57:05 +0900 Subject: [PATCH 09/20] fix --- package-lock.json | 398 +++++++++--------- packages/cdk/lambda/speechToSpeechTask.ts | 90 ++-- .../web/src/hooks/useSpeechToSpeech/index.ts | 5 +- .../hooks/useSpeechToSpeech/useChatHistory.ts | 110 +++-- packages/web/src/pages/SpeechToSpeechPage.tsx | 47 ++- 5 files changed, 334 insertions(+), 316 deletions(-) diff --git a/package-lock.json b/package-lock.json index c12e80353..056815b20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -228,9 +228,9 @@ } }, "node_modules/@aws-amplify/data-schema": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/@aws-amplify/data-schema/-/data-schema-1.20.3.tgz", - "integrity": "sha512-Q9Vsh+FYmS9QqPCwWW6ciPX2m0IkC+PR/sQl26PN4zojPcUFP9z8ilAFqISkB8/WkqIHZ0XIrw8fuAyCv3w1dA==", + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/@aws-amplify/data-schema/-/data-schema-1.20.4.tgz", + "integrity": "sha512-YQ6KMvBnAAEQFYobR1pZjLr1QXlpgJGfEoSMfIMEjtrOiLiJmZEFJauWjO1Z7kUN/eaHldVcaysJ1IcSM9y3Sg==", "license": "Apache-2.0", "dependencies": { "@aws-amplify/data-schema-types": "*", @@ -485,9 +485,9 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.231", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.231.tgz", - "integrity": "sha512-vPqD/K2pK/ALhU5r5Nafdc2nLB+LJKxNyxUmQnLsazU6AWDJfkqjHQx8m3J4Cjl2C3chQkIRMdzSDuXIlo43GA==", + "version": "2.2.232", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.232.tgz", + "integrity": "sha512-x9aFQG9gA+RgGj9bGB+WC6y1Nq2/Y8R2yXFoKWoQZOet8PRFJ8M5/FeXoh9XmdWI4weJVctLU4WTIve6rOvPtA==", "license": "Apache-2.0" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { @@ -965,9 +965,9 @@ } }, "node_modules/@aws-sdk/client-dynamodb": { - "version": "3.787.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.787.0.tgz", - "integrity": "sha512-mqR+MrO/Jef9fHU0QH4jDW5zJ3RtHUnyt6OVwO0ezQD5hWajIwT8fl2Bp0zqakhkhUAiW7FpGAiF+G+kEqb/rQ==", + "version": "3.788.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.788.0.tgz", + "integrity": "sha512-agm2+5p47uDbAcWRJjRqsIIpbjig5P0AMB/1lQjkQKTnGOfdgzIrFLtQ3tO0kWWbTb7gH6Pg5EviuTC8j6axLQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -7054,13 +7054,13 @@ } }, "node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.787.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.787.0.tgz", - "integrity": "sha512-961/DbHJMh9EC+4nI1l6/81WkRoUtHcFa5T7PbJ41hWPge5IMy+JwDZwpF/e77kvws2loqATEZ0FRa0APgaN4A==", + "version": "3.789.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.789.0.tgz", + "integrity": "sha512-ut3d1vaHLxZCh2rc6MwghTz4DxPPoCDP0GjRUsiysG3IHx97akUt2XwPrjNkot0/Hg4JJUZzl95QCjaIf/pUxg==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.775.0", - "@aws-sdk/util-dynamodb": "3.787.0", + "@aws-sdk/util-dynamodb": "3.788.0", "@smithy/core": "^3.2.0", "@smithy/smithy-client": "^4.2.0", "@smithy/types": "^4.2.0", @@ -7070,7 +7070,7 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.787.0" + "@aws-sdk/client-dynamodb": "^3.788.0" } }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { @@ -7772,9 +7772,9 @@ } }, "node_modules/@aws-sdk/util-dynamodb": { - "version": "3.787.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.787.0.tgz", - "integrity": "sha512-wPccKwwNs36sGoE3GuyJebQentnMnQqf5EaTonqFuC+sfUqp49SwF0waExYKQihp64i0mYpl7UjLL7ALUn9vWw==", + "version": "3.788.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.788.0.tgz", + "integrity": "sha512-+7M+Lr5oroXtY3SDa4nvB1OSI6+zqjiNGbVKFdgPonArLTGsNYP5lWh5y5GksFE8A19aZun4yUSrny+pbEarhw==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -7783,7 +7783,7 @@ "node": ">=18.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.787.0" + "@aws-sdk/client-dynamodb": "^3.788.0" } }, "node_modules/@aws-sdk/util-endpoints": { @@ -10297,9 +10297,9 @@ } }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.5.1.tgz", - "integrity": "sha512-soEIOALTfTK6EjmKMMoLugwaP0rzkad90iIWd1hMO9ARkSAyjfMfkRRhLvD5qH7vvM0Cg72pieUfR6yh6XxC4w==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.6.0.tgz", + "integrity": "sha512-WhCn7Z7TauhBtmzhvKpoQs0Wwb/kBcy4CwpuI0/eEIr2Lx2auxmulAzLr91wVZJaz47iUZdkXOK7WlAfxGKCnA==", "dev": true, "license": "MIT", "dependencies": { @@ -11508,16 +11508,16 @@ } }, "node_modules/@pkgr/core": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.2.tgz", - "integrity": "sha512-25L86MyPvnlQoX2MTIV2OiUcb6vJ6aRbFa9pbwByn95INKD5mFH2smgjDhq+fwJoqAgvgbdJLj6Tz7V9X5CFAQ==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.4.tgz", + "integrity": "sha512-ROFF39F6ZrnzSUEmQQZUar0Jt4xVoP9WnDRdWwF4NNcXs3xBTLgBUDoOwW141y1jP+S8nahIbdxbFC7IShw9Iw==", "dev": true, "license": "MIT", "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, "funding": { - "url": "https://opencollective.com/unts" + "url": "https://opencollective.com/pkgr" } }, "node_modules/@popperjs/core": { @@ -12469,9 +12469,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.39.0.tgz", - "integrity": "sha512-lGVys55Qb00Wvh8DMAocp5kIcaNzEFTmGhfFd88LfaogYTRKrdxgtlO5H6S49v2Nd8R2C6wLOal0qv6/kCkOwA==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.40.0.tgz", + "integrity": "sha512-+Fbls/diZ0RDerhE8kyC6hjADCXA1K4yVNlH0EYfd2XjyH0UGgzaQ8MlT0pCXAThfxv3QUAczHaL+qSv1E4/Cg==", "cpu": [ "arm" ], @@ -12482,9 +12482,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.39.0.tgz", - "integrity": "sha512-It9+M1zE31KWfqh/0cJLrrsCPiF72PoJjIChLX+rEcujVRCb4NLQ5QzFkzIZW8Kn8FTbvGQBY5TkKBau3S8cCQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.40.0.tgz", + "integrity": "sha512-PPA6aEEsTPRz+/4xxAmaoWDqh67N7wFbgFUJGMnanCFs0TV99M0M8QhhaSCks+n6EbQoFvLQgYOGXxlMGQe/6w==", "cpu": [ "arm64" ], @@ -12495,9 +12495,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.39.0.tgz", - "integrity": "sha512-lXQnhpFDOKDXiGxsU9/l8UEGGM65comrQuZ+lDcGUx+9YQ9dKpF3rSEGepyeR5AHZ0b5RgiligsBhWZfSSQh8Q==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.40.0.tgz", + "integrity": "sha512-GwYOcOakYHdfnjjKwqpTGgn5a6cUX7+Ra2HeNj/GdXvO2VJOOXCiYYlRFU4CubFM67EhbmzLOmACKEfvp3J1kQ==", "cpu": [ "arm64" ], @@ -12508,9 +12508,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.39.0.tgz", - "integrity": "sha512-mKXpNZLvtEbgu6WCkNij7CGycdw9cJi2k9v0noMb++Vab12GZjFgUXD69ilAbBh034Zwn95c2PNSz9xM7KYEAQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.40.0.tgz", + "integrity": "sha512-CoLEGJ+2eheqD9KBSxmma6ld01czS52Iw0e2qMZNpPDlf7Z9mj8xmMemxEucinev4LgHalDPczMyxzbq+Q+EtA==", "cpu": [ "x64" ], @@ -12521,9 +12521,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.39.0.tgz", - "integrity": "sha512-jivRRlh2Lod/KvDZx2zUR+I4iBfHcu2V/BA2vasUtdtTN2Uk3jfcZczLa81ESHZHPHy4ih3T/W5rPFZ/hX7RtQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.40.0.tgz", + "integrity": "sha512-r7yGiS4HN/kibvESzmrOB/PxKMhPTlz+FcGvoUIKYoTyGd5toHp48g1uZy1o1xQvybwwpqpe010JrcGG2s5nkg==", "cpu": [ "arm64" ], @@ -12534,9 +12534,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.39.0.tgz", - "integrity": "sha512-8RXIWvYIRK9nO+bhVz8DwLBepcptw633gv/QT4015CpJ0Ht8punmoHU/DuEd3iw9Hr8UwUV+t+VNNuZIWYeY7Q==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.40.0.tgz", + "integrity": "sha512-mVDxzlf0oLzV3oZOr0SMJ0lSDd3xC4CmnWJ8Val8isp9jRGl5Dq//LLDSPFrasS7pSm6m5xAcKaw3sHXhBjoRw==", "cpu": [ "x64" ], @@ -12547,9 +12547,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.39.0.tgz", - "integrity": "sha512-mz5POx5Zu58f2xAG5RaRRhp3IZDK7zXGk5sdEDj4o96HeaXhlUwmLFzNlc4hCQi5sGdR12VDgEUqVSHer0lI9g==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.40.0.tgz", + "integrity": "sha512-y/qUMOpJxBMy8xCXD++jeu8t7kzjlOCkoxxajL58G62PJGBZVl/Gwpm7JK9+YvlB701rcQTzjUZ1JgUoPTnoQA==", "cpu": [ "arm" ], @@ -12560,9 +12560,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.39.0.tgz", - "integrity": "sha512-+YDwhM6gUAyakl0CD+bMFpdmwIoRDzZYaTWV3SDRBGkMU/VpIBYXXEvkEcTagw/7VVkL2vA29zU4UVy1mP0/Yw==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.40.0.tgz", + "integrity": "sha512-GoCsPibtVdJFPv/BOIvBKO/XmwZLwaNWdyD8TKlXuqp0veo2sHE+A/vpMQ5iSArRUz/uaoj4h5S6Pn0+PdhRjg==", "cpu": [ "arm" ], @@ -12573,9 +12573,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.39.0.tgz", - "integrity": "sha512-EKf7iF7aK36eEChvlgxGnk7pdJfzfQbNvGV/+l98iiMwU23MwvmV0Ty3pJ0p5WQfm3JRHOytSIqD9LB7Bq7xdQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.40.0.tgz", + "integrity": "sha512-L5ZLphTjjAD9leJzSLI7rr8fNqJMlGDKlazW2tX4IUF9P7R5TMQPElpH82Q7eNIDQnQlAyiNVfRPfP2vM5Avvg==", "cpu": [ "arm64" ], @@ -12586,9 +12586,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.39.0.tgz", - "integrity": "sha512-vYanR6MtqC7Z2SNr8gzVnzUul09Wi1kZqJaek3KcIlI/wq5Xtq4ZPIZ0Mr/st/sv/NnaPwy/D4yXg5x0B3aUUA==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.40.0.tgz", + "integrity": "sha512-ATZvCRGCDtv1Y4gpDIXsS+wfFeFuLwVxyUBSLawjgXK2tRE6fnsQEkE4csQQYWlBlsFztRzCnBvWVfcae/1qxQ==", "cpu": [ "arm64" ], @@ -12599,9 +12599,9 @@ ] }, "node_modules/@rollup/rollup-linux-loongarch64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.39.0.tgz", - "integrity": "sha512-NMRUT40+h0FBa5fb+cpxtZoGAggRem16ocVKIv5gDB5uLDgBIwrIsXlGqYbLwW8YyO3WVTk1FkFDjMETYlDqiw==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.40.0.tgz", + "integrity": "sha512-wG9e2XtIhd++QugU5MD9i7OnpaVb08ji3P1y/hNbxrQ3sYEelKJOq1UJ5dXczeo6Hj2rfDEL5GdtkMSVLa/AOg==", "cpu": [ "loong64" ], @@ -12612,9 +12612,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.39.0.tgz", - "integrity": "sha512-0pCNnmxgduJ3YRt+D+kJ6Ai/r+TaePu9ZLENl+ZDV/CdVczXl95CbIiwwswu4L+K7uOIGf6tMo2vm8uadRaICQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.40.0.tgz", + "integrity": "sha512-vgXfWmj0f3jAUvC7TZSU/m/cOE558ILWDzS7jBhiCAFpY2WEBn5jqgbqvmzlMjtp8KlLcBlXVD2mkTSEQE6Ixw==", "cpu": [ "ppc64" ], @@ -12625,9 +12625,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.39.0.tgz", - "integrity": "sha512-t7j5Zhr7S4bBtksT73bO6c3Qa2AV/HqiGlj9+KB3gNF5upcVkx+HLgxTm8DK4OkzsOYqbdqbLKwvGMhylJCPhQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.40.0.tgz", + "integrity": "sha512-uJkYTugqtPZBS3Z136arevt/FsKTF/J9dEMTX/cwR7lsAW4bShzI2R0pJVw+hcBTWF4dxVckYh72Hk3/hWNKvA==", "cpu": [ "riscv64" ], @@ -12638,9 +12638,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.39.0.tgz", - "integrity": "sha512-m6cwI86IvQ7M93MQ2RF5SP8tUjD39Y7rjb1qjHgYh28uAPVU8+k/xYWvxRO3/tBN2pZkSMa5RjnPuUIbrwVxeA==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.40.0.tgz", + "integrity": "sha512-rKmSj6EXQRnhSkE22+WvrqOqRtk733x3p5sWpZilhmjnkHkpeCgWsFFo0dGnUGeA+OZjRl3+VYq+HyCOEuwcxQ==", "cpu": [ "riscv64" ], @@ -12651,9 +12651,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.39.0.tgz", - "integrity": "sha512-iRDJd2ebMunnk2rsSBYlsptCyuINvxUfGwOUldjv5M4tpa93K8tFMeYGpNk2+Nxl+OBJnBzy2/JCscGeO507kA==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.40.0.tgz", + "integrity": "sha512-SpnYlAfKPOoVsQqmTFJ0usx0z84bzGOS9anAC0AZ3rdSo3snecihbhFTlJZ8XMwzqAcodjFU4+/SM311dqE5Sw==", "cpu": [ "s390x" ], @@ -12664,9 +12664,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.39.0.tgz", - "integrity": "sha512-t9jqYw27R6Lx0XKfEFe5vUeEJ5pF3SGIM6gTfONSMb7DuG6z6wfj2yjcoZxHg129veTqU7+wOhY6GX8wmf90dA==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.40.0.tgz", + "integrity": "sha512-RcDGMtqF9EFN8i2RYN2W+64CdHruJ5rPqrlYw+cgM3uOVPSsnAQps7cpjXe9be/yDp8UC7VLoCoKC8J3Kn2FkQ==", "cpu": [ "x64" ], @@ -12677,9 +12677,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.39.0.tgz", - "integrity": "sha512-ThFdkrFDP55AIsIZDKSBWEt/JcWlCzydbZHinZ0F/r1h83qbGeenCt/G/wG2O0reuENDD2tawfAj2s8VK7Bugg==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.40.0.tgz", + "integrity": "sha512-HZvjpiUmSNx5zFgwtQAV1GaGazT2RWvqeDi0hV+AtC8unqqDSsaFjPxfsO6qPtKRRg25SisACWnJ37Yio8ttaw==", "cpu": [ "x64" ], @@ -12690,9 +12690,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.39.0.tgz", - "integrity": "sha512-jDrLm6yUtbOg2TYB3sBF3acUnAwsIksEYjLeHL+TJv9jg+TmTwdyjnDex27jqEMakNKf3RwwPahDIt7QXCSqRQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.40.0.tgz", + "integrity": "sha512-UtZQQI5k/b8d7d3i9AZmA/t+Q4tk3hOC0tMOMSq2GlMYOfxbesxG4mJSeDp0EHs30N9bsfwUvs3zF4v/RzOeTQ==", "cpu": [ "arm64" ], @@ -12703,9 +12703,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.39.0.tgz", - "integrity": "sha512-6w9uMuza+LbLCVoNKL5FSLE7yvYkq9laSd09bwS0tMjkwXrmib/4KmoJcrKhLWHvw19mwU+33ndC69T7weNNjQ==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.40.0.tgz", + "integrity": "sha512-+m03kvI2f5syIqHXCZLPVYplP8pQch9JHyXKZ3AGMKlg8dCyr2PKHjwRLiW53LTrN/Nc3EqHOKxUxzoSPdKddA==", "cpu": [ "ia32" ], @@ -12716,9 +12716,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.39.0.tgz", - "integrity": "sha512-yAkUOkIKZlK5dl7u6dg897doBgLXmUHhIINM2c+sND3DZwnrdQkkSiDh7N75Ll4mM4dxSkYfXqU9fW3lLkMFug==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.40.0.tgz", + "integrity": "sha512-lpPE1cLfP5oPzVjKMx10pgBmKELQnFJXHgvtHCtuJWOv8MxqdEIMNtgHgBFf7Ea2/7EuVwa9fodWUfXAlXZLZQ==", "cpu": [ "x64" ], @@ -13971,9 +13971,9 @@ } }, "node_modules/@swc/helpers": { - "version": "0.5.15", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", - "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "version": "0.5.17", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", + "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.8.0" @@ -16331,9 +16331,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.189.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.189.0.tgz", - "integrity": "sha512-B5Uha7uRntOAyuKfU0eFtxij3ZVTzGAbetw5qaXlURa68wsWpKlU72/OyKugB6JYkhjCZkSTVVBxd1pVTosxEw==", + "version": "2.189.1", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.189.1.tgz", + "integrity": "sha512-9JU0yUr2iRTJ1oCPrHyx7hOtBDWyUfyOcdb6arlumJnMcQr2cyAMASY8HuAXHc8Y10ipVp8dRTW+J4/132IIYA==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -18186,9 +18186,9 @@ } }, "node_modules/css-selector-parser": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.1.1.tgz", - "integrity": "sha512-Y+DuvJ7JAjpL1f4DeILe5sXCC3kRXMl0DxM4lAWbS8/jEZ29o3V0L5TL6zIifj4Csmj6c+jiF2ENjida2OVOGA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.1.2.tgz", + "integrity": "sha512-WfUcL99xWDs7b3eZPoRszWVfbNo8ErCF15PTvVROjkShGlAfjIkG6hlfj/sl6/rfo5Q9x9ryJ3VqVnAZDA+gcw==", "funding": [ { "type": "github", @@ -18965,9 +18965,9 @@ } }, "node_modules/deploy-time-build": { - "version": "0.3.25", - "resolved": "https://registry.npmjs.org/deploy-time-build/-/deploy-time-build-0.3.25.tgz", - "integrity": "sha512-to7sWVtvTNyRZEtJ+/ed5ZUhObvdnPhzdifvHDKRPYaxLoLZQZu34DetqSGwm+A5UOXVku+6bbEkhQOF4BpfZQ==", + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/deploy-time-build/-/deploy-time-build-0.3.29.tgz", + "integrity": "sha512-U7S7izP56fFr45WLduQpa9/A9rVLnstu38YtLUsXdynkLBKikSLzWCmsMyP8Wa54HEmjOuplPbwRJi58lcAIDQ==", "license": "MIT", "peerDependencies": { "aws-cdk-lib": "^2.38.0", @@ -19259,9 +19259,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.136", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.136.tgz", - "integrity": "sha512-kL4+wUTD7RSA5FHx5YwWtjDnEEkIIikFgWHR4P6fqjw1PPLlqYkxeOb++wAauAssat0YClCy8Y3C5SxgSkjibQ==", + "version": "1.5.137", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.137.tgz", + "integrity": "sha512-/QSJaU2JyIuTbbABAo/crOs+SuAZLS+fVVS10PVrIT9hrRkmZl8Hb0xPSkKRUUWHQtYzXHpQUW3Dy5hwMzGZkA==", "license": "ISC" }, "node_modules/elliptic": { @@ -22114,7 +22114,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -24834,9 +24833,9 @@ } }, "node_modules/jotai": { - "version": "2.12.2", - "resolved": "https://registry.npmjs.org/jotai/-/jotai-2.12.2.tgz", - "integrity": "sha512-oN8715y7MkjXlSrpyjlR887TOuc/NLZMs9gvgtfWH/JP47ChwO0lR2ijSwBvPMYyXRAPT+liIAhuBavluKGgtA==", + "version": "2.12.3", + "resolved": "https://registry.npmjs.org/jotai/-/jotai-2.12.3.tgz", + "integrity": "sha512-DpoddSkmPGXMFtdfnoIHfueFeGP643nqYUWC6REjUcME+PG2UkAtYnLbffRDw3OURI9ZUTcRWkRGLsOvxuWMCg==", "license": "MIT", "engines": { "node": ">=12.20.0" @@ -26808,9 +26807,9 @@ } }, "node_modules/novel/node_modules/@types/node": { - "version": "22.14.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.0.tgz", - "integrity": "sha512-Kmpl+z84ILoG+3T/zQFyAJsU6EPTmOCj8/2+83fSN6djd6I4o7uOuGIH6vq3PrjY5BGitSbFuMN18j3iknubbA==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -27982,9 +27981,9 @@ } }, "node_modules/prosemirror-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/prosemirror-commands/-/prosemirror-commands-1.7.0.tgz", - "integrity": "sha512-6toodS4R/Aah5pdsrIwnTYPEjW70SlO5a66oo5Kk+CIrgJz3ukOoS+FYDGqvQlAX5PxoGWDX1oD++tn5X3pyRA==", + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/prosemirror-commands/-/prosemirror-commands-1.7.1.tgz", + "integrity": "sha512-rT7qZnQtx5c0/y/KlYaGvtG411S97UaL6gdp6RIZ23DLHanMYLyfGBV5DtSnZdthQql7W+lEVbpSfwtO8T+L2w==", "license": "MIT", "dependencies": { "prosemirror-model": "^1.0.0", @@ -28111,16 +28110,16 @@ } }, "node_modules/prosemirror-tables": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.6.4.tgz", - "integrity": "sha512-TkDY3Gw52gRFRfRn2f4wJv5WOgAOXLJA2CQJYIJ5+kdFbfj3acR4JUW6LX2e1hiEBiUwvEhzH5a3cZ5YSztpIA==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.7.0.tgz", + "integrity": "sha512-dc9+u4aqT+biw3J/v7p5LyH8uqqXSAjdszfLtrCnDbpMr1F+Gsjtkdiij/1p8qM1gBOCfQeiahhk2pOO9Aa8xA==", "license": "MIT", "dependencies": { "prosemirror-keymap": "^1.2.2", - "prosemirror-model": "^1.24.1", + "prosemirror-model": "^1.25.0", "prosemirror-state": "^1.4.3", - "prosemirror-transform": "^1.10.2", - "prosemirror-view": "^1.37.2" + "prosemirror-transform": "^1.10.3", + "prosemirror-view": "^1.39.1" } }, "node_modules/prosemirror-trailing-node": { @@ -29605,9 +29604,9 @@ "license": "Unlicense" }, "node_modules/rollup": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.39.0.tgz", - "integrity": "sha512-thI8kNc02yNvnmJp8dr3fNWJ9tCONDhp6TV35X6HkKGGs9E6q7YWCHbe5vKiTa7TAiNcFEmXKj3X/pG2b3ci0g==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.40.0.tgz", + "integrity": "sha512-Noe455xmA96nnqH5piFtLobsGbCij7Tu+tb3c1vYjNbTkfzGqXqQXG3wJaYXkRZuQ0vEYN4bhwg7QnIrqB5B+w==", "license": "MIT", "dependencies": { "@types/estree": "1.0.7" @@ -29620,26 +29619,26 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.39.0", - "@rollup/rollup-android-arm64": "4.39.0", - "@rollup/rollup-darwin-arm64": "4.39.0", - "@rollup/rollup-darwin-x64": "4.39.0", - "@rollup/rollup-freebsd-arm64": "4.39.0", - "@rollup/rollup-freebsd-x64": "4.39.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.39.0", - "@rollup/rollup-linux-arm-musleabihf": "4.39.0", - "@rollup/rollup-linux-arm64-gnu": "4.39.0", - "@rollup/rollup-linux-arm64-musl": "4.39.0", - "@rollup/rollup-linux-loongarch64-gnu": "4.39.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.39.0", - "@rollup/rollup-linux-riscv64-gnu": "4.39.0", - "@rollup/rollup-linux-riscv64-musl": "4.39.0", - "@rollup/rollup-linux-s390x-gnu": "4.39.0", - "@rollup/rollup-linux-x64-gnu": "4.39.0", - "@rollup/rollup-linux-x64-musl": "4.39.0", - "@rollup/rollup-win32-arm64-msvc": "4.39.0", - "@rollup/rollup-win32-ia32-msvc": "4.39.0", - "@rollup/rollup-win32-x64-msvc": "4.39.0", + "@rollup/rollup-android-arm-eabi": "4.40.0", + "@rollup/rollup-android-arm64": "4.40.0", + "@rollup/rollup-darwin-arm64": "4.40.0", + "@rollup/rollup-darwin-x64": "4.40.0", + "@rollup/rollup-freebsd-arm64": "4.40.0", + "@rollup/rollup-freebsd-x64": "4.40.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.40.0", + "@rollup/rollup-linux-arm-musleabihf": "4.40.0", + "@rollup/rollup-linux-arm64-gnu": "4.40.0", + "@rollup/rollup-linux-arm64-musl": "4.40.0", + "@rollup/rollup-linux-loongarch64-gnu": "4.40.0", + "@rollup/rollup-linux-powerpc64le-gnu": "4.40.0", + "@rollup/rollup-linux-riscv64-gnu": "4.40.0", + "@rollup/rollup-linux-riscv64-musl": "4.40.0", + "@rollup/rollup-linux-s390x-gnu": "4.40.0", + "@rollup/rollup-linux-x64-gnu": "4.40.0", + "@rollup/rollup-linux-x64-musl": "4.40.0", + "@rollup/rollup-win32-arm64-msvc": "4.40.0", + "@rollup/rollup-win32-ia32-msvc": "4.40.0", + "@rollup/rollup-win32-x64-msvc": "4.40.0", "fsevents": "~2.3.2" } }, @@ -29888,7 +29887,6 @@ "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver" @@ -30782,13 +30780,13 @@ } }, "node_modules/synckit": { - "version": "0.11.3", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.3.tgz", - "integrity": "sha512-szhWDqNNI9etJUvbZ1/cx1StnZx8yMmFxme48SwR4dty4ioSY50KEZlpv0qAfgc1fpRzuh9hBXEzoCpJ779dLg==", + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.4.tgz", + "integrity": "sha512-Q/XQKRaJiLiFIBNN+mndW7S/RHxvwzuZS6ZwmRzUBqJBv/5QIKCEwkBC8GBf8EQJKYnaFs0wOZbKTXBPj8L9oQ==", "dev": true, "license": "MIT", "dependencies": { - "@pkgr/core": "^0.2.1", + "@pkgr/core": "^0.2.3", "tslib": "^2.8.1" }, "engines": { @@ -31287,9 +31285,9 @@ "license": "Apache-2.0" }, "node_modules/ts-jest": { - "version": "29.3.1", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.3.1.tgz", - "integrity": "sha512-FT2PIRtZABwl6+ZCry8IY7JZ3xMuppsEV9qFVHOVe8jDzggwUZ9TsM4chyJxL9yi6LvkqcZYU3LmapEE454zBQ==", + "version": "29.3.2", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.3.2.tgz", + "integrity": "sha512-bJJkrWc6PjFVz5g2DGCNUo8z7oFEYaz1xP1NpeDU7KNLMWPpEyV8Chbpkn8xjzgRDpQhnGMyvyldoL7h8JXyug==", "dev": true, "license": "MIT", "dependencies": { @@ -31301,7 +31299,7 @@ "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", "semver": "^7.7.1", - "type-fest": "^4.38.0", + "type-fest": "^4.39.1", "yargs-parser": "^21.1.1" }, "bin": { @@ -31350,9 +31348,9 @@ } }, "node_modules/ts-jest/node_modules/type-fest": { - "version": "4.39.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.39.1.tgz", - "integrity": "sha512-uW9qzd66uyHYxwyVBYiwS4Oi0qZyUqwjU+Oevr6ZogYiXt99EOYtwvzMSLw1c3lYo2HzJsep/NB23iEVEgjG/w==", + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.40.0.tgz", + "integrity": "sha512-ABHZ2/tS2JkvH1PEjxFDTUWC8dB5OsIGZP4IFLhR293GqT5Y5qB1WwL2kMPYhQW9DVgVD8Hd7I8gjwPIf5GFkw==", "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { @@ -31586,15 +31584,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.29.1.tgz", - "integrity": "sha512-f8cDkvndhbQMPcysk6CUSGBWV+g1utqdn71P5YKwMumVMOG/5k7cHq0KyG4O52nB0oKS4aN2Tp5+wB4APJGC+w==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.30.1.tgz", + "integrity": "sha512-D7lC0kcehVH7Mb26MRQi64LMyRJsj3dToJxM1+JVTl53DQSV5/7oUGWQLcKl1C1KnoVHxMMU2FNQMffr7F3Row==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.29.1", - "@typescript-eslint/parser": "8.29.1", - "@typescript-eslint/utils": "8.29.1" + "@typescript-eslint/eslint-plugin": "8.30.1", + "@typescript-eslint/parser": "8.30.1", + "@typescript-eslint/utils": "8.30.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31609,17 +31607,17 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.29.1.tgz", - "integrity": "sha512-ba0rr4Wfvg23vERs3eB+P3lfj2E+2g3lhWcCVukUuhtcdUx5lSIFZlGFEBHKr+3zizDa/TvZTptdNHVZWAkSBg==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.30.1.tgz", + "integrity": "sha512-v+VWphxMjn+1t48/jO4t950D6KR8JaJuNXzi33Ve6P8sEmPr5k6CEXjdGwT6+LodVnEa91EQCtwjWNUCPweo+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.29.1", - "@typescript-eslint/type-utils": "8.29.1", - "@typescript-eslint/utils": "8.29.1", - "@typescript-eslint/visitor-keys": "8.29.1", + "@typescript-eslint/scope-manager": "8.30.1", + "@typescript-eslint/type-utils": "8.30.1", + "@typescript-eslint/utils": "8.30.1", + "@typescript-eslint/visitor-keys": "8.30.1", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -31639,16 +31637,16 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/parser": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.29.1.tgz", - "integrity": "sha512-zczrHVEqEaTwh12gWBIJWj8nx+ayDcCJs06yoNMY0kwjMWDM6+kppljY+BxWI06d2Ja+h4+WdufDcwMnnMEWmg==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.30.1.tgz", + "integrity": "sha512-H+vqmWwT5xoNrXqWs/fesmssOW70gxFlgcMlYcBaWNPIEWDgLa4W9nkSPmhuOgLnXq9QYgkZ31fhDyLhleCsAg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.29.1", - "@typescript-eslint/types": "8.29.1", - "@typescript-eslint/typescript-estree": "8.29.1", - "@typescript-eslint/visitor-keys": "8.29.1", + "@typescript-eslint/scope-manager": "8.30.1", + "@typescript-eslint/types": "8.30.1", + "@typescript-eslint/typescript-estree": "8.30.1", + "@typescript-eslint/visitor-keys": "8.30.1", "debug": "^4.3.4" }, "engines": { @@ -31664,14 +31662,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/scope-manager": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.29.1.tgz", - "integrity": "sha512-2nggXGX5F3YrsGN08pw4XpMLO1Rgtnn4AzTegC2MDesv6q3QaTU5yU7IbS1tf1IwCR0Hv/1EFygLn9ms6LIpDA==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.30.1.tgz", + "integrity": "sha512-+C0B6ChFXZkuaNDl73FJxRYT0G7ufVPOSQkqkpM/U198wUwUFOtgo1k/QzFh1KjpBitaK7R1tgjVz6o9HmsRPg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.1", - "@typescript-eslint/visitor-keys": "8.29.1" + "@typescript-eslint/types": "8.30.1", + "@typescript-eslint/visitor-keys": "8.30.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31682,14 +31680,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/type-utils": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.29.1.tgz", - "integrity": "sha512-DkDUSDwZVCYN71xA4wzySqqcZsHKic53A4BLqmrWFFpOpNSoxX233lwGu/2135ymTCR04PoKiEEEvN1gFYg4Tw==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.30.1.tgz", + "integrity": "sha512-64uBF76bfQiJyHgZISC7vcNz3adqQKIccVoKubyQcOnNcdJBvYOILV1v22Qhsw3tw3VQu5ll8ND6hycgAR5fEA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.29.1", - "@typescript-eslint/utils": "8.29.1", + "@typescript-eslint/typescript-estree": "8.30.1", + "@typescript-eslint/utils": "8.30.1", "debug": "^4.3.4", "ts-api-utils": "^2.0.1" }, @@ -31706,9 +31704,9 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/types": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.29.1.tgz", - "integrity": "sha512-VT7T1PuJF1hpYC3AGm2rCgJBjHL3nc+A/bhOp9sGMKfi5v0WufsX/sHCFBfNTx2F+zA6qBc/PD0/kLRLjdt8mQ==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.30.1.tgz", + "integrity": "sha512-81KawPfkuulyWo5QdyG/LOKbspyyiW+p4vpn4bYO7DM/hZImlVnFwrpCTnmNMOt8CvLRr5ojI9nU1Ekpw4RcEw==", "dev": true, "license": "MIT", "engines": { @@ -31720,14 +31718,14 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/typescript-estree": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.29.1.tgz", - "integrity": "sha512-l1enRoSaUkQxOQnbi0KPUtqeZkSiFlqrx9/3ns2rEDhGKfTa+88RmXqedC1zmVTOWrLc2e6DEJrTA51C9iLH5g==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.30.1.tgz", + "integrity": "sha512-kQQnxymiUy9tTb1F2uep9W6aBiYODgq5EMSk6Nxh4Z+BDUoYUSa029ISs5zTzKBFnexQEh71KqwjKnRz58lusQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.1", - "@typescript-eslint/visitor-keys": "8.29.1", + "@typescript-eslint/types": "8.30.1", + "@typescript-eslint/visitor-keys": "8.30.1", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -31747,16 +31745,16 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/utils": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.29.1.tgz", - "integrity": "sha512-QAkFEbytSaB8wnmB+DflhUPz6CLbFWE2SnSCrRMEa+KnXIzDYbpsn++1HGvnfAsUY44doDXmvRkO5shlM/3UfA==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.30.1.tgz", + "integrity": "sha512-T/8q4R9En2tcEsWPQgB5BQ0XJVOtfARcUvOa8yJP3fh9M/mXraLxZrkCfGb6ChrO/V3W+Xbd04RacUEqk1CFEQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.29.1", - "@typescript-eslint/types": "8.29.1", - "@typescript-eslint/typescript-estree": "8.29.1" + "@typescript-eslint/scope-manager": "8.30.1", + "@typescript-eslint/types": "8.30.1", + "@typescript-eslint/typescript-estree": "8.30.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -31771,13 +31769,13 @@ } }, "node_modules/typescript-eslint/node_modules/@typescript-eslint/visitor-keys": { - "version": "8.29.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.29.1.tgz", - "integrity": "sha512-RGLh5CRaUEf02viP5c1Vh1cMGffQscyHe7HPAzGpfmfflFg1wUz2rYxd+OZqwpeypYvZ8UxSxuIpF++fmOzEcg==", + "version": "8.30.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.30.1.tgz", + "integrity": "sha512-aEhgas7aJ6vZnNFC7K4/vMGDGyOiqWcYZPpIWrTKuTAlsvDNKy2GFDqh9smL+iq069ZvR0YzEeq0B8NJlLzjFA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.29.1", + "@typescript-eslint/types": "8.30.1", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -33612,7 +33610,6 @@ } }, "packages/common": { - "name": "@generative-ai-use-cases/common", "dependencies": { "aws-jwt-verify": "^4.0.0" } @@ -33634,7 +33631,6 @@ } }, "packages/types": { - "name": "@types/generative-ai-use-cases", "dependencies": { "@aws-sdk/client-bedrock-agent-runtime": "^3.755.0", "@aws-sdk/client-bedrock-runtime": "^3.755.0", diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index 8f6702181..d1eb6b9ae 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -34,22 +34,30 @@ let audioOutputQueue: string[] = []; let promptName = randomUUID(); let audioContentId = randomUUID(); +const clearQueue = () => { + eventQueue = []; + audioInputQueue = []; + audioOutputQueue = []; +}; + const initialize = () => { isActive = false; isProcessingAudio = false; isAudioStarted = false; - eventQueue = []; - audioInputQueue = []; - audioOutputQueue = []; + clearQueue(); }; const dispatchEvent = async (channel: EventsChannel, event: SpeechToSpeechEventType, data: any = undefined) => { - await channel.publish({ - direction: 'btoc', - event, - data, - } as SpeechToSpeechEvent); + try { + await channel.publish({ + direction: 'btoc', + event, + data, + } as SpeechToSpeechEvent); + } catch (e) { + console.error('Failed to publish the event via channel. The channel might be closed', event, data); + } }; const enqueueSessionStart = () => { @@ -173,7 +181,9 @@ const enqueueSessionEnd = () => { const enqueueAudioStop = () => { isAudioStarted = false; - isActive = false; + + clearQueue(); + // isActive = false; eventQueue.push({ event: { @@ -245,24 +255,33 @@ const createAsyncIterator = () => { [Symbol.asyncIterator]: () => { return { next: async (): Promise> => { - while (eventQueue.length === 0 && isActive) { - await new Promise(s => setTimeout(s, 100)); - } + try { + while (eventQueue.length === 0 && isActive) { + await new Promise(s => setTimeout(s, 100)); + } - if (!isActive) { - return { value: undefined, done: true }; - } + const nextEvent = eventQueue.shift(); - const nextEvent = eventQueue.shift(); + if (!nextEvent) { + return { value: undefined, done: true }; + } - return { - value: { - chunk: { - bytes: new TextEncoder().encode(JSON.stringify(nextEvent)), + if (nextEvent.event.sessionEnd) { + isActive = false; + } + + return { + value: { + chunk: { + bytes: new TextEncoder().encode(JSON.stringify(nextEvent)), + }, }, - }, - done: false, - }; + done: false, + }; + } catch (e) { + console.error('Error in asyncIterator', e); + return { value: undefined, done: true }; + } }, }; }, @@ -309,16 +328,21 @@ const processResponseStream = async (channel: EventsChannel, response: InvokeMod const textResponse = new TextDecoder().decode(event.chunk.bytes); const jsonResponse = JSON.parse(textResponse); - console.log('Bedrock response', jsonResponse); - if (jsonResponse.event?.audioOutput) { await enqueueAudioOutput(channel, jsonResponse.event.audioOutput.content); } else if (jsonResponse.event?.contentEnd && jsonResponse.event?.contentEnd?.type === 'AUDIO') { await forcePublishAudioOutput(channel); } else if (jsonResponse.event?.contentStart && jsonResponse.event?.contentStart?.type === 'TEXT') { + let generationStage = null; + + if (jsonResponse.event?.contentStart?.additionalModelFields) { + generationStage = JSON.parse(jsonResponse.event?.contentStart?.additionalModelFields).generationStage; + } + await dispatchEvent(channel, 'textStart', { id: jsonResponse.event?.contentStart?.contentId, role: jsonResponse.event?.contentStart?.role?.toLowerCase(), + generationStage, }); } else if (jsonResponse.event?.textOutput) { await dispatchEvent(channel, 'textOutput', { @@ -336,7 +360,7 @@ const processResponseStream = async (channel: EventsChannel, response: InvokeMod } } } catch (e) { - console.error(e); + console.error('Error in processResponseStream', e); } }; @@ -417,16 +441,6 @@ export const handler = async (event: { channelId: string }) => { enqueueAudioStop(); enqueuePromptEnd(); enqueueSessionEnd(); - - if (channel) { - try { - await dispatchEvent(channel, 'end'); - channel.close(); - } catch (e) { - console.error(e); - throw e; - } - } } } }, @@ -463,7 +477,7 @@ export const handler = async (event: { channelId: string }) => { // Start response stream await processResponseStream(channel, response); } catch (e) { - console.error(e); + console.error('Error in main process', e); } finally { if (channel) { try { @@ -473,7 +487,7 @@ export const handler = async (event: { channelId: string }) => { } }); } catch (e) { - console.error(e); + console.error('Error during finalization', e); } } diff --git a/packages/web/src/hooks/useSpeechToSpeech/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts index 1e05327ff..9388118bd 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/index.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts @@ -53,7 +53,7 @@ const base64ToFloat32Array = (base64String: string) => { export const useSpeechToSpeech = () => { const api = useHttp(); - const { clear, messages, setupSystemPrompt, onTextOutput, onTextStop } = useChatHistory(); + const { clear, messages, setupSystemPrompt, onTextStart, onTextOutput, onTextStop, isAssistantSpeeching } = useChatHistory(); const [isActive, setIsActive] = useState(false); const [isLoading, setIsLoading] = useState(false); const systemPromptRef = useRef(''); @@ -131,7 +131,6 @@ export const useSpeechToSpeech = () => { const event = data?.event; if (event && event.direction === 'btoc') { if (event.event === 'ready') { - console.log('Now ready to speech-to-speech!'); startRecording().then(() => { setIsLoading(false); }); @@ -154,6 +153,7 @@ export const useSpeechToSpeech = () => { } } else if (event.event === 'textStart') { console.log('textStart', event.data); + onTextStart(event.data); } else if (event.event === 'textOutput') { console.log('textOutput', event.data); onTextOutput(event.data); @@ -263,6 +263,7 @@ export const useSpeechToSpeech = () => { messages, isActive, isLoading, + isAssistantSpeeching, startSession, closeSession, } diff --git a/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts index e5e82684d..521aef355 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts @@ -1,109 +1,93 @@ import { useState, useRef } from 'react'; import { UnrecordedMessage } from 'generative-ai-use-cases'; -type SpeechToSpeechMessage = UnrecordedMessage & { isPartial: boolean }; - -// stopReason が INTERRUPTED だったら content は空 -// そこから先のそのロールの発言は無視される (基本的にアシスタントの発言と思われる) -// ユーザーの発言が到着したら、interrupted を解除 - const useChatHistory = () => { - const [messages, setMessages] = useState([]); + const [messages, setMessages] = useState([]); + const [isAssistantSpeeching, setIsAssistantSpeeching] = useState(false); const messageCache = useRef>({}); + const generationStageCache = useRef>({}); const stopReasonCache = useRef>({}); - const interrupted = useRef(false); const clear = () => { setMessages([]); messageCache.current = {}; + generationStageCache.current = {}; + stopReasonCache.current = {}; } const setupSystemPrompt = (prompt: string) => { setMessages([{ role: 'system', content: prompt, - isPartial: false, }]); }; - const updateMessages = (newMessage: SpeechToSpeechMessage) => { - if (interrupted.current) { - if (newMessage.role === 'assistant') { - return; - } else { - interrupted.current = false; - } + const tryUpdateMessage = (id: string) => { + if ( + !messageCache.current[id] || + !generationStageCache.current[id] || + !stopReasonCache.current[id] + ) { + return; + } + + if (generationStageCache.current[id] !== 'FINAL') { + return; + } + + if (stopReasonCache.current[id] === 'INTERRUPTED') { + return; } setMessages((messages) => { const lastMessageIndex = messages.length - 1; const lastMessage = messages[lastMessageIndex]; const messagesWithoutLast = messages.slice(0, lastMessageIndex); - - if (lastMessage) { - if (lastMessage.role !== newMessage.role) { - return [...messagesWithoutLast, lastMessage, newMessage]; - } else { - if (lastMessage.isPartial && !newMessage.isPartial) { - return [...messagesWithoutLast, newMessage]; - } else { - const updatedLastMessage = { - ...lastMessage, - content: lastMessage.content + ' ' + newMessage.content, - isPartial: newMessage.isPartial, - }; - return [...messagesWithoutLast, updatedLastMessage]; - } - } - } - - if (lastMessage) { - return [...messagesWithoutLast, lastMessage]; + const role = messageCache.current[id].role; + const content = messageCache.current[id].content; + + if (lastMessage.role === role) { + const updatedLastMessage: UnrecordedMessage = { + ...lastMessage, + content: lastMessage.content + ' ' + content, + }; + return [...messagesWithoutLast, updatedLastMessage]; } else { - return [...messagesWithoutLast]; + const newMessage: UnrecordedMessage = { role, content }; + return [...messagesWithoutLast, lastMessage, newMessage]; } }); - }; + } - const onTextOutput = (data: { id: string, role: string, content: string}) => { - if (stopReasonCache.current[data.id]) { - const newMessage: SpeechToSpeechMessage = { - role: data.role as 'user' | 'assistant', - content: data.content, - isPartial: stopReasonCache.current[data.id] === 'PARTIAL_TURN', - }; + const onTextStart = (data: { id: string, role: string, generationStage: string}) => { + generationStageCache.current[data.id] = data.generationStage; + tryUpdateMessage(data.id); - updateMessages(newMessage); + if (data.role === 'assistant' && data.generationStage === 'SPECULATIVE') { + setIsAssistantSpeeching(true); } else { - messageCache.current[data.id] = { role: data.role as 'user' | 'assistant', content: data.content }; + setIsAssistantSpeeching(false); } }; - const onTextStop = (data: { id: string, stopReason: string }) => { - if (data.stopReason === 'INTERRUPTED') { - interrupted.current = true; - return; - } - - if (messageCache.current[data.id]) { - const newMessage: SpeechToSpeechMessage = { - role: messageCache.current[data.id].role, - content: messageCache.current[data.id].content, - isPartial: data.stopReason === 'PARTIAL_TURN', - }; + const onTextOutput = (data: { id: string, role: string, content: string}) => { + messageCache.current[data.id] = { role: data.role as 'user' | 'assistant', content: data.content }; + tryUpdateMessage(data.id); + }; - updateMessages(newMessage); - } else { - stopReasonCache.current[data.id] = data.stopReason; - } + const onTextStop = (data: { id: string, stopReason: string }) => { + stopReasonCache.current[data.id] = data.stopReason; + tryUpdateMessage(data.id); }; return { clear, messages, setupSystemPrompt, + onTextStart, onTextOutput, onTextStop, + isAssistantSpeeching, }; } diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index 0c8d1c471..c43b9d133 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -7,6 +7,8 @@ import Switch from '../components/Switch'; import ExpandableField from '../components/ExpandableField'; import Button from '../components/Button'; import InputChatContent from '../components/InputChatContent'; +import ScrollTopBottom from '../components/ScrollTopBottom'; +import useFollow from '../hooks/useFollow'; const SpeechToSpeech: React.FC = () => { const { t } = useTranslation(); @@ -14,6 +16,7 @@ const SpeechToSpeech: React.FC = () => { messages, isActive, isLoading, + isAssistantSpeeching, startSession, closeSession, } = useSpeechToSpeech(); @@ -21,6 +24,7 @@ const SpeechToSpeech: React.FC = () => { // TODO: avoid hardcoding const [systemPrompt, setSystemPrompt] = useState('You are an AI assistant.'); const [inputSystemPrompt, setInputSystemPrompt] = useState(systemPrompt); + const { scrollableContainer, setFollowing } = useFollow(); const messagesWithoutSystemPrompt = useMemo(() => { return messages.filter(m => m.role !== 'system'); @@ -42,30 +46,48 @@ const SpeechToSpeech: React.FC = () => {
- Speech to Speech (beta) + Speech to Speech
-
- -
+ {!isEmpty && ( +
+ +
+ )} -
+
{showingMessages.map((m, idx) => { return ( -
+
+ {idx === 0 && ( +
+ )} +
) })} + {isAssistantSpeeching && (
+ +
+
)} +
+ +
+
@@ -79,6 +101,7 @@ const SpeechToSpeech: React.FC = () => { outlined className="text-xs" onClick={() => { + setInputSystemPrompt('You are an AI assistant.') setSystemPrompt('You are an AI assistant.') }}> {t('chat.initialize')} @@ -103,7 +126,7 @@ const SpeechToSpeech: React.FC = () => { )} {!isActive ? ( - ) : ( From a517bef77eb2395da7c380ba3e30a7e60e081610 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Thu, 17 Apr 2025 12:40:49 +0900 Subject: [PATCH 10/20] fix --- packages/web/src/hooks/useSpeechToSpeech/index.ts | 5 ----- packages/web/src/pages/SpeechToSpeechPage.tsx | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/web/src/hooks/useSpeechToSpeech/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts index 9388118bd..803812263 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/index.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts @@ -135,9 +135,7 @@ export const useSpeechToSpeech = () => { setIsLoading(false); }); } else if (event.event === 'end') { - console.log('Received "end" event'); if (isActive) { - console.log('Close the session'); closeSession(); } } else if (event.event === 'audioOutput' && audioPlayerRef.current) { @@ -152,13 +150,10 @@ export const useSpeechToSpeech = () => { } } } else if (event.event === 'textStart') { - console.log('textStart', event.data); onTextStart(event.data); } else if (event.event === 'textOutput') { - console.log('textOutput', event.data); onTextOutput(event.data); } else if (event.event === 'textStop') { - console.log('textStop', event.data); onTextStop(event.data); } } diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index c43b9d133..1d945d88a 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -8,7 +8,9 @@ import ExpandableField from '../components/ExpandableField'; import Button from '../components/Button'; import InputChatContent from '../components/InputChatContent'; import ScrollTopBottom from '../components/ScrollTopBottom'; +import Alert from '../components/Alert.tsx'; import useFollow from '../hooks/useFollow'; +import BedrockIcon from '../assets/bedrock.svg?react'; const SpeechToSpeech: React.FC = () => { const { t } = useTranslation(); @@ -49,6 +51,17 @@ const SpeechToSpeech: React.FC = () => { Speech to Speech
+ {isEmpty && ( +
+ + Speech to Speech はまだ実験的な段階です。アーキテクチャ等は今後変更される可能性があります。また、会話履歴は保存されません。 + +
+ +
+
+ )} + {!isEmpty && (
{ {!isActive ? ( ) : ( -
+
+ {!isLoading && !isActive && ( + + <> +
+ +
- } - onSend={() => { - setSystemPrompt(inputSystemPrompt); - }} - /> - - - )} + } + onSend={() => { + setSystemPrompt(inputSystemPrompt); + }} + /> + + + )} {!isActive ? ( - ) : ( - )} +
-
- ); + + ); }; export default SpeechToSpeech; From f23d51422afe5fe9422586b11ad1a8bed833b8b8 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Thu, 17 Apr 2025 14:52:05 +0900 Subject: [PATCH 12/20] fix --- .../hooks/useSpeechToSpeech/AudioRecorder.js | 165 ++++++++++++++++++ .../AudioRecorderProcessor.worklet.js | 31 ++++ .../web/src/hooks/useSpeechToSpeech/index.ts | 110 +++++------- packages/web/src/pages/SpeechToSpeechPage.tsx | 10 +- 4 files changed, 247 insertions(+), 69 deletions(-) create mode 100644 packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js create mode 100644 packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js new file mode 100644 index 000000000..6fdec7f59 --- /dev/null +++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js @@ -0,0 +1,165 @@ +import { ObjectExt } from './ObjectsExt.js'; +const AudioRecorderWorkletUrl = new URL( + './AudioRecorderProcessor.worklet.js', + import.meta.url +).toString(); + +export class AudioRecorder { + constructor() { + this.onAudioRecordedListeners = []; + this.onErrorListeners = []; + this.initialized = false; + } + + addEventListener(event, callback) { + switch (event) { + case 'onAudioRecorded': + this.onAudioRecordedListeners.push(callback); + break; + case 'onError': + this.onErrorListeners.push(callback); + break; + default: + console.error( + 'Listener registered for event type: ' + + JSON.stringify(event) + + ' which is not supported' + ); + } + } + + async start() { + try { + this.audioContext = new AudioContext({ sampleRate: 16000 }); + + // Get user media stream + try { + this.audioStream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + } catch (error) { + // Handle permission denied or device not available errors + const errorType = error.name || 'UnknownError'; + const errorMessage = error.message || 'Failed to access microphone'; + + // Notify error listeners + this.onErrorListeners.forEach(listener => listener({ + type: errorType, + message: errorMessage, + originalError: error + })); + + // Don't throw, just return false to indicate failure + console.error('Microphone access error:', errorType, errorMessage); + return false; + } + + this.sourceNode = this.audioContext.createMediaStreamSource(this.audioStream); + + // Add the audio worklet module + try { + await this.audioContext.audioWorklet.addModule(AudioRecorderWorkletUrl); + } catch (error) { + this.onErrorListeners.forEach(listener => listener({ + type: 'WorkletError', + message: 'Failed to load audio worklet', + originalError: error + })); + this.cleanup(); + return false; + } + + this.workletNode = new AudioWorkletNode( + this.audioContext, + 'audio-recorder-processor' + ); + + // Connect the source to the worklet + this.sourceNode.connect(this.workletNode); + this.workletNode.connect(this.audioContext.destination); + + // Listen for audio data from the worklet + this.workletNode.port.onmessage = (event) => { + if (event.data.type === 'audio') { + const audioData = event.data.audioData; + // Notify listeners that audio was recorded + this.onAudioRecordedListeners.forEach(listener => listener(audioData)); + } + }; + + // Start recording + this.workletNode.port.postMessage({ + type: 'start' + }); + + this.initialized = true; + return true; + } catch (error) { + // Catch any other unexpected errors + this.onErrorListeners.forEach(listener => listener({ + type: 'InitializationError', + message: 'Failed to initialize audio recorder', + originalError: error + })); + this.cleanup(); + return false; + } + } + + cleanup() { + if (ObjectExt.exists(this.workletNode)) { + try { + this.workletNode.disconnect(); + } catch (e) { + console.error('Error disconnecting worklet node:', e); + } + } + + if (ObjectExt.exists(this.sourceNode)) { + try { + this.sourceNode.disconnect(); + } catch (e) { + console.error('Error disconnecting source node:', e); + } + } + + if (ObjectExt.exists(this.audioStream)) { + try { + this.audioStream.getTracks().forEach(track => track.stop()); + } catch (e) { + console.error('Error stopping audio tracks:', e); + } + } + + if (ObjectExt.exists(this.audioContext)) { + try { + this.audioContext.close(); + } catch (e) { + console.error('Error closing audio context:', e); + } + } + + this.initialized = false; + this.audioContext = null; + this.audioStream = null; + this.sourceNode = null; + this.workletNode = null; + } + + stop() { + if (this.initialized) { + // Stop recording + if (ObjectExt.exists(this.workletNode)) { + this.workletNode.port.postMessage({ + type: 'stop' + }); + } + + this.cleanup(); + } + } +} diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js new file mode 100644 index 000000000..9ede38c7b --- /dev/null +++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js @@ -0,0 +1,31 @@ +class AudioRecorderProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.isRecording = false; + this.port.onmessage = (event) => { + if (event.data.type === 'start') { + this.isRecording = true; + } else if (event.data.type === 'stop') { + this.isRecording = false; + } + }; + } + + process(inputs, outputs, parameters) { + if (!this.isRecording || !inputs[0] || !inputs[0][0]) { + return true; + } + + const input = inputs[0][0]; + + // Send the audio data to the main thread + this.port.postMessage({ + type: 'audio', + audioData: input.slice() + }); + + return true; + } +} + +registerProcessor('audio-recorder-processor', AudioRecorderProcessor); diff --git a/packages/web/src/hooks/useSpeechToSpeech/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts index 98f1bf6e0..b5b2bae1e 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/index.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts @@ -1,6 +1,7 @@ import { useRef, useState } from 'react'; import { events, EventsChannel } from 'aws-amplify/data'; import { AudioPlayer } from './AudioPlayer'; +import { AudioRecorder } from './AudioRecorder'; import { v4 as uuid } from 'uuid'; import useHttp from '../../hooks/useHttp'; import useChatHistory from './useChatHistory'; @@ -66,12 +67,19 @@ export const useSpeechToSpeech = () => { const [isLoading, setIsLoading] = useState(false); const systemPromptRef = useRef(''); const audioPlayerRef = useRef(null); + const audioRecorderRef = useRef(null); const channelRef = useRef(null); - const audioContextRef = useRef(null); - const audioStreamRef = useRef(null); - const sourceNodeRef = useRef(null); - const processorRef = useRef(null); const audioInputQueue = useRef([]); + const [errorMessages, setErrorMessages] = useState([]); + + const resetState = () => { + setIsLoading(false); + setIsActive(false); + audioRecorderRef.current = null; + audioPlayerRef.current = null; + channelRef.current = null; + audioInputQueue.current = []; + }; const dispatchEvent = async ( event: SpeechToSpeechEventType, @@ -88,25 +96,28 @@ export const useSpeechToSpeech = () => { }; const initAudio = async () => { - const audioStream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }); + const audioPlayer = new AudioPlayer(); + audioPlayerRef.current = audioPlayer; - const audioContext = new AudioContext({ - sampleRate: 16000, + const audioRecorder = new AudioRecorder(); + audioRecorder.addEventListener('onAudioRecorded', (audioData: Float32Array) => { + const int16Array = float32ArrayToInt16Array(audioData); + const base64Data = arrayBufferToBase64(int16Array.buffer); + audioInputQueue.current.push(base64Data); }); - audioStreamRef.current = audioStream; - audioContextRef.current = audioContext; - - const audioPlayer = new AudioPlayer(); - await audioPlayer.start(); + // Add error listener to handle microphone permission issues + audioRecorder.addEventListener('onError', (error: { type: string; message: string }) => { + console.error('Audio recorder error:', error.type, error.message); + // You can add UI notification here if needed + if (error.type === 'NotAllowedError' || error.type === 'PermissionDeniedError') { + // Handle microphone permission denied specifically + resetState(); + setErrorMessages([...errorMessages, 'The microphone is not available. Please grant permission to use the microphone.']); + } + }); - audioPlayerRef.current = audioPlayer; + audioRecorderRef.current = audioRecorder; }; const processAudioInput = async () => { @@ -137,10 +148,9 @@ export const useSpeechToSpeech = () => { audioInputQueue.current = []; const channelId = uuid(); - console.log(`/${NAMESPACE}/${channelId}`); const channel = await events.connect(`/${NAMESPACE}/${channelId}`); - channelRef.current = channel; + channelRef.current = channel; channel.subscribe({ // eslint-disable-next-line @typescript-eslint/no-explicit-any next: (data: any) => { @@ -184,11 +194,7 @@ export const useSpeechToSpeech = () => { }; const startRecording = async () => { - if ( - !audioContextRef.current || - !audioStreamRef.current || - !systemPromptRef.current - ) { + if (!audioPlayerRef.current || !audioRecorderRef.current || !systemPromptRef.current) { return; } @@ -198,29 +204,13 @@ export const useSpeechToSpeech = () => { setupSystemPrompt(systemPromptRef.current); - const sourceNode = audioContextRef.current.createMediaStreamSource( - audioStreamRef.current - ); + await audioPlayerRef.current.start(); - if (audioContextRef.current.createScriptProcessor) { - const processor = audioContextRef.current.createScriptProcessor( - 512, - 1, - 1 - ); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - processor.onaudioprocess = (e: any) => { - const inputData = e.inputBuffer.getChannelData(0); - const int16Array = float32ArrayToInt16Array(inputData); - const base64Data = arrayBufferToBase64(int16Array.buffer); - audioInputQueue.current.push(base64Data); - }; - - sourceNode.connect(processor); - processor.connect(audioContextRef.current.destination); - - sourceNodeRef.current = sourceNode; - processorRef.current = processor; + // Start recording using the AudioRecorder and check for success + const success = await audioRecorderRef.current.start(); + + if (!success) { + return; } setIsActive(true); @@ -231,14 +221,9 @@ export const useSpeechToSpeech = () => { const stopRecording = async () => { setIsActive(false); - if (processorRef.current) { - processorRef.current.disconnect(); - processorRef.current = null; - } - - if (sourceNodeRef.current) { - sourceNodeRef.current.disconnect(); - sourceNodeRef.current = null; + if (audioRecorderRef.current) { + audioRecorderRef.current.stop(); + audioRecorderRef.current = null; } if (audioPlayerRef.current) { @@ -246,18 +231,6 @@ export const useSpeechToSpeech = () => { audioPlayerRef.current = null; } - if (audioStreamRef.current) { - audioStreamRef.current - .getTracks() - .forEach((track: MediaStreamTrack) => track.stop()); - audioStreamRef.current = null; - } - - if (audioContextRef.current) { - audioContextRef.current.close(); - audioContextRef.current = null; - } - await dispatchEvent('audioStop'); }; @@ -290,5 +263,6 @@ export const useSpeechToSpeech = () => { isAssistantSpeeching, startSession, closeSession, + errorMessages, }; }; diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index c7308d237..add378500 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -1,4 +1,4 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect } from 'react'; import { useSpeechToSpeech } from '../hooks/useSpeechToSpeech'; import { useTranslation } from 'react-i18next'; import { @@ -15,6 +15,7 @@ import ScrollTopBottom from '../components/ScrollTopBottom'; import Alert from '../components/Alert.tsx'; import useFollow from '../hooks/useFollow'; import BedrockIcon from '../assets/bedrock.svg?react'; +import { toast } from 'sonner'; const SpeechToSpeech: React.FC = () => { const { t } = useTranslation(); @@ -25,6 +26,7 @@ const SpeechToSpeech: React.FC = () => { isAssistantSpeeching, startSession, closeSession, + errorMessages, } = useSpeechToSpeech(); const [showSystemPrompt, setShowSystemPrompt] = useState(false); // TODO: avoid hardcoding @@ -48,6 +50,12 @@ const SpeechToSpeech: React.FC = () => { return messagesWithoutSystemPrompt.length === 0; }, [messagesWithoutSystemPrompt]); + useEffect(() => { + if (errorMessages.length > 0) { + toast.error(errorMessages[errorMessages.length - 1]); + } + }, [errorMessages]); + return ( <>
From 59336aa5c86e97d0d42ab0c2fda46f80f001a8bc Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Thu, 17 Apr 2025 17:55:23 +0900 Subject: [PATCH 13/20] fix --- packages/cdk/cdk.json | 1 + packages/cdk/lambda/speechToSpeechTask.ts | 13 +- .../cdk/lambda/startSpeechToSpeechSession.ts | 5 +- .../cdk/lib/construct/speech-to-speech.ts | 11 + packages/cdk/lib/construct/web.ts | 4 + .../cdk/lib/generative-ai-use-cases-stack.ts | 6 + packages/cdk/lib/stack-input.ts | 17 + packages/cdk/parameter.ts | 4 + .../generative-ai-use-cases.test.ts.snap | 558 +++++++++++++++++- .../cdk/test/generative-ai-use-cases.test.ts | 3 + packages/common/src/application/model.ts | 9 + packages/types/src/model.d.ts | 3 + .../web/public/locales/translation/en.yaml | 9 + .../web/public/locales/translation/ja.yaml | 9 + packages/web/src/App.tsx | 7 + packages/web/src/hooks/useModel.ts | 27 + .../AudioPlayerProcessor.worklet.js | 2 +- .../hooks/useSpeechToSpeech/AudioRecorder.js | 50 +- .../AudioRecorderProcessor.worklet.js | 2 +- .../web/src/hooks/useSpeechToSpeech/index.ts | 53 +- packages/web/src/pages/SpeechToSpeechPage.tsx | 49 +- packages/web/src/vite-env.d.ts | 1 + setup-env.sh | 1 + web_devw_win.ps1 | 1 + 24 files changed, 784 insertions(+), 61 deletions(-) diff --git a/packages/cdk/cdk.json b/packages/cdk/cdk.json index 6144db127..b60cee708 100644 --- a/packages/cdk/cdk.json +++ b/packages/cdk/cdk.json @@ -48,6 +48,7 @@ ], "imageGenerationModelIds": ["amazon.nova-canvas-v1:0"], "videoGenerationModelIds": ["amazon.nova-reel-v1:0"], + "speechToSpeechModelIds": ["amazon.nova-sonic-v1:0"], "endpointNames": [], "agentEnabled": false, "searchAgentEnabled": false, diff --git a/packages/cdk/lambda/speechToSpeechTask.ts b/packages/cdk/lambda/speechToSpeechTask.ts index 59c0413a6..5c87b8317 100644 --- a/packages/cdk/lambda/speechToSpeechTask.ts +++ b/packages/cdk/lambda/speechToSpeechTask.ts @@ -12,6 +12,7 @@ import { NodeHttp2Handler } from '@smithy/node-http-handler'; import { SpeechToSpeechEventType, SpeechToSpeechEvent, + Model, } from 'generative-ai-use-cases'; Object.assign(global, { WebSocket: require('ws') }); @@ -26,6 +27,7 @@ let isProcessingAudio = false; let isAudioStarted = false; // Queues +// eslint-disable-next-line @typescript-eslint/no-explicit-any let eventQueue: Array = []; let audioInputQueue: string[] = []; let audioOutputQueue: string[] = []; @@ -51,6 +53,7 @@ const initialize = () => { const dispatchEvent = async ( channel: EventsChannel, event: SpeechToSpeechEventType, + // eslint-disable-next-line @typescript-eslint/no-explicit-any data: any = undefined ) => { try { @@ -304,6 +307,7 @@ const createAsyncIterator = () => { return: async () => { return { value: undefined, done: true }; }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any throw: async (error: any) => { console.error(error); throw error; @@ -397,11 +401,11 @@ const processResponseStream = async ( } }; -export const handler = async (event: { channelId: string }) => { +export const handler = async (event: { channelId: string; model: Model }) => { let channel: EventsChannel | null = null; try { - console.log('channelId', event.channelId); + console.log('event', event); initialize(); @@ -412,7 +416,7 @@ export const handler = async (event: { channelId: string }) => { console.log('promptName', promptName); const bedrock = new BedrockRuntimeClient({ - region: 'us-east-1', // TODO + region: event.model.region, requestHandler: new NodeHttp2Handler({ requestTimeout: 300000, sessionTimeout: 300000, @@ -461,6 +465,7 @@ export const handler = async (event: { channelId: string }) => { console.log('Connected!'); channel.subscribe({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any next: async (data: any) => { const event = data?.event; if (event && event.direction === 'ctob') { @@ -504,7 +509,7 @@ export const handler = async (event: { channelId: string }) => { const response = await bedrock.send( new InvokeModelWithBidirectionalStreamCommand({ - modelId: 'amazon.nova-sonic-v1:0', + modelId: event.model.modelId, body: asyncIterator, }) ); diff --git a/packages/cdk/lambda/startSpeechToSpeechSession.ts b/packages/cdk/lambda/startSpeechToSpeechSession.ts index 2ab27c582..2231e634a 100644 --- a/packages/cdk/lambda/startSpeechToSpeechSession.ts +++ b/packages/cdk/lambda/startSpeechToSpeechSession.ts @@ -8,15 +8,16 @@ import { export const handler = async ( event: APIGatewayProxyEvent ): Promise => { + console.log(event.body); // TODO: delete try { - const { channel } = JSON.parse(event.body!); + const { channel, model } = JSON.parse(event.body!); const lambda = new LambdaClient({}); await lambda.send( new InvokeCommand({ FunctionName: process.env.SPEECH_TO_SPEECH_TASK_FUNCTION_ARN, InvocationType: InvocationType.Event, - Payload: JSON.stringify({ channelId: channel }), + Payload: JSON.stringify({ channelId: channel, model }), }) ); diff --git a/packages/cdk/lib/construct/speech-to-speech.ts b/packages/cdk/lib/construct/speech-to-speech.ts index 239e0e0f9..8f2803def 100644 --- a/packages/cdk/lib/construct/speech-to-speech.ts +++ b/packages/cdk/lib/construct/speech-to-speech.ts @@ -6,11 +6,14 @@ import * as agw from 'aws-cdk-lib/aws-apigateway'; import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; import { Runtime } from 'aws-cdk-lib/aws-lambda'; import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { ModelConfiguration } from 'generative-ai-use-cases'; +import { BEDROCK_SPEECH_TO_SPEECH_MODELS } from '@generative-ai-use-cases/common'; export interface SpeechToSpeechProps { envSuffix: string; userPool: cognito.UserPool; api: agw.RestApi; + speechToSpeechModelIds: ModelConfiguration[]; } export class SpeechToSpeech extends Construct { @@ -20,6 +23,14 @@ export class SpeechToSpeech extends Construct { constructor(scope: Construct, id: string, props: SpeechToSpeechProps) { super(scope, id); + const speechToSpeechModelIds = props.speechToSpeechModelIds; + + for (const model of speechToSpeechModelIds) { + if (!BEDROCK_SPEECH_TO_SPEECH_MODELS.includes(model.modelId)) { + throw new Error(`Unsupported Model Name: ${model.modelId}`); + } + } + const channelNamespaceName = 'speech-to-speech'; const eventApi = new appsync.EventApi(this, 'EventApi', { apiName: `SpeechToSpeech${props.envSuffix}`, diff --git a/packages/cdk/lib/construct/web.ts b/packages/cdk/lib/construct/web.ts index 2722e925f..8eea43df3 100644 --- a/packages/cdk/lib/construct/web.ts +++ b/packages/cdk/lib/construct/web.ts @@ -49,6 +49,7 @@ export interface WebProps { hiddenUseCases: HiddenUseCases; speechToSpeechNamespace: string; speechToSpeechEventApiEndpoint: string; + speechToSpeechModelIds: ModelConfiguration[]; } export class Web extends Construct { @@ -202,6 +203,9 @@ export class Web extends Construct { VITE_APP_SPEECH_TO_SPEECH_NAMESPACE: props.speechToSpeechNamespace, VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT: props.speechToSpeechEventApiEndpoint, + VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS: JSON.stringify( + props.speechToSpeechModelIds + ), }, }); // Enhance computing resources diff --git a/packages/cdk/lib/generative-ai-use-cases-stack.ts b/packages/cdk/lib/generative-ai-use-cases-stack.ts index 4016489e2..62d81efa0 100644 --- a/packages/cdk/lib/generative-ai-use-cases-stack.ts +++ b/packages/cdk/lib/generative-ai-use-cases-stack.ts @@ -112,6 +112,7 @@ export class GenerativeAiUseCasesStack extends Stack { envSuffix: params.env, api: api.api, userPool: auth.userPool, + speechToSpeechModelIds: params.speechToSpeechModelIds, }); // Web Frontend @@ -145,6 +146,7 @@ export class GenerativeAiUseCasesStack extends Stack { useCaseBuilderEnabled: params.useCaseBuilderEnabled, speechToSpeechNamespace: speechToSpeech.namespace, speechToSpeechEventApiEndpoint: speechToSpeech.eventApiEndpoint, + speechToSpeechModelIds: params.speechToSpeechModelIds, // Frontend hiddenUseCases: params.hiddenUseCases, // Custom Domain @@ -324,6 +326,10 @@ export class GenerativeAiUseCasesStack extends Stack { value: speechToSpeech.eventApiEndpoint, }); + new CfnOutput(this, 'SpeechToSpeechModelIds', { + value: JSON.stringify(params.speechToSpeechModelIds), + }); + this.userPool = auth.userPool; this.userPoolClient = auth.client; diff --git a/packages/cdk/lib/stack-input.ts b/packages/cdk/lib/stack-input.ts index 3196e0fba..6def4549d 100644 --- a/packages/cdk/lib/stack-input.ts +++ b/packages/cdk/lib/stack-input.ts @@ -68,6 +68,17 @@ export const stackInputSchema = z.object({ ]) ) .default(['amazon.nova-reel-v1:0']), + speechToSpeechModelIds: z + .array( + z.union([ + z.string(), + z.object({ + modelId: z.string(), + region: z.string(), + }), + ]) + ) + .default(['amazon.nova-sonic-v1:0']), endpointNames: z.array(z.string()).default([]), crossAccountBedrockRoleArn: z.string().nullish(), // RAG @@ -165,6 +176,12 @@ export const processedStackInputSchema = stackInputSchema.extend({ region: z.string(), }) ), + speechToSpeechModelIds: z.array( + z.object({ + modelId: z.string(), + region: z.string(), + }) + ), }); export type StackInput = z.infer; diff --git a/packages/cdk/parameter.ts b/packages/cdk/parameter.ts index 535c1f599..c1b03ad67 100644 --- a/packages/cdk/parameter.ts +++ b/packages/cdk/parameter.ts @@ -67,5 +67,9 @@ export const getParams = (app: cdk.App): ProcessedStackInput => { params.videoGenerationModelIds, params.modelRegion ), + speechToSpeechModelIds: convertToModelConfiguration( + params.speechToSpeechModelIds, + params.modelRegion + ), }; }; diff --git a/packages/cdk/test/__snapshots__/generative-ai-use-cases.test.ts.snap b/packages/cdk/test/__snapshots__/generative-ai-use-cases.test.ts.snap index 43b50943b..1941dda08 100644 --- a/packages/cdk/test/__snapshots__/generative-ai-use-cases.test.ts.snap +++ b/packages/cdk/test/__snapshots__/generative-ai-use-cases.test.ts.snap @@ -2041,6 +2041,29 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "SelfSignUpEnabled": { "Value": "true", }, + "SpeechToSpeechEventApiEndpoint": { + "Value": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "Dns.Http", + ], + }, + "/event", + ], + ], + }, + }, + "SpeechToSpeechModelIds": { + "Value": "[{"modelId":"amazon.nova-sonic-v1:0","region":"us-east-1"}]", + }, + "SpeechToSpeechNamespace": { + "Value": "speech-to-speech", + }, "UseCaseBuilderEnabled": { "Value": "true", }, @@ -2159,7 +2182,7 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "Type": "AWS::IAM::Role", "UpdateReplacePolicy": "Delete", }, - "APIApiDeployment3A502123d75c6f1f887fef40596e8e44b7c34d5d": { + "APIApiDeployment3A502123457785fef82d1521cf56189f5ab6294b": { "DeletionPolicy": "Delete", "DependsOn": [ "APIApiApi4XXDCF913C8", @@ -2230,6 +2253,9 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "APIApisharesshareshareId3696CDAF", "APIApisharesshareOPTIONS6D42A67D", "APIApisharesshareF2EC0273", + "APIApispeechtospeechOPTIONS92DBE1B9", + "APIApispeechtospeechPOST8D76474A", + "APIApispeechtospeechD6FA255B", "APIApisystemcontextssystemContextIdDELETEB527E743", "APIApisystemcontextssystemContextIdOPTIONS96E8D02C", "APIApisystemcontextssystemContextId9D6F9E56", @@ -2306,7 +2332,7 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` ], "Properties": { "DeploymentId": { - "Ref": "APIApiDeployment3A502123d75c6f1f887fef40596e8e44b7c34d5d", + "Ref": "APIApiDeployment3A502123457785fef82d1521cf56189f5ab6294b", }, "RestApiId": { "Ref": "APIApiFFA96F67", @@ -5869,6 +5895,172 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "Type": "AWS::ApiGateway::Method", "UpdateReplacePolicy": "Delete", }, + "APIApispeechtospeechD6FA255B": { + "DeletionPolicy": "Delete", + "Properties": { + "ParentId": { + "Fn::GetAtt": [ + "APIApiFFA96F67", + "RootResourceId", + ], + }, + "PathPart": "speech-to-speech", + "RestApiId": { + "Ref": "APIApiFFA96F67", + }, + }, + "Type": "AWS::ApiGateway::Resource", + "UpdateReplacePolicy": "Delete", + }, + "APIApispeechtospeechOPTIONS92DBE1B9": { + "DeletionPolicy": "Delete", + "Properties": { + "ApiKeyRequired": false, + "AuthorizationType": "NONE", + "HttpMethod": "OPTIONS", + "Integration": { + "IntegrationResponses": [ + { + "ResponseParameters": { + "method.response.header.Access-Control-Allow-Headers": "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token,X-Amz-User-Agent'", + "method.response.header.Access-Control-Allow-Methods": "'OPTIONS,GET,PUT,POST,DELETE,PATCH,HEAD'", + "method.response.header.Access-Control-Allow-Origin": "'*'", + }, + "StatusCode": "204", + }, + ], + "RequestTemplates": { + "application/json": "{ statusCode: 200 }", + }, + "Type": "MOCK", + }, + "MethodResponses": [ + { + "ResponseParameters": { + "method.response.header.Access-Control-Allow-Headers": true, + "method.response.header.Access-Control-Allow-Methods": true, + "method.response.header.Access-Control-Allow-Origin": true, + }, + "StatusCode": "204", + }, + ], + "ResourceId": { + "Ref": "APIApispeechtospeechD6FA255B", + }, + "RestApiId": { + "Ref": "APIApiFFA96F67", + }, + }, + "Type": "AWS::ApiGateway::Method", + "UpdateReplacePolicy": "Delete", + }, + "APIApispeechtospeechPOST8D76474A": { + "DeletionPolicy": "Delete", + "Properties": { + "AuthorizationType": "COGNITO_USER_POOLS", + "AuthorizerId": { + "Ref": "SpeechToSpeechAuthorizerF61277A4", + }, + "HttpMethod": "POST", + "Integration": { + "IntegrationHttpMethod": "POST", + "Type": "AWS_PROXY", + "Uri": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":apigateway:us-east-1:lambda:path/2015-03-31/functions/", + { + "Fn::GetAtt": [ + "SpeechToSpeechStartSession80A7495E", + "Arn", + ], + }, + "/invocations", + ], + ], + }, + }, + "ResourceId": { + "Ref": "APIApispeechtospeechD6FA255B", + }, + "RestApiId": { + "Ref": "APIApiFFA96F67", + }, + }, + "Type": "AWS::ApiGateway::Method", + "UpdateReplacePolicy": "Delete", + }, + "APIApispeechtospeechPOSTApiPermissionGenerativeAiUseCasesStackAPIApi89219E17POSTspeechtospeech0FB686CB": { + "DeletionPolicy": "Delete", + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "SpeechToSpeechStartSession80A7495E", + "Arn", + ], + }, + "Principal": "apigateway.amazonaws.com", + "SourceArn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":execute-api:us-east-1:123456890123:", + { + "Ref": "APIApiFFA96F67", + }, + "/", + { + "Ref": "APIApiDeploymentStageapiCD55D117", + }, + "/POST/speech-to-speech", + ], + ], + }, + }, + "Type": "AWS::Lambda::Permission", + "UpdateReplacePolicy": "Delete", + }, + "APIApispeechtospeechPOSTApiPermissionTestGenerativeAiUseCasesStackAPIApi89219E17POSTspeechtospeech2C9E93F5": { + "DeletionPolicy": "Delete", + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "SpeechToSpeechStartSession80A7495E", + "Arn", + ], + }, + "Principal": "apigateway.amazonaws.com", + "SourceArn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":execute-api:us-east-1:123456890123:", + { + "Ref": "APIApiFFA96F67", + }, + "/test-invoke-stage/POST/speech-to-speech", + ], + ], + }, + }, + "Type": "AWS::Lambda::Permission", + "UpdateReplacePolicy": "Delete", + }, "APIApisystemcontexts57785227": { "DeletionPolicy": "Delete", "Properties": { @@ -12937,6 +13129,23 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "VITE_APP_SAML_COGNITO_DOMAIN_NAME": "", "VITE_APP_SAML_COGNITO_FEDERATED_IDENTITY_PROVIDER_NAME": "", "VITE_APP_SELF_SIGN_UP_ENABLED": "true", + "VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "Dns.Http", + ], + }, + "/event", + ], + ], + }, + "VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS": "[{"modelId":"amazon.nova-sonic-v1:0","region":"us-east-1"}]", + "VITE_APP_SPEECH_TO_SPEECH_NAMESPACE": "speech-to-speech", "VITE_APP_USER_POOL_CLIENT_ID": { "Ref": "AuthUserPoolclientA74673A9", }, @@ -14941,7 +15150,7 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "Arn", ], }, - "Runtime": "nodejs18.x", + "Runtime": "nodejs20.x", "Timeout": 300, }, "Type": "AWS::Lambda::Function", @@ -15919,6 +16128,349 @@ exports[`GenerativeAiUseCases matches the snapshot 5`] = ` "Type": "AWS::WAFv2::WebACL", "UpdateReplacePolicy": "Delete", }, + "SpeechToSpeechAuthorizerF61277A4": { + "DeletionPolicy": "Delete", + "Properties": { + "IdentitySource": "method.request.header.Authorization", + "Name": "GenerativeAiUseCasesStackSpeechToSpeechAuthorizerC597101F", + "ProviderARNs": [ + { + "Fn::GetAtt": [ + "AuthUserPool8115E87F", + "Arn", + ], + }, + ], + "RestApiId": { + "Ref": "APIApiFFA96F67", + }, + "Type": "COGNITO_USER_POOLS", + }, + "Type": "AWS::ApiGateway::Authorizer", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechChannelNameCA32A058": { + "DeletionPolicy": "Delete", + "Properties": { + "ApiId": { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "ApiId", + ], + }, + "Name": "speech-to-speech", + }, + "Type": "AWS::AppSync::ChannelNamespace", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechEventApi1E2E9AB4": { + "DeletionPolicy": "Delete", + "Properties": { + "EventConfig": { + "AuthProviders": [ + { + "AuthType": "AWS_IAM", + }, + { + "AuthType": "AMAZON_COGNITO_USER_POOLS", + "CognitoConfig": { + "AwsRegion": "us-east-1", + "UserPoolId": { + "Ref": "AuthUserPool8115E87F", + }, + }, + }, + ], + "ConnectionAuthModes": [ + { + "AuthType": "AWS_IAM", + }, + { + "AuthType": "AMAZON_COGNITO_USER_POOLS", + }, + ], + "DefaultPublishAuthModes": [ + { + "AuthType": "AWS_IAM", + }, + { + "AuthType": "AMAZON_COGNITO_USER_POOLS", + }, + ], + "DefaultSubscribeAuthModes": [ + { + "AuthType": "AWS_IAM", + }, + { + "AuthType": "AMAZON_COGNITO_USER_POOLS", + }, + ], + }, + "Name": "SpeechToSpeech", + }, + "Type": "AWS::AppSync::Api", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechStartSession80A7495E": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "SpeechToSpeechStartSessionServiceRoleDefaultPolicy4D6D3AC7", + "SpeechToSpeechStartSessionServiceRoleEBE56984", + ], + "Properties": { + "Code": { + "S3Bucket": "cdk-hnb659fds-assets-123456890123-us-east-1", + "S3Key": "HASH-REPLACED.zip", + }, + "Environment": { + "Variables": { + "SPEECH_TO_SPEECH_TASK_FUNCTION_ARN": { + "Fn::GetAtt": [ + "SpeechToSpeechTaskC1149BF3", + "Arn", + ], + }, + }, + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "SpeechToSpeechStartSessionServiceRoleEBE56984", + "Arn", + ], + }, + "Runtime": "nodejs18.x", + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechStartSessionServiceRoleDefaultPolicy4D6D3AC7": { + "DeletionPolicy": "Delete", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "SpeechToSpeechTaskC1149BF3", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "SpeechToSpeechTaskC1149BF3", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "SpeechToSpeechStartSessionServiceRoleDefaultPolicy4D6D3AC7", + "Roles": [ + { + "Ref": "SpeechToSpeechStartSessionServiceRoleEBE56984", + }, + ], + }, + "Type": "AWS::IAM::Policy", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechStartSessionServiceRoleEBE56984": { + "DeletionPolicy": "Delete", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechTaskC1149BF3": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "SpeechToSpeechTaskServiceRoleDefaultPolicy1048205C", + "SpeechToSpeechTaskServiceRole6B9DD524", + ], + "Properties": { + "Code": { + "S3Bucket": "cdk-hnb659fds-assets-123456890123-us-east-1", + "S3Key": "HASH-REPLACED.zip", + }, + "Environment": { + "Variables": { + "EVENT_API_ENDPOINT": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "Dns.Http", + ], + }, + "/event", + ], + ], + }, + "NAMESPACE": "speech-to-speech", + }, + }, + "Handler": "index.handler", + "MemorySize": 512, + "Role": { + "Fn::GetAtt": [ + "SpeechToSpeechTaskServiceRole6B9DD524", + "Arn", + ], + }, + "Runtime": "nodejs18.x", + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechTaskServiceRole6B9DD524": { + "DeletionPolicy": "Delete", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + "UpdateReplacePolicy": "Delete", + }, + "SpeechToSpeechTaskServiceRoleDefaultPolicy1048205C": { + "DeletionPolicy": "Delete", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "appsync:EventConnect", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":appsync:us-east-1:123456890123:apis/", + { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "ApiId", + ], + }, + ], + ], + }, + }, + { + "Action": [ + "appsync:EventPublish", + "appsync:EventSubscribe", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":appsync:us-east-1:123456890123:apis/", + { + "Fn::GetAtt": [ + "SpeechToSpeechEventApi1E2E9AB4", + "ApiId", + ], + }, + "/channelNamespace/speech-to-speech", + ], + ], + }, + }, + { + "Action": "bedrock:*", + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "SpeechToSpeechTaskServiceRoleDefaultPolicy1048205C", + "Roles": [ + { + "Ref": "SpeechToSpeechTaskServiceRole6B9DD524", + }, + ], + }, + "Type": "AWS::IAM::Policy", + "UpdateReplacePolicy": "Delete", + }, "TranscribeAudioBucket39DFF290": { "DeletionPolicy": "Delete", "Properties": { diff --git a/packages/cdk/test/generative-ai-use-cases.test.ts b/packages/cdk/test/generative-ai-use-cases.test.ts index 2000e5f75..3a526cbab 100644 --- a/packages/cdk/test/generative-ai-use-cases.test.ts +++ b/packages/cdk/test/generative-ai-use-cases.test.ts @@ -41,6 +41,9 @@ describe('GenerativeAiUseCases', () => { videoGenerationModelIds: [ { modelId: 'amazon.nova-reel-v1:0', region: 'us-east-1' }, ], + speechToSpeechModelIds: [ + { modelId: 'amazon.nova-sonic-v1:0', region: 'us-east-1' }, + ], endpointNames: [], agentEnabled: true, searchAgentEnabled: true, diff --git a/packages/common/src/application/model.ts b/packages/common/src/application/model.ts index db866fbed..c481ffd8c 100644 --- a/packages/common/src/application/model.ts +++ b/packages/common/src/application/model.ts @@ -19,6 +19,7 @@ const MODEL_FEATURE: Record = { VIDEO_GEN: { video_gen: true }, EMBEDDING: { embedding: true }, RERANKING: { reranking: true }, + SPEECH_TO_SPEECH: { speechToSpeech: true }, // Additional Flags LIGHT: { light: true }, }; @@ -182,6 +183,11 @@ export const modelFeatureFlags: Record = { 'amazon.rerank-v1:0': MODEL_FEATURE.RERANKING, // Cohere 'cohere.rerank-v3-5:0': MODEL_FEATURE.RERANKING, + + // === Speech to Speech === + + // Amazon + 'amazon.nova-sonic-v1:0': MODEL_FEATURE.SPEECH_TO_SPEECH, }; export const BEDROCK_TEXT_MODELS = Object.keys(modelFeatureFlags).filter( @@ -199,3 +205,6 @@ export const BEDROCK_EMBEDDING_MODELS = Object.keys(modelFeatureFlags).filter( export const BEDROCK_RERANKING_MODELS = Object.keys(modelFeatureFlags).filter( (model) => modelFeatureFlags[model].reranking ); +export const BEDROCK_SPEECH_TO_SPEECH_MODELS = Object.keys( + modelFeatureFlags +).filter((model) => modelFeatureFlags[model].speechToSpeech); diff --git a/packages/types/src/model.d.ts b/packages/types/src/model.d.ts index b740e29c3..b4fa2f3a3 100644 --- a/packages/types/src/model.d.ts +++ b/packages/types/src/model.d.ts @@ -12,6 +12,9 @@ export type FeatureFlags = { embedding?: boolean; reranking?: boolean; + + speechToSpeech?: boolean; + // Additional Flags light?: boolean; }; diff --git a/packages/web/public/locales/translation/en.yaml b/packages/web/public/locales/translation/en.yaml index 89bd55f66..d8fbdddb2 100644 --- a/packages/web/public/locales/translation/en.yaml +++ b/packages/web/public/locales/translation/en.yaml @@ -624,6 +624,7 @@ navigation: ragChat: RAG Chat settings: Settings speechRecognition: Speech Recognition + speechToSpeech: Speech to Speech summary: Summary textGeneration: Text Generation translation: Translation @@ -726,6 +727,14 @@ sketch: clear: Clear pen_size: Pen Size upload_image: Upload Image +speech_to_speech: + close: Close session + default_system_prompt: You are an AI assistant. + experimental_warning: >- + Speech to Speech is still in an experimental stage. The architecture, etc. may be changed in the future. Conversation history will not be saved. The supported language is English only. + experimental_warning_title: About Speech to Speech + start: Start new session + title: Speech to Speech summarize: additional_context: Additional context additional_context_placeholder: 'You can enter additional points to consider (e.g., casualness, etc.)' diff --git a/packages/web/public/locales/translation/ja.yaml b/packages/web/public/locales/translation/ja.yaml index d3d03e684..e550a8461 100644 --- a/packages/web/public/locales/translation/ja.yaml +++ b/packages/web/public/locales/translation/ja.yaml @@ -543,6 +543,7 @@ navigation: ragChat: RAG チャット settings: 設定 speechRecognition: 音声認識 + speechToSpeech: 音声チャット summary: 要約 textGeneration: 文章生成 translation: 翻訳 @@ -626,6 +627,14 @@ sketch: clear: クリア pen_size: ペンサイズ upload_image: 画像をアップロード +speech_to_speech: + close: セッションを終了する + default_system_prompt: You are an AI assistant. + experimental_warning: >- + 音声チャットはまだ実験的な段階です。アーキテクチャ等は今後変更される可能性があります。会話履歴は保存されません。対応言語は英語のみです。 + experimental_warning_title: 音声チャットについて + start: セッションを始める + title: 音声チャット summarize: additional_context: 追加コンテキスト additional_context_placeholder: 追加で考慮してほしい点を入力することができます（カジュアルさ等） diff --git a/packages/web/src/App.tsx b/packages/web/src/App.tsx index cae7dba39..2b76c34cd 100644 --- a/packages/web/src/App.tsx +++ b/packages/web/src/App.tsx @@ -19,6 +19,7 @@ import { PiVideoCamera, PiFlowArrow, PiMagicWand, + PiMicrophoneBold, PiTreeStructure, } from 'react-icons/pi'; import { Outlet } from 'react-router-dom'; @@ -131,6 +132,12 @@ const App: React.FC = () => { display: 'usecase' as const, } : null, + { + label: t('navigation.speechToSpeech'), + to: '/speech-to-speech', + icon: , + display: 'usecase' as const, + }, enabled('generate') ? { label: t('navigation.textGeneration'), diff --git a/packages/web/src/hooks/useModel.ts b/packages/web/src/hooks/useModel.ts index 778e04c08..6bd8fe003 100644 --- a/packages/web/src/hooks/useModel.ts +++ b/packages/web/src/hooks/useModel.ts @@ -57,6 +57,21 @@ const videoModelConfigs = ( const videoGenModelIds: string[] = videoModelConfigs.map( (model) => model.modelId ); +const speechToSpeechModelConfigs = ( + JSON.parse( + import.meta.env.VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS + ) as ModelConfiguration[] +) + .map( + (model: ModelConfiguration): ModelConfiguration => ({ + modelId: model.modelId.trim(), + region: model.region.trim(), + }) + ) + .filter((model) => model.modelId); +const speechToSpeechModelIds: string[] = speechToSpeechModelConfigs.map( + (model) => model.modelId +); const agentNames: string[] = JSON.parse(import.meta.env.VITE_APP_AGENT_NAMES) .map((name: string) => name.trim()) @@ -106,6 +121,16 @@ const videoGenModels = [ }) as Model ), ]; +const speechToSpeechModels = [ + ...speechToSpeechModelConfigs.map( + (model) => + ({ + modelId: model.modelId, + type: 'bedrock', + region: model.region, + }) as Model + ), +]; const agentModels = [ ...agentNames.map( (name) => ({ modelId: name, type: 'bedrockAgent' }) as Model @@ -148,4 +173,6 @@ export const MODELS = { searchAgent: searchAgent, flows, flowChatEnabled: flows.length > 0, + speechToSpeechModelIds: speechToSpeechModelIds, + speechToSpeechModels: speechToSpeechModels, }; diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js index 4ca74abb5..4976f8890 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js +++ b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js @@ -48,7 +48,7 @@ class ExpandableBuffer { if (this.writeIndex - this.readIndex >= this.initialBufferLength) { // Filled the initial buffer length, so we can start playback with some cushion this.isInitialBuffering = false; - // console.log("Initial audio buffer filled"); + // console.log('Initial audio buffer filled'); } } diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js index 6fdec7f59..3dda54db9 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js +++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js @@ -47,28 +47,34 @@ export class AudioRecorder { const errorMessage = error.message || 'Failed to access microphone'; // Notify error listeners - this.onErrorListeners.forEach(listener => listener({ - type: errorType, - message: errorMessage, - originalError: error - })); + this.onErrorListeners.forEach((listener) => + listener({ + type: errorType, + message: errorMessage, + originalError: error, + }) + ); // Don't throw, just return false to indicate failure console.error('Microphone access error:', errorType, errorMessage); return false; } - this.sourceNode = this.audioContext.createMediaStreamSource(this.audioStream); + this.sourceNode = this.audioContext.createMediaStreamSource( + this.audioStream + ); // Add the audio worklet module try { await this.audioContext.audioWorklet.addModule(AudioRecorderWorkletUrl); } catch (error) { - this.onErrorListeners.forEach(listener => listener({ - type: 'WorkletError', - message: 'Failed to load audio worklet', - originalError: error - })); + this.onErrorListeners.forEach((listener) => + listener({ + type: 'WorkletError', + message: 'Failed to load audio worklet', + originalError: error, + }) + ); this.cleanup(); return false; } @@ -87,24 +93,28 @@ export class AudioRecorder { if (event.data.type === 'audio') { const audioData = event.data.audioData; // Notify listeners that audio was recorded - this.onAudioRecordedListeners.forEach(listener => listener(audioData)); + this.onAudioRecordedListeners.forEach((listener) => + listener(audioData) + ); } }; // Start recording this.workletNode.port.postMessage({ - type: 'start' + type: 'start', }); this.initialized = true; return true; } catch (error) { // Catch any other unexpected errors - this.onErrorListeners.forEach(listener => listener({ - type: 'InitializationError', - message: 'Failed to initialize audio recorder', - originalError: error - })); + this.onErrorListeners.forEach((listener) => + listener({ + type: 'InitializationError', + message: 'Failed to initialize audio recorder', + originalError: error, + }) + ); this.cleanup(); return false; } @@ -129,7 +139,7 @@ export class AudioRecorder { if (ObjectExt.exists(this.audioStream)) { try { - this.audioStream.getTracks().forEach(track => track.stop()); + this.audioStream.getTracks().forEach((track) => track.stop()); } catch (e) { console.error('Error stopping audio tracks:', e); } @@ -155,7 +165,7 @@ export class AudioRecorder { // Stop recording if (ObjectExt.exists(this.workletNode)) { this.workletNode.port.postMessage({ - type: 'stop' + type: 'stop', }); } diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js index 9ede38c7b..4cba5c70c 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js +++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js @@ -21,7 +21,7 @@ class AudioRecorderProcessor extends AudioWorkletProcessor { // Send the audio data to the main thread this.port.postMessage({ type: 'audio', - audioData: input.slice() + audioData: input.slice(), }); return true; diff --git a/packages/web/src/hooks/useSpeechToSpeech/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts index b5b2bae1e..0efba412c 100644 --- a/packages/web/src/hooks/useSpeechToSpeech/index.ts +++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts @@ -8,6 +8,7 @@ import useChatHistory from './useChatHistory'; import { SpeechToSpeechEventType, SpeechToSpeechEvent, + Model, } from 'generative-ai-use-cases'; const NAMESPACE = import.meta.env.VITE_APP_SPEECH_TO_SPEECH_NAMESPACE!; @@ -100,22 +101,34 @@ export const useSpeechToSpeech = () => { audioPlayerRef.current = audioPlayer; const audioRecorder = new AudioRecorder(); - audioRecorder.addEventListener('onAudioRecorded', (audioData: Float32Array) => { - const int16Array = float32ArrayToInt16Array(audioData); - const base64Data = arrayBufferToBase64(int16Array.buffer); - audioInputQueue.current.push(base64Data); - }); + audioRecorder.addEventListener( + 'onAudioRecorded', + (audioData: Float32Array) => { + const int16Array = float32ArrayToInt16Array(audioData); + const base64Data = arrayBufferToBase64(int16Array.buffer); + audioInputQueue.current.push(base64Data); + } + ); // Add error listener to handle microphone permission issues - audioRecorder.addEventListener('onError', (error: { type: string; message: string }) => { - console.error('Audio recorder error:', error.type, error.message); - // You can add UI notification here if needed - if (error.type === 'NotAllowedError' || error.type === 'PermissionDeniedError') { - // Handle microphone permission denied specifically - resetState(); - setErrorMessages([...errorMessages, 'The microphone is not available. Please grant permission to use the microphone.']); + audioRecorder.addEventListener( + 'onError', + (error: { type: string; message: string }) => { + console.error('Audio recorder error:', error.type, error.message); + // You can add UI notification here if needed + if ( + error.type === 'NotAllowedError' || + error.type === 'PermissionDeniedError' + ) { + // Handle microphone permission denied specifically + resetState(); + setErrorMessages([ + ...errorMessages, + 'The microphone is not available. Please grant permission to use the microphone.', + ]); + } } - }); + ); audioRecorderRef.current = audioRecorder; }; @@ -144,7 +157,7 @@ export const useSpeechToSpeech = () => { setTimeout(() => processAudioInput(), 0); }; - const connectToAppSync = async () => { + const connectToAppSync = async (model: Model) => { audioInputQueue.current = []; const channelId = uuid(); @@ -190,11 +203,15 @@ export const useSpeechToSpeech = () => { }, }); - await api.post('speech-to-speech', { channel: channelId }); + await api.post('speech-to-speech', { channel: channelId, model }); }; const startRecording = async () => { - if (!audioPlayerRef.current || !audioRecorderRef.current || !systemPromptRef.current) { + if ( + !audioPlayerRef.current || + !audioRecorderRef.current || + !systemPromptRef.current + ) { return; } @@ -234,7 +251,7 @@ export const useSpeechToSpeech = () => { await dispatchEvent('audioStop'); }; - const startSession = async (systemPrompt: string) => { + const startSession = async (systemPrompt: string, model: Model) => { if (isActive || isLoading) { return; } @@ -245,7 +262,7 @@ export const useSpeechToSpeech = () => { systemPromptRef.current = systemPrompt; - await connectToAppSync(); + await connectToAppSync(model); await initAudio(); }; diff --git a/packages/web/src/pages/SpeechToSpeechPage.tsx b/packages/web/src/pages/SpeechToSpeechPage.tsx index add378500..3f5e91394 100644 --- a/packages/web/src/pages/SpeechToSpeechPage.tsx +++ b/packages/web/src/pages/SpeechToSpeechPage.tsx @@ -13,9 +13,11 @@ import Button from '../components/Button'; import InputChatContent from '../components/InputChatContent'; import ScrollTopBottom from '../components/ScrollTopBottom'; import Alert from '../components/Alert.tsx'; +import Select from '../components/Select'; import useFollow from '../hooks/useFollow'; import BedrockIcon from '../assets/bedrock.svg?react'; import { toast } from 'sonner'; +import { MODELS } from '../hooks/useModel'; const SpeechToSpeech: React.FC = () => { const { t } = useTranslation(); @@ -29,10 +31,13 @@ const SpeechToSpeech: React.FC = () => { errorMessages, } = useSpeechToSpeech(); const [showSystemPrompt, setShowSystemPrompt] = useState(false); - // TODO: avoid hardcoding - const [systemPrompt, setSystemPrompt] = useState('You are an AI assistant.'); + const [systemPrompt, setSystemPrompt] = useState( + t('speech_to_speech.default_system_prompt') + ); const [inputSystemPrompt, setInputSystemPrompt] = useState(systemPrompt); const { scrollableContainer, setFollowing } = useFollow(); + const { speechToSpeechModelIds, speechToSpeechModels } = MODELS; + const [modelId, setModelId] = useState(speechToSpeechModelIds[0]); // TODO (query parameters?) const messagesWithoutSystemPrompt = useMemo(() => { return messages.filter((m) => m.role !== 'system'); @@ -60,17 +65,28 @@ const SpeechToSpeech: React.FC = () => { <>
- Speech to Speech + {t('speech_to_speech.title')}
- {isEmpty && ( + {isEmpty && !isLoading && ( +
+