Flow Chat
+
+
+ 
+
+
+ Voice Chat
++ In Voice Chat, you can have a bidirectional voice chat with + generative AI. Similar to natural conversation, you can also + interrupt and speak while the AI is talking. Also, by setting a + system prompt, you can have voice conversations with AI that has + specific roles. +
+
+
+
+
+
+ 音声チャット
++ 音声チャットでは生成 AI + と双方向の音声によるチャットが可能です。自然な会話と同様、AI + の発言中に割り込んで話すこともできます。また、システムプロンプトを設定することで、特定の役割を持った + AI と音声で会話することもできます。 +
+
- {chatContent?.role === 'system' && (
+ {chatContent?.role === 'system' && !props.hideSaveSystemContext && (
{
diff --git a/packages/web/src/hooks/useModel.ts b/packages/web/src/hooks/useModel.ts
index 778e04c08..6bd8fe003 100644
--- a/packages/web/src/hooks/useModel.ts
+++ b/packages/web/src/hooks/useModel.ts
@@ -57,6 +57,21 @@ const videoModelConfigs = (
const videoGenModelIds: string[] = videoModelConfigs.map(
(model) => model.modelId
);
+const speechToSpeechModelConfigs = (
+ JSON.parse(
+ import.meta.env.VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS
+ ) as ModelConfiguration[]
+)
+ .map(
+ (model: ModelConfiguration): ModelConfiguration => ({
+ modelId: model.modelId.trim(),
+ region: model.region.trim(),
+ })
+ )
+ .filter((model) => model.modelId);
+const speechToSpeechModelIds: string[] = speechToSpeechModelConfigs.map(
+ (model) => model.modelId
+);
const agentNames: string[] = JSON.parse(import.meta.env.VITE_APP_AGENT_NAMES)
.map((name: string) => name.trim())
@@ -106,6 +121,16 @@ const videoGenModels = [
}) as Model
),
];
+const speechToSpeechModels = [
+ ...speechToSpeechModelConfigs.map(
+ (model) =>
+ ({
+ modelId: model.modelId,
+ type: 'bedrock',
+ region: model.region,
+ }) as Model
+ ),
+];
const agentModels = [
...agentNames.map(
(name) => ({ modelId: name, type: 'bedrockAgent' }) as Model
@@ -148,4 +173,6 @@ export const MODELS = {
searchAgent: searchAgent,
flows,
flowChatEnabled: flows.length > 0,
+ speechToSpeechModelIds: speechToSpeechModelIds,
+ speechToSpeechModels: speechToSpeechModels,
};
diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioPlayer.js b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayer.js
new file mode 100644
index 000000000..840b17b4c
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayer.js
@@ -0,0 +1,144 @@
+// This file is copied from https://github.com/aws-samples/amazon-nova-samples/blob/main/speech-to-speech/sample-codes/websocket-nodejs/public/src/lib/play/AudioPlayer.js
+import { ObjectExt } from './ObjectsExt.js';
+const AudioPlayerWorkletUrl = new URL(
+ './AudioPlayerProcessor.worklet.js',
+ import.meta.url
+).toString();
+
+export class AudioPlayer {
+ constructor() {
+ this.onAudioPlayedListeners = [];
+ this.initialized = false;
+ }
+
+ addEventListener(event, callback) {
+ switch (event) {
+ case 'onAudioPlayed':
+ this.onAudioPlayedListeners.push(callback);
+ break;
+ default:
+ console.error(
+ 'Listener registered for event type: ' +
+ JSON.stringify(event) +
+ ' which is not supported'
+ );
+ }
+ }
+
+ async start() {
+ this.audioContext = new AudioContext({ sampleRate: 24000 });
+ this.analyser = this.audioContext.createAnalyser();
+ this.analyser.fftSize = 512;
+
+ // Chrome caches worklet code more aggressively, so add a nocache parameter to make sure we get the latest
+ await this.audioContext.audioWorklet.addModule(AudioPlayerWorkletUrl); // + "?nocache=" + Date.now());
+ this.workletNode = new AudioWorkletNode(
+ this.audioContext,
+ 'audio-player-processor'
+ );
+ this.workletNode.connect(this.analyser);
+ this.analyser.connect(this.audioContext.destination);
+ this.recorderNode = this.audioContext.createScriptProcessor(512, 1, 1);
+ this.recorderNode.onaudioprocess = (event) => {
+ // Pass the input along as-is
+ const inputData = event.inputBuffer.getChannelData(0);
+ const outputData = event.outputBuffer.getChannelData(0);
+ outputData.set(inputData);
+ // Notify listeners that the audio was played
+ const samples = new Float32Array(outputData.length);
+ samples.set(outputData);
+ this.onAudioPlayedListeners.map((listener) => listener(samples));
+ };
+ this.#maybeOverrideInitialBufferLength();
+ this.initialized = true;
+ }
+
+ bargeIn() {
+ this.workletNode.port.postMessage({
+ type: 'barge-in',
+ });
+ }
+
+ stop() {
+ if (ObjectExt.exists(this.audioContext)) {
+ this.audioContext.close();
+ }
+
+ if (ObjectExt.exists(this.analyser)) {
+ this.analyser.disconnect();
+ }
+
+ if (ObjectExt.exists(this.workletNode)) {
+ this.workletNode.disconnect();
+ }
+
+ if (ObjectExt.exists(this.recorderNode)) {
+ this.recorderNode.disconnect();
+ }
+
+ this.initialized = false;
+ this.audioContext = null;
+ this.analyser = null;
+ this.workletNode = null;
+ this.recorderNode = null;
+ }
+
+ #maybeOverrideInitialBufferLength() {
+ // Read a user-specified initial buffer length from the URL parameters to help with tinkering
+ const params = new URLSearchParams(window.location.search);
+ const value = params.get('audioPlayerInitialBufferLength');
+ if (value === null) {
+ return; // No override specified
+ }
+ const bufferLength = parseInt(value);
+ if (isNaN(bufferLength)) {
+ console.error(
+ 'Invalid audioPlayerInitialBufferLength value:',
+ JSON.stringify(value)
+ );
+ return;
+ }
+ this.workletNode.port.postMessage({
+ type: 'initial-buffer-length',
+ bufferLength: bufferLength,
+ });
+ }
+
+ playAudio(samples) {
+ if (!this.initialized) {
+ console.error(
+ 'The audio player is not initialized. Call init() before attempting to play audio.'
+ );
+ return;
+ }
+ this.workletNode.port.postMessage({
+ type: 'audio',
+ audioData: samples,
+ });
+ }
+
+ getSamples() {
+ if (!this.initialized) {
+ return null;
+ }
+ const bufferLength = this.analyser.frequencyBinCount;
+ const dataArray = new Uint8Array(bufferLength);
+ this.analyser.getByteTimeDomainData(dataArray);
+ return [...dataArray].map((e) => e / 128 - 1);
+ }
+
+ getVolume() {
+ if (!this.initialized) {
+ return 0;
+ }
+ const bufferLength = this.analyser.frequencyBinCount;
+ const dataArray = new Uint8Array(bufferLength);
+ this.analyser.getByteTimeDomainData(dataArray);
+ let normSamples = [...dataArray].map((e) => e / 128 - 1);
+ let sum = 0;
+ for (let i = 0; i < normSamples.length; i++) {
+ sum += normSamples[i] * normSamples[i];
+ }
+ return Math.sqrt(sum / normSamples.length);
+ }
+}
diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js
new file mode 100644
index 000000000..b2592a6e6
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/AudioPlayerProcessor.worklet.js
@@ -0,0 +1,116 @@
+// This file is copied from https://github.com/aws-samples/amazon-nova-samples/blob/main/speech-to-speech/sample-codes/websocket-nodejs/public/src/lib/play/AudioPlayerProcessor.worklet.js
+// Audio sample buffer to minimize reallocations
+class ExpandableBuffer {
+ constructor() {
+ // Start with one second's worth of buffered audio capacity before needing to expand
+ this.buffer = new Float32Array(24000);
+ this.readIndex = 0;
+ this.writeIndex = 0;
+ this.underflowedSamples = 0;
+ this.isInitialBuffering = true;
+ this.initialBufferLength = 24000; // One second
+ this.lastWriteTime = 0;
+ }
+
+ logTimeElapsedSinceLastWrite() {
+ const now = Date.now();
+ if (this.lastWriteTime !== 0) {
+ const elapsed = now - this.lastWriteTime;
+ // console.log(`Elapsed time since last audio buffer write: ${elapsed} ms`);
+ }
+ this.lastWriteTime = now;
+ }
+
+ write(samples) {
+ this.logTimeElapsedSinceLastWrite();
+ if (this.writeIndex + samples.length <= this.buffer.length) {
+ // Enough space to append the new samples
+ } else {
+ // Not enough space ...
+ if (samples.length <= this.readIndex) {
+ // ... but we can shift samples to the beginning of the buffer
+ const subarray = this.buffer.subarray(this.readIndex, this.writeIndex);
+ // console.log(`Shifting the audio buffer of length ${subarray.length} by ${this.readIndex}`);
+ this.buffer.set(subarray);
+ } else {
+ // ... and we need to grow the buffer capacity to make room for more audio
+ const newLength =
+ (samples.length + this.writeIndex - this.readIndex) * 2;
+ const newBuffer = new Float32Array(newLength);
+ // console.log(`Expanding the audio buffer from ${this.buffer.length} to ${newLength}`);
+ newBuffer.set(this.buffer.subarray(this.readIndex, this.writeIndex));
+ this.buffer = newBuffer;
+ }
+ this.writeIndex -= this.readIndex;
+ this.readIndex = 0;
+ }
+ this.buffer.set(samples, this.writeIndex);
+ this.writeIndex += samples.length;
+ if (this.writeIndex - this.readIndex >= this.initialBufferLength) {
+ // Filled the initial buffer length, so we can start playback with some cushion
+ this.isInitialBuffering = false;
+ // console.log('Initial audio buffer filled');
+ }
+ }
+
+ read(destination) {
+ let copyLength = 0;
+ if (!this.isInitialBuffering) {
+ // Only start to play audio after we've built up some initial cushion
+ copyLength = Math.min(
+ destination.length,
+ this.writeIndex - this.readIndex
+ );
+ }
+ destination.set(
+ this.buffer.subarray(this.readIndex, this.readIndex + copyLength)
+ );
+ this.readIndex += copyLength;
+ if (copyLength > 0 && this.underflowedSamples > 0) {
+ // console.log(`Detected audio buffer underflow of ${this.underflowedSamples} samples`);
+ this.underflowedSamples = 0;
+ }
+ if (copyLength < destination.length) {
+ // Not enough samples (buffer underflow). Fill the rest with silence.
+ destination.fill(0, copyLength);
+ this.underflowedSamples += destination.length - copyLength;
+ }
+ if (copyLength === 0) {
+ // Ran out of audio, so refill the buffer to the initial length before playing more
+ this.isInitialBuffering = true;
+ }
+ }
+
+ clearBuffer() {
+ this.readIndex = 0;
+ this.writeIndex = 0;
+ }
+}
+
+class AudioPlayerProcessor extends AudioWorkletProcessor {
+ constructor() {
+ super();
+ this.playbackBuffer = new ExpandableBuffer();
+ this.port.onmessage = (event) => {
+ if (event.data.type === 'audio') {
+ this.playbackBuffer.write(event.data.audioData);
+ } else if (event.data.type === 'initial-buffer-length') {
+ // Override the current playback initial buffer length
+ const newLength = event.data.bufferLength;
+ this.playbackBuffer.initialBufferLength = newLength;
+ // amazonq-ignore-next-line
+ // console.log(`Changed initial audio buffer length to: ${newLength}`)
+ } else if (event.data.type === 'barge-in') {
+ this.playbackBuffer.clearBuffer();
+ }
+ };
+ }
+
+ process(inputs, outputs, parameters) {
+ const output = outputs[0][0]; // Assume one output with one channel
+ this.playbackBuffer.read(output);
+ return true; // True to continue processing
+ }
+}
+
+registerProcessor('audio-player-processor', AudioPlayerProcessor);
diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js
new file mode 100644
index 000000000..3dda54db9
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorder.js
@@ -0,0 +1,175 @@
+import { ObjectExt } from './ObjectsExt.js';
+const AudioRecorderWorkletUrl = new URL(
+ './AudioRecorderProcessor.worklet.js',
+ import.meta.url
+).toString();
+
+export class AudioRecorder {
+ constructor() {
+ this.onAudioRecordedListeners = [];
+ this.onErrorListeners = [];
+ this.initialized = false;
+ }
+
+ addEventListener(event, callback) {
+ switch (event) {
+ case 'onAudioRecorded':
+ this.onAudioRecordedListeners.push(callback);
+ break;
+ case 'onError':
+ this.onErrorListeners.push(callback);
+ break;
+ default:
+ console.error(
+ 'Listener registered for event type: ' +
+ JSON.stringify(event) +
+ ' which is not supported'
+ );
+ }
+ }
+
+ async start() {
+ try {
+ this.audioContext = new AudioContext({ sampleRate: 16000 });
+
+ // Get user media stream
+ try {
+ this.audioStream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ },
+ });
+ } catch (error) {
+ // Handle permission denied or device not available errors
+ const errorType = error.name || 'UnknownError';
+ const errorMessage = error.message || 'Failed to access microphone';
+
+ // Notify error listeners
+ this.onErrorListeners.forEach((listener) =>
+ listener({
+ type: errorType,
+ message: errorMessage,
+ originalError: error,
+ })
+ );
+
+ // Don't throw, just return false to indicate failure
+ console.error('Microphone access error:', errorType, errorMessage);
+ return false;
+ }
+
+ this.sourceNode = this.audioContext.createMediaStreamSource(
+ this.audioStream
+ );
+
+ // Add the audio worklet module
+ try {
+ await this.audioContext.audioWorklet.addModule(AudioRecorderWorkletUrl);
+ } catch (error) {
+ this.onErrorListeners.forEach((listener) =>
+ listener({
+ type: 'WorkletError',
+ message: 'Failed to load audio worklet',
+ originalError: error,
+ })
+ );
+ this.cleanup();
+ return false;
+ }
+
+ this.workletNode = new AudioWorkletNode(
+ this.audioContext,
+ 'audio-recorder-processor'
+ );
+
+ // Connect the source to the worklet
+ this.sourceNode.connect(this.workletNode);
+ this.workletNode.connect(this.audioContext.destination);
+
+ // Listen for audio data from the worklet
+ this.workletNode.port.onmessage = (event) => {
+ if (event.data.type === 'audio') {
+ const audioData = event.data.audioData;
+ // Notify listeners that audio was recorded
+ this.onAudioRecordedListeners.forEach((listener) =>
+ listener(audioData)
+ );
+ }
+ };
+
+ // Start recording
+ this.workletNode.port.postMessage({
+ type: 'start',
+ });
+
+ this.initialized = true;
+ return true;
+ } catch (error) {
+ // Catch any other unexpected errors
+ this.onErrorListeners.forEach((listener) =>
+ listener({
+ type: 'InitializationError',
+ message: 'Failed to initialize audio recorder',
+ originalError: error,
+ })
+ );
+ this.cleanup();
+ return false;
+ }
+ }
+
+ cleanup() {
+ if (ObjectExt.exists(this.workletNode)) {
+ try {
+ this.workletNode.disconnect();
+ } catch (e) {
+ console.error('Error disconnecting worklet node:', e);
+ }
+ }
+
+ if (ObjectExt.exists(this.sourceNode)) {
+ try {
+ this.sourceNode.disconnect();
+ } catch (e) {
+ console.error('Error disconnecting source node:', e);
+ }
+ }
+
+ if (ObjectExt.exists(this.audioStream)) {
+ try {
+ this.audioStream.getTracks().forEach((track) => track.stop());
+ } catch (e) {
+ console.error('Error stopping audio tracks:', e);
+ }
+ }
+
+ if (ObjectExt.exists(this.audioContext)) {
+ try {
+ this.audioContext.close();
+ } catch (e) {
+ console.error('Error closing audio context:', e);
+ }
+ }
+
+ this.initialized = false;
+ this.audioContext = null;
+ this.audioStream = null;
+ this.sourceNode = null;
+ this.workletNode = null;
+ }
+
+ stop() {
+ if (this.initialized) {
+ // Stop recording
+ if (ObjectExt.exists(this.workletNode)) {
+ this.workletNode.port.postMessage({
+ type: 'stop',
+ });
+ }
+
+ this.cleanup();
+ }
+ }
+}
diff --git a/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js
new file mode 100644
index 000000000..4cba5c70c
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/AudioRecorderProcessor.worklet.js
@@ -0,0 +1,31 @@
+class AudioRecorderProcessor extends AudioWorkletProcessor {
+ constructor() {
+ super();
+ this.isRecording = false;
+ this.port.onmessage = (event) => {
+ if (event.data.type === 'start') {
+ this.isRecording = true;
+ } else if (event.data.type === 'stop') {
+ this.isRecording = false;
+ }
+ };
+ }
+
+ process(inputs, outputs, parameters) {
+ if (!this.isRecording || !inputs[0] || !inputs[0][0]) {
+ return true;
+ }
+
+ const input = inputs[0][0];
+
+ // Send the audio data to the main thread
+ this.port.postMessage({
+ type: 'audio',
+ audioData: input.slice(),
+ });
+
+ return true;
+ }
+}
+
+registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
diff --git a/packages/web/src/hooks/useSpeechToSpeech/ObjectsExt.js b/packages/web/src/hooks/useSpeechToSpeech/ObjectsExt.js
new file mode 100644
index 000000000..6299473ff
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/ObjectsExt.js
@@ -0,0 +1,18 @@
+// This file is copied from https://github.com/aws-samples/amazon-nova-samples/blob/main/speech-to-speech/sample-codes/websocket-nodejs/public/src/lib/util/ObjectsExt.js
+export class ObjectExt {
+ static exists(obj) {
+ return obj !== undefined && obj !== null;
+ }
+
+ static checkArgument(condition, message) {
+ if (!condition) {
+ throw TypeError(message);
+ }
+ }
+
+ static checkExists(obj, message) {
+ if (ObjectsExt.exists(obj)) {
+ throw TypeError(message);
+ }
+ }
+}
diff --git a/packages/web/src/hooks/useSpeechToSpeech/index.ts b/packages/web/src/hooks/useSpeechToSpeech/index.ts
new file mode 100644
index 000000000..4fdadda81
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/index.ts
@@ -0,0 +1,290 @@
+import { useRef, useState } from 'react';
+import { events, EventsChannel } from 'aws-amplify/data';
+import { AudioPlayer } from './AudioPlayer';
+import { AudioRecorder } from './AudioRecorder';
+import { v4 as uuid } from 'uuid';
+import useHttp from '../../hooks/useHttp';
+import useChatHistory from './useChatHistory';
+import {
+ SpeechToSpeechEventType,
+ SpeechToSpeechEvent,
+ Model,
+} from 'generative-ai-use-cases';
+
+const NAMESPACE = import.meta.env.VITE_APP_SPEECH_TO_SPEECH_NAMESPACE!;
+const MIN_AUDIO_CHUNKS_PER_BATCH = 10;
+const MAX_AUDIO_CHUNKS_PER_BATCH = 20;
+
+const arrayBufferToBase64 = (buffer: ArrayBuffer) => {
+ const binary = [];
+ const bytes = new Uint8Array(buffer);
+ for (let i = 0; i < bytes.byteLength; i++) {
+ binary.push(String.fromCharCode(bytes[i]));
+ }
+ return btoa(binary.join(''));
+};
+
+const float32ArrayToInt16Array = (float32Array: Float32Array): Int16Array => {
+ const int16Array = new Int16Array(float32Array.length);
+ for (let i = 0; i < float32Array.length; i++) {
+ int16Array[i] = Math.max(-1, Math.min(1, float32Array[i])) * 0x7fff;
+ }
+ return int16Array;
+};
+
+const base64ToFloat32Array = (base64String: string) => {
+ try {
+ const binaryString = atob(base64String);
+ const bytes = new Uint8Array(binaryString.length);
+ for (let i = 0; i < binaryString.length; i++) {
+ bytes[i] = binaryString.charCodeAt(i);
+ }
+
+ const int16Array = new Int16Array(bytes.buffer);
+ const float32Array = new Float32Array(int16Array.length);
+ for (let i = 0; i < int16Array.length; i++) {
+ float32Array[i] = int16Array[i] / 32768.0;
+ }
+
+ return float32Array;
+ } catch (error) {
+ console.error('Error in base64ToFloat32Array:', error);
+ throw error;
+ }
+};
+
+export const useSpeechToSpeech = () => {
+ const api = useHttp();
+ const {
+ clear,
+ messages,
+ setupSystemPrompt,
+ onTextStart,
+ onTextOutput,
+ onTextStop,
+ isAssistantSpeeching,
+ } = useChatHistory();
+ const [isActive, setIsActive] = useState(false);
+ const [isLoading, setIsLoading] = useState(false);
+ const systemPromptRef = useRef('');
+ const audioPlayerRef = useRef(null);
+ const audioRecorderRef = useRef(null);
+ const channelRef = useRef(null);
+ const audioInputQueue = useRef([]);
+ const [errorMessages, setErrorMessages] = useState([]);
+
+ const resetState = () => {
+ setIsLoading(false);
+ setIsActive(false);
+ audioRecorderRef.current = null;
+ audioPlayerRef.current = null;
+ channelRef.current = null;
+ audioInputQueue.current = [];
+ };
+
+ const dispatchEvent = async (
+ event: SpeechToSpeechEventType,
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ data: any = undefined
+ ) => {
+ if (channelRef.current) {
+ await channelRef.current.publish({
+ direction: 'ctob',
+ event,
+ data,
+ } as SpeechToSpeechEvent);
+ }
+ };
+
+ const initAudio = async () => {
+ const audioPlayer = new AudioPlayer();
+ audioPlayerRef.current = audioPlayer;
+
+ const audioRecorder = new AudioRecorder();
+ audioRecorder.addEventListener(
+ 'onAudioRecorded',
+ (audioData: Float32Array) => {
+ const int16Array = float32ArrayToInt16Array(audioData);
+ const base64Data = arrayBufferToBase64(int16Array.buffer);
+ audioInputQueue.current.push(base64Data);
+ }
+ );
+
+ // Add error listener to handle microphone permission issues
+ audioRecorder.addEventListener(
+ 'onError',
+ (error: { type: string; message: string }) => {
+ console.error('Audio recorder error:', error.type, error.message);
+ // You can add UI notification here if needed
+ if (
+ error.type === 'NotAllowedError' ||
+ error.type === 'PermissionDeniedError'
+ ) {
+ // Handle microphone permission denied specifically
+ resetState();
+ setErrorMessages([
+ ...errorMessages,
+ 'The microphone is not available. Please grant permission to use the microphone.',
+ ]);
+ }
+ }
+ );
+
+ audioRecorderRef.current = audioRecorder;
+ };
+
+ const processAudioInput = async () => {
+ if (audioInputQueue.current.length > MIN_AUDIO_CHUNKS_PER_BATCH) {
+ const chunksToProcess: string[] = [];
+
+ let processedChunks = 0;
+
+ while (
+ audioInputQueue.current.length > 0 &&
+ processedChunks < MAX_AUDIO_CHUNKS_PER_BATCH
+ ) {
+ const chunk = audioInputQueue.current.shift();
+
+ if (chunk) {
+ chunksToProcess.push(chunk);
+ processedChunks += 1;
+ }
+ }
+
+ await dispatchEvent('audioInput', chunksToProcess);
+ }
+
+ setTimeout(() => processAudioInput(), 0);
+ };
+
+ const connectToAppSync = async (model: Model) => {
+ audioInputQueue.current = [];
+
+ const channelId = uuid();
+ const channel = await events.connect(`/${NAMESPACE}/${channelId}`);
+
+ channelRef.current = channel;
+ channel.subscribe({
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ next: (data: any) => {
+ const event = data?.event;
+ if (event && event.direction === 'btoc') {
+ if (event.event === 'ready') {
+ startRecording().then(() => {
+ setIsLoading(false);
+ });
+ } else if (event.event === 'end') {
+ closeSession();
+ } else if (event.event === 'audioOutput' && audioPlayerRef.current) {
+ const chunks: string[] = event.data;
+
+ while (chunks.length > 0) {
+ const chunk = chunks.shift();
+
+ if (chunk) {
+ const audioData = base64ToFloat32Array(chunk);
+ audioPlayerRef.current.playAudio(audioData);
+ }
+ }
+ } else if (event.event === 'textStart') {
+ onTextStart(event.data);
+ } else if (event.event === 'textOutput') {
+ onTextOutput(event.data);
+ } else if (event.event === 'textStop') {
+ onTextStop(event.data);
+
+ if (
+ event.data.stopReason &&
+ event.data.stopReason === 'INTERRUPTED'
+ ) {
+ audioPlayerRef.current?.bargeIn();
+ }
+ }
+ }
+ },
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ error: (e: any) => {
+ console.error(e);
+ },
+ });
+
+ await api.post('speech-to-speech', { channel: channelId, model });
+ };
+
+ const startRecording = async () => {
+ if (
+ !audioPlayerRef.current ||
+ !audioRecorderRef.current ||
+ !systemPromptRef.current
+ ) {
+ return;
+ }
+
+ await dispatchEvent('promptStart');
+ await dispatchEvent('systemPrompt', systemPromptRef.current);
+ await dispatchEvent('audioStart');
+
+ setupSystemPrompt(systemPromptRef.current);
+
+ await audioPlayerRef.current.start();
+
+ // Start recording using the AudioRecorder and check for success
+ const success = await audioRecorderRef.current.start();
+
+ if (!success) {
+ return;
+ }
+
+ setIsActive(true);
+
+ processAudioInput();
+ };
+
+ const stopRecording = async () => {
+ setIsActive(false);
+
+ if (audioRecorderRef.current) {
+ audioRecorderRef.current.stop();
+ audioRecorderRef.current = null;
+ }
+
+ if (audioPlayerRef.current) {
+ audioPlayerRef.current.stop();
+ audioPlayerRef.current = null;
+ }
+
+ await dispatchEvent('audioStop');
+ };
+
+ const startSession = async (systemPrompt: string, model: Model) => {
+ if (isActive || isLoading) {
+ return;
+ }
+
+ clear();
+
+ setIsLoading(true);
+
+ systemPromptRef.current = systemPrompt;
+
+ await connectToAppSync(model);
+ await initAudio();
+ };
+
+ const closeSession = async () => {
+ await stopRecording();
+
+ setIsActive(false);
+ setIsLoading(false);
+ };
+
+ return {
+ messages,
+ isActive,
+ isLoading,
+ isAssistantSpeeching,
+ startSession,
+ closeSession,
+ errorMessages,
+ };
+};
diff --git a/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts
new file mode 100644
index 000000000..8466db755
--- /dev/null
+++ b/packages/web/src/hooks/useSpeechToSpeech/useChatHistory.ts
@@ -0,0 +1,196 @@
+import { useState, useRef, useMemo } from 'react';
+import { UnrecordedMessage } from 'generative-ai-use-cases';
+
+type EventMessage = {
+ id: string;
+ role: string;
+ content: string;
+ generationStage: string;
+ stopReason: string;
+};
+
+type EventMessageCacheEntity = Partial & {
+ id: string;
+};
+
+const useChatHistory = () => {
+ const [eventMessages, setEventMessages] = useState([]);
+ const eventMessageCache = useRef>({});
+
+ const clear = () => {
+ setEventMessages([]);
+ eventMessageCache.current = {};
+ };
+
+ const setupSystemPrompt = (prompt: string) => {
+ setEventMessages([
+ [
+ {
+ id: 'system',
+ role: 'system',
+ content: prompt,
+ generationStage: 'FINAL',
+ stopReason: 'END_TURN',
+ },
+ ],
+ ]);
+ };
+
+ // Since AppSync Events are not FIFO, the order of events may be switched.
+ // For example, onTextOutput might come before onTextStart.
+ // This is a process where if a new message has the same role as the previous message, the messages are combined.
+ // Otherwise (if it's a different role), it's added as a new message.
+ const tryUpdateEventMessage = (tmpEventMessage: EventMessageCacheEntity) => {
+ const currentCacheEntity = eventMessageCache.current[tmpEventMessage.id];
+
+ const newEntity = {
+ id: tmpEventMessage.id,
+ role: tmpEventMessage.role ?? currentCacheEntity?.role,
+ content: tmpEventMessage.content ?? currentCacheEntity?.content,
+ generationStage:
+ tmpEventMessage.generationStage ?? currentCacheEntity?.generationStage,
+ stopReason: tmpEventMessage.stopReason ?? currentCacheEntity?.stopReason,
+ };
+
+ eventMessageCache.current[tmpEventMessage.id] = newEntity;
+
+ if (
+ newEntity.role &&
+ newEntity.content &&
+ newEntity.generationStage &&
+ newEntity.stopReason
+ ) {
+ setEventMessages((prevEventMessages) => {
+ const lastEventMessagesIndex = prevEventMessages.length - 1;
+ const lastEventMessages = prevEventMessages[lastEventMessagesIndex];
+ const eventMessagesWithoutLast = prevEventMessages.slice(
+ 0,
+ lastEventMessagesIndex
+ );
+
+ if (lastEventMessages[0].role === newEntity.role) {
+ if (newEntity.generationStage === 'FINAL') {
+ const countFinals = lastEventMessages.filter(
+ (m) => m.generationStage === 'FINAL'
+ ).length;
+ const beforeEventMessages = lastEventMessages.slice(0, countFinals);
+ const afterEventMessages = lastEventMessages.slice(countFinals + 1);
+ return [
+ ...eventMessagesWithoutLast,
+ [...beforeEventMessages, newEntity, ...afterEventMessages],
+ ];
+ } else {
+ return [
+ ...eventMessagesWithoutLast,
+ [...lastEventMessages, newEntity],
+ ];
+ }
+ } else {
+ return [...eventMessagesWithoutLast, lastEventMessages, [newEntity]];
+ }
+ });
+ }
+ };
+
+ const onTextStart = (data: {
+ id: string;
+ role: string;
+ generationStage: string;
+ }) => {
+ tryUpdateEventMessage({
+ id: data.id,
+ role: data.role,
+ generationStage: data.generationStage,
+ });
+ };
+
+ const onTextOutput = (data: {
+ id: string;
+ role: string;
+ content: string;
+ }) => {
+ tryUpdateEventMessage({
+ id: data.id,
+ role: data.role,
+ content: data.content,
+ });
+ };
+
+ const onTextStop = (data: { id: string; stopReason: string }) => {
+ tryUpdateEventMessage({ id: data.id, stopReason: data.stopReason });
+ };
+
+ // This is where the interruption processing is inserted.
+ // When an interruption occurs:
+ // 1) If interruptedIndex === 0, the assistant's entire utterance is canceled, so we connect the user's statements before and after.
+ // 2) If interruptedIndex > 0, the assistant was interrupted after speaking partially, so we display up to the point where the utterance was completed in the UI.
+ const messages: UnrecordedMessage[] = useMemo(() => {
+ let interrupted: boolean = false;
+
+ const res: UnrecordedMessage[] = [];
+
+ for (const ms of eventMessages) {
+ if (interrupted) {
+ res[res.length - 1].content =
+ res[res.length - 1].content +
+ ' ' +
+ ms.map((m: EventMessage) => m.content).join(' ');
+ interrupted = false;
+ } else {
+ const interruptedIndex = ms.findIndex(
+ (m: EventMessage) => m.stopReason === 'INTERRUPTED'
+ );
+
+ if (interruptedIndex === 0) {
+ interrupted = true;
+ } else if (interruptedIndex > 0) {
+ res.push({
+ role: ms[0].role as 'system' | 'user' | 'assistant',
+ content: ms
+ .slice(0, interruptedIndex)
+ .map((m: EventMessage) => m.content)
+ .join(' '),
+ });
+ } else {
+ res.push({
+ role: ms[0].role as 'system' | 'user' | 'assistant',
+ content: ms.map((m: EventMessage) => m.content).join(' '),
+ });
+ }
+ }
+ }
+
+ return res;
+ }, [eventMessages]);
+
+ const isAssistantSpeeching = useMemo(() => {
+ if (eventMessages.length === 0) {
+ return false;
+ }
+
+ const lastEventMessages = eventMessages[eventMessages.length - 1];
+
+ if (lastEventMessages[0].role === 'assistant') {
+ const hasSpeculative =
+ lastEventMessages.filter(
+ (e: EventMessage) => e.generationStage === 'SPECULATIVE'
+ ).length > 0;
+ return hasSpeculative;
+ }
+
+ return false;
+ }, [eventMessages]);
+
+ return {
+ clear,
+ messages,
+ setupSystemPrompt,
+ onTextStart,
+ onTextOutput,
+ onTextStop,
+ isAssistantSpeeching,
+ eventMessages,
+ };
+};
+
+export default useChatHistory;
diff --git a/packages/web/src/main.tsx b/packages/web/src/main.tsx
index 0bdcad680..d109396d4 100644
--- a/packages/web/src/main.tsx
+++ b/packages/web/src/main.tsx
@@ -27,6 +27,7 @@ import OptimizePromptPage from './pages/OptimizePromptPage';
import TranscribePage from './pages/TranscribePage';
import AgentChatPage from './pages/AgentChatPage.tsx';
import FlowChatPage from './pages/FlowChatPage';
+import VoiceChatPage from './pages/VoiceChatPage';
import { MODELS } from './hooks/useModel';
import { Authenticator } from '@aws-amplify/ui-react';
import UseCaseBuilderEditPage from './pages/useCaseBuilder/UseCaseBuilderEditPage.tsx';
@@ -48,7 +49,12 @@ const samlAuthEnabled: boolean =
import.meta.env.VITE_APP_SAMLAUTH_ENABLED === 'true';
const agentEnabled: boolean = import.meta.env.VITE_APP_AGENT_ENABLED === 'true';
const inlineAgents: boolean = import.meta.env.VITE_APP_INLINE_AGENTS === 'true';
-const { visionEnabled, imageGenModelIds, videoGenModelIds } = MODELS;
+const {
+ visionEnabled,
+ imageGenModelIds,
+ videoGenModelIds,
+ speechToSpeechModelIds,
+} = MODELS;
const useCaseBuilderEnabled: boolean =
import.meta.env.VITE_APP_USE_CASE_BUILDER_ENABLED === 'true';
// eslint-disable-next-line react-hooks/rules-of-hooks
@@ -167,6 +173,12 @@ const routes: RouteObject[] = [
element: {
description={t('landing.use_cases.web_content.description')}
/>
)}
- {enabled('image') && (
+ {imageGenModelIds.length > 0 && enabled('image') && (
{
description={t('landing.use_cases.image.description')}
/>
)}
- {enabled('video') && (
+ {videoGenModelIds.length > 0 && enabled('video') && (
{
+ const { t } = useTranslation();
+ const {
+ messages,
+ isActive,
+ isLoading,
+ isAssistantSpeeching,
+ startSession,
+ closeSession,
+ errorMessages,
+ } = useSpeechToSpeech();
+ const [showSystemPrompt, setShowSystemPrompt] = useState(false);
+ const [systemPrompt, setSystemPrompt] = useState(
+ t('voiceChat.default_system_prompt')
+ );
+ const [inputSystemPrompt, setInputSystemPrompt] = useState(systemPrompt);
+ const { scrollableContainer, setFollowing } = useFollow();
+ const { speechToSpeechModelIds, speechToSpeechModels } = MODELS;
+ const [modelId, setModelId] = useState(speechToSpeechModelIds[0]);
+ // Define useRef for the cleanup function when leaving the page.
+ // To get the latest value without including isActive in the dependency array.
+ const isActiveRef = useRef(isActive);
+
+ const messagesWithoutSystemPrompt = useMemo(() => {
+ return messages.filter((m) => m.role !== 'system');
+ }, [messages]);
+
+ const showingMessages = useMemo(() => {
+ if (showSystemPrompt) {
+ return messages;
+ } else {
+ return messagesWithoutSystemPrompt;
+ }
+ }, [messages, messagesWithoutSystemPrompt, showSystemPrompt]);
+
+ const isEmpty = useMemo(() => {
+ return messagesWithoutSystemPrompt.length === 0;
+ }, [messagesWithoutSystemPrompt]);
+
+ useEffect(() => {
+ if (errorMessages.length > 0) {
+ toast.error(errorMessages[errorMessages.length - 1]);
+ }
+ }, [errorMessages]);
+
+ useEffect(() => {
+ isActiveRef.current = isActive;
+ }, [isActive]);
+
+ // Cleanup function
+ useEffect(() => {
+ return () => {
+ if (isActiveRef.current) {
+ closeSession();
+ }
+ };
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ return (
+ <>
+
+
+
+ );
+};
+
+export default VoiceChatPage;
diff --git a/packages/web/src/vite-env.d.ts b/packages/web/src/vite-env.d.ts
index 03c332816..b1a88f36c 100644
--- a/packages/web/src/vite-env.d.ts
+++ b/packages/web/src/vite-env.d.ts
@@ -27,6 +27,9 @@ interface ImportMetaEnv {
readonly VITE_APP_USE_CASE_BUILDER_ENABLED: string;
readonly VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN: string;
readonly VITE_APP_HIDDEN_USE_CASES: string;
+ readonly VITE_APP_SPEECH_TO_SPEECH_NAMESPACE: string;
+ readonly VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT: string;
+ readonly VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS: string;
}
interface ImportMeta {
diff --git a/packages/web/tsconfig.json b/packages/web/tsconfig.json
index 00001ded2..53638b9e7 100644
--- a/packages/web/tsconfig.json
+++ b/packages/web/tsconfig.json
@@ -9,6 +9,7 @@
/* Bundler mode */
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
+ "allowJs": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
diff --git a/setup-env.sh b/setup-env.sh
index fa4ecf20f..f24058ada 100755
--- a/setup-env.sh
+++ b/setup-env.sh
@@ -47,3 +47,6 @@ export VITE_APP_INLINE_AGENTS=$(extract_value "$stack_output" InlineAgents)
export VITE_APP_USE_CASE_BUILDER_ENABLED=$(extract_value "$stack_output" UseCaseBuilderEnabled)
export VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN=$(extract_value "$stack_output" OptimizePromptFunctionArn)
export VITE_APP_HIDDEN_USE_CASES=$(extract_value "$stack_output" HiddenUseCases)
+export VITE_APP_SPEECH_TO_SPEECH_NAMESPACE=$(extract_value "$stack_output" SpeechToSpeechNamespace)
+export VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT=$(extract_value "$stack_output" SpeechToSpeechEventApiEndpoint)
+export VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS=$(extract_value "$stack_output" SpeechToSpeechModelIds)
diff --git a/web_devw_win.ps1 b/web_devw_win.ps1
index ec6fce7ee..8cf86892f 100644
--- a/web_devw_win.ps1
+++ b/web_devw_win.ps1
@@ -80,5 +80,8 @@ $env:VITE_APP_INLINE_AGENTS = Extract-Value $stack_output "InlineAgents"
$env:VITE_APP_USE_CASE_BUILDER_ENABLED = Extract-Value $stack_output "UseCaseBuilderEnabled"
$env:VITE_APP_OPTIMIZE_PROMPT_FUNCTION_ARN = Extract-Value $stack_output "OptimizePromptFunctionArn"
$env:VITE_APP_HIDDEN_USE_CASES = Extract-Value $stack_output "HiddenUseCases"
+$env:VITE_APP_SPEECH_TO_SPEECH_NAMESPACE = Extract-Value $stack_output "SpeechToSpeechNamespace"
+$env:VITE_APP_SPEECH_TO_SPEECH_EVENT_API_ENDPOINT = Extract-Value $stack_output "SpeechToSpeechEventApiEndpoint"
+$env:VITE_APP_SPEECH_TO_SPEECH_MODEL_IDS = Extract-Value $stack_output "SpeechToSpeechModelIds"
npm -w packages/web run dev
+ {t('voiceChat.title')}
+
+
+ {isEmpty && !isLoading && !isActive && (
+
+
+ )}
+
+ {isEmpty && !isLoading && !isActive && (
+
+
+ {t('voiceChat.experimental_warning')}
+
+
+ )}
+
+ {isEmpty && isLoading && (
+
+
+
+
+
+ )}
+
+ {isEmpty && !isLoading && isActive && (
+
+
+ )}
+
+ {!isEmpty && (
+ {t('voiceChat.im_listening')}
+
+
+ )}
+
+ {!isEmpty && (
+
+ {showingMessages.map((m, idx) => {
+ return (
+
+ )}
+
+
+ {idx === 0 && (
+
+ )}
+
+ );
+ })}
+
+
+
+
+ {!isLoading && !isActive && (
+
+ <>
+
+ )}
+
+ {!isActive ? (
+
+ ) : (
+
+ )}
+
+
+
+
+
+