import gradio as gr from fastrtc import ReplyOnPause, AlgoOptions, SileroVadOptions, AdditionalOutputs, WebRTC, get_cloudflare_turn_credentials_async, get_cloudflare_turn_credentials #get_hf_turn_credentials, import os from dotenv import load_dotenv import time import numpy as np import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from backend.tts import synthesize_text from backend.asr import transcribe_audio from backend.utils import preprocess_audio, is_valid_turn from backend.main import stream_chat_response from pydub import AudioSegment load_dotenv(override=True) phone_waiting_sound = AudioSegment.from_mp3("frontend/phone-ringing-382734.mp3")[:1000] sound_samples = np.array(phone_waiting_sound.get_array_of_samples(), dtype=np.int16) if phone_waiting_sound.channels > 1: sound_samples = sound_samples.reshape((-1, phone_waiting_sound.channels)).mean(axis=1) sound_samples = sound_samples.astype(np.float32) / 32768.0 # Normalize to [-1, def startup(_): yield (phone_waiting_sound.frame_rate, sound_samples) STARTUP_MESSAGE = "สวัสดีค่ะ มีข้อมูลสอบถามด้านใดคะ?" yield from synthesize_text(STARTUP_MESSAGE) time.sleep(2) yield AdditionalOutputs([{"role": "assistant", "content": STARTUP_MESSAGE}]) custom_css = """ /* Overall Gradio page styling: hot pink background */ body { /* background-color: #ff69b4; /* Hot pink */ margin: 0; padding: 0; font-family: sans-serif;} /* Title styling */ h1 { color: #fff; text-shadow: 1px 1px 2px #ff85a2; font-size: 2.5em; margin-bottom: 20px; text-align: center; } /* Style the column holding the telephone interface */ .phone-column { max-width: 350px !important; /* Limit the width of the phone column */ margin: 0 auto; /* Center the column */ border-radius: 20px; background-color: #ff69b4; /* Lighter pink for telephone interface */ box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); padding: 20px; } /* Conversation history box styling */ #conversation-history-chatbot { background-color: #ffc0cb; /* Lighter pink for conversation history */ border: 1px solid #ccc; border-radius: 10px; padding: 10px; box-shadow: 0 0 15px rgba(0, 0, 0, 0.2); } """ def response(audio: tuple[int, np.ndarray] | None, conversation_history): """ Handles user audio input, transcribes it, streams LLM text via backend.main, and synthesizes chunks to audio while updating the conversation history. """ print(f"--- Latency Breakdown ---") start_time = time.time() if conversation_history is None: conversation_history = [] previous_history = list(conversation_history) if not audio or audio[1] is None or not np.any(audio[1]): print("No audio input detected; skipping response generation.") print(f"------------------------") return import soundfile as sf sample_rate, audio_array = audio try: processed_audio = preprocess_audio((sample_rate, audio_array), target_frame_rate=16000) except Exception as audio_err: print(f"Audio preprocessing failed: {audio_err}") print(f"------------------------") return t0 = time.time() transcription = transcribe_audio( processed_audio) t_asr = time.time() - t0 print(f"ASR: {t_asr:.4f}s") if not transcription.strip(): print("No valid transcription; skipping response generation.") print(f"------------------------") return user_turn = {"role": "user", "content": transcription} print(f"User: {transcription}") if is_valid_turn(user_turn): conversation_history.append(user_turn) yield AdditionalOutputs(conversation_history) print("Conversation history:", conversation_history) assistant_turn = {"role": "assistant", "content": ""} conversation_history.append(assistant_turn) history_for_stream = [dict(turn) for turn in previous_history if is_valid_turn(turn)] text_buffer = "" full_response = "" delimiter_count = 0 n_threshold = 3 max_n_threshold = 5 lang = "th" chunk_count = 0 first_chunk_sent = False start_llm_stream = time.time() try: for text_chunk in stream_chat_response(history_for_stream, transcription): if not isinstance(text_chunk, str): text_chunk = str(text_chunk) i = 0 while i < len(text_chunk): char = text_chunk[i] text_buffer += char full_response += char assistant_turn["content"] = full_response.strip() is_delimiter = False if char in {' ', '\n'}: is_delimiter = True delimiter_count += 1 if i + 1 < len(text_chunk) and text_chunk[i + 1] == 'ๆ': text_buffer += text_chunk[i + 1] full_response += text_chunk[i + 1] i += 1 send_now = False if not first_chunk_sent: if is_delimiter and text_buffer.strip(): send_now = True else: if delimiter_count >= n_threshold and text_buffer.strip(): send_now = True if n_threshold < max_n_threshold: n_threshold += 1 if send_now: buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs(conversation_history) i += 1 if text_buffer.strip(): buffer_to_send = text_buffer.strip() try: if buffer_to_send and buffer_to_send.endswith('วันที่'): buffer_to_send = buffer_to_send[:-len('วันที่')] if buffer_to_send and first_chunk_sent and buffer_to_send.endswith('ค่ะ'): buffer_to_send = buffer_to_send[:-len('ค่ะ')] except Exception: buffer_to_send = buffer_to_send.replace('ค่ะ', '') if buffer_to_send: chunk_count += 1 if chunk_count == 1: first_llm_chunk_time = time.time() t_llm_first_token = first_llm_chunk_time - start_llm_stream print(f"LLM TTFC: {t_llm_first_token:.4f}s (Time To First Chunk)") yield from synthesize_text(buffer_to_send, lang=lang) first_chunk_sent = True text_buffer = "" delimiter_count = 0 yield AdditionalOutputs(conversation_history) except Exception as e: print(f"An error occurred during response generation or synthesis: {e}") error_message = "ขออภัยค่ะ เกิดข้อผิดพลาดบางอย่าง" try: yield from synthesize_text(error_message, lang=lang) except Exception as synth_error: print(f"Could not synthesize error message: {synth_error}") assistant_turn["content"] = (assistant_turn.get("content", "") + f" [Error: {e}]").strip() yield AdditionalOutputs(conversation_history) total_latency = time.time() - start_time print(f"Total: {total_latency:.4f}s") print(f"------------------------") async def get_credentials(): return await get_cloudflare_turn_credentials_async(hf_token=os.getenv('HF_TOKEN')) with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="pink", secondary_hue="pink")) as demo: gr.HTML("""

1157 Voicebot Demo

""") with gr.Row(): with gr.Column(scale=1, elem_classes=["phone-column"]): audio = WebRTC( mode="send-receive", modality="audio", track_constraints={ "echoCancellation": True, "noiseSuppression": {"exact": True}, "autoGainControl": {"exact": True} }, rtc_configuration=get_credentials, server_rtc_configuration=get_cloudflare_turn_credentials(ttl=360_000), icon="https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif", icon_button_color="#17dbaa", pulse_color="#b0f83b", button_labels={"start": "Call", "stop": "Hang up", "waiting": "Connecting…"}, icon_radius=45, height="650px", width="100%", container=False, elem_id="phone-call-webrtc" ) with gr.Column(): conversation_history = gr.Chatbot( label="Conversation History", type="messages", value=[], height="675px", resizable=True, avatar_images=(None, "https://i.pinimg.com/originals/0c/67/5a/0c675a8e1061478d2b7b21b330093444.gif"), ) gr.DeepLinkButton() audio.stream( fn=ReplyOnPause( response, algo_options=AlgoOptions( audio_chunk_duration=0.6, started_talking_threshold=0.3, speech_threshold=0.6 ), model_options=SileroVadOptions( threshold=0.8, min_speech_duration_ms=300, max_speech_duration_s=float("inf"), min_silence_duration_ms=1200, ), can_interrupt=True, startup_fn=startup, ), inputs=[audio, conversation_history], outputs=[audio], concurrency_limit=1000, time_limit=8192 ) audio.on_additional_outputs( lambda history: history, outputs=[conversation_history], queue=True, show_progress="hidden" ) demo.queue(default_concurrency_limit=1000) demo.launch(debug=True, show_error=True, share=True)