Response 3D Chatbot Service - Complete Developer Documentation¶

system = platform.system().lower()
current_dir = os.path.dirname(os.path.abspath(__file__))

if system == "windows":
    rhubarbExePath = os.path.join(current_dir, "Rhubarb-Lip-Sync-1.13.0-Windows", "rhubarb.exe")
elif system == "darwin":  # macOS
    rhubarbExePath = os.path.join(current_dir, "Rhubarb-Lip-Sync-1.13.0-macOS", "rhubarb")
elif system == "linux":
    rhubarbExePath = os.path.join(current_dir, "Rhubarb", "rhubarb")
else:
    raise RuntimeError(f"Unsupported platform: {system}")

# Verify executable exists
if not os.path.exists(rhubarbExePath):
    raise RuntimeError(f"Rhubarb executable not found at: {rhubarbExePath}")

async def text_to_speech_azure(text, voice, session_id):
    # Clean text (remove contact info)
    cleaned_text = remove_contact_numbers(text)

    # Configure Azure TTS
    speech_config = speechsdk.SpeechConfig(
        subscription="9N41NOfD...",  # ⚠️ Hardcoded!
        region="centralindia"
    )
    speech_config.speech_synthesis_voice_name = voice

    # Generate WAV file
    wav_file = os.path.join(OUTPUT_DIR, f"{session_id}.wav")
    audio_config = speechsdk.audio.AudioOutputConfig(filename=wav_file)

    speech_synthesizer = speechsdk.SpeechSynthesizer(
        speech_config=speech_config,
        audio_config=audio_config
    )

    result = speech_synthesizer.speak_text_async(cleaned_text).get()

    if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:
        return wav_file
    else:
        raise Exception(f"TTS failed: {result.cancellation_details.reason}")

def convert_wav_to_pcm(input_wav, output_wav):
    """Convert WAV to PCM format using FFmpeg (required by Rhubarb)"""
    try:
        subprocess.run(
            ["ffmpeg", "-i", input_wav, "-acodec", "pcm_s16le", output_wav],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True
        )
        return output_wav if os.path.exists(output_wav) else None
    except subprocess.CalledProcessError as e:
        logger.error(f"FFmpeg conversion failed: {e.stderr.decode()}")
        return None

def generate_lip_sync(wav_file, user_id):
    """Generate lip-sync JSON using Rhubarb executable"""
    json_file = os.path.join(OUTPUT_DIR, f"{user_id}.json")

    try:
        result = subprocess.run(
            [rhubarbExePath, "-f", "json", "-o", json_file, wav_file, "-r", "phonetic"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True
        )
        return json_file if os.path.exists(json_file) else None
    except Exception as e:
        logger.error(f"Rhubarb lip-sync failed: {e}")
        return None

def parse_lip_sync(json_file, sound_file):
    """Parse Rhubarb JSON and add sound file reference"""
    with open(json_file, "r") as file:
        lip_sync_data = json.load(file)

    # Add sound file path to metadata
    lip_sync_data["metadata"]["soundFile"] = sound_file
    return lip_sync_data

{
  "metadata": {
    "soundFile": "tts_audio/session_123.wav",
    "duration": 2.5
  },
  "mouthCues": [
    { "start": 0.0, "end": 0.1, "value": "X" },
    { "start": 0.1, "end": 0.25, "value": "A" },
    { "start": 0.25, "end": 0.4, "value": "B" },
    { "start": 0.4, "end": 0.55, "value": "C" },
    { "start": 0.55, "end": 0.7, "value": "D" },
    { "start": 0.7, "end": 0.85, "value": "E" },
    { "start": 0.85, "end": 1.0, "value": "F" },
    { "start": 1.0, "end": 1.2, "value": "G" },
    { "start": 1.2, "end": 1.5, "value": "H" },
    { "start": 1.5, "end": 2.5, "value": "X" }
  ]
}

GET /lip-sync?message=Hello!%20How%20can%20I%20help%20you?&user_id=User-123456&voice_selection=Female_2&project_id=User-123456_Project_1

{
    "lipsync": {
        "metadata": {
            "soundFile": "tts_audio/User-123456.wav",
            "duration": 1.8
        },
        "mouthCues": [
            {"start": 0.00, "end": 0.10, "value": "X"},
            {"start": 0.10, "end": 0.25, "value": "H"},
            ...
        ]
    },
    "audio": "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA..."
}

subscription_key = "AZxDVMYB08AaUip0i5ed1sy73ZpUsqencYYxKDbm6nfWfG1AqPZ3JQQJ99BCACYeBjFXJ3w3AAABACOGVUo7"  # ⚠️

endpoint_gpt4 = "https://machineagentopenai.openai.azure.com/openai/deployments/gpt-4-0613/..."
deployment_gpt4 = "gpt-4-0613"

client = AzureOpenAI(
    azure_endpoint=endpoint_gpt4,
    api_key=subscription_key,
    api_version="2024-02-15-preview"
)

def call_openai_4(messages):
    response = client.chat.completions.create(
        model="deployment_gpt4",
        messages=messages,
        temperature=0.7
    )
    return response.choices[0].message.content

endpoint_gpt35 = "https://machineagentopenai.openai.azure.com/openai/deployments/gpt-35-turbo-16k-0613/..."
deployment_gpt35 = "gpt-35-turbo-16k-0613"

AZURE_LLAMA_ENDPOINT = "https://Llama-3-3-70B-Instruct-ulmca.eastus.models.ai.azure.com/chat/completions"
AZURE_API_KEY = "JOfcw0VW0dS31Z8XgkNRSP9tUaBiwUYZ"  # ⚠️ Hardcoded!

def call_llama(messages):
    payload = {"messages": messages, "temperature": 0.7, "max_tokens": 50}
    response = requests.post(AZURE_LLAMA_ENDPOINT, json=payload, headers=llama_HEADERS)
    return response.json()["choices"][0]["message"]["content"]

deepseek_api_url = "https://DeepSeek-R1-imalr.eastus2.models.ai.azure.com/chat/completions"
deepseek_api_key = "GwUcGzHhhUbvApfMR4aq1ZPFUic6lbWE"  # ⚠️ Hardcoded!

def call_deepseek(messages):
    response = requests.post(deepseek_api_url, headers=deepseekheaders, json=data)
    answer = response.json()["choices"][0]["message"]["content"]
    # Remove <think> tags
    return re.sub(r"<think>.*?</think>", "", answer, flags=re.DOTALL).strip()

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "rules": [
        {
            "type": "blocked_keywords",
            "keywords": ["profanity", "inappropriate"],
            "action": "reject"
        },
        {
            "type": "topic_restriction",
            "allowed_topics": ["products", "services", "support"],
            "action": "redirect"
        }
    ],
    "enabled": true
}

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "session_id": "session_123",
    "timestamp": ISODate("2025-01-15T14:00:00Z"),
    "metrics": {
        "total_time_ms": 4500,
        "milvus_time_ms": 120,
        "llm_time_ms": 2500,
        "tts_time_ms": 1200,
        "lipsync_time_ms": 680
    },
    "tokens": {
        "input": 150,
        "output": 80,
        "total": 230
    },
    "model_used": "gpt-35-turbo-16k-0613"
}

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "form_config": {
        "fields": ["name", "email", "phone", "company"],
        "trigger_message": "Could you please provide some details by filling the form?",
        "show_after_messages": 4
    },
    "leads": [
        {
            "name": "John Doe",
            "email": "john@example.com",
            "phone": "+1-555-123-4567",
            "company": "Acme Corp",
            "submitted_at": ISODate("2025-01-15T14:05:00Z"),
            "session_id": "session_123"
        }
    ]
}

from openai import AsyncAzureOpenAI

async_client = AsyncAzureOpenAI(
    azure_endpoint=endpoint_gpt35,
    api_key=subscription_key,
    api_version="2024-02-15-preview"
)

async def stream_response(messages):
    stream = await async_client.chat.completions.create(
        model="gpt-35-turbo-16k-0613",
        messages=messages,
        temperature=0.7,
        stream=True
    )

    async for chunk in stream:
        if chunk.choices[0].delta.content:
            yield chunk.choices[0].delta.content

@app.post("/v2/get-response-stream")
async def get_response_stream(...):
    return StreamingResponse(
        stream_response(messages),
        media_type="text/event-stream"
    )

# Config A: target_url only
{
    "target_url": "https://example.com/products",
    "utm_config": {}
}
# Score for "https://example.com/products?page=1": 5 points

# Config B: target_url + UTM source
{
    "target_url": "https://example.com/products",
    "utm_config": {"source": "google"}
}
# Score for "https://example.com/products?utm_source=google": 12 points (10 + 2)

# Config C: multiple UTM params
{
    "target_url": "https://example.com/products",
    "utm_config": {"source": "google", "medium": "cpc", "campaign": "summer"}
}
# Score for "https://example.com/products?utm_source=google&utm_medium=cpc&utm_campaign=summer": 16 points (10 + 2 + 2 + 2)

system_prompt = system_prompt + f"\n\n[UTM-Specific Instructions]:\n{utm_instructions}"

[UTM-Specific Instructions]:
- This user came from Google Ads "Summer Sale" campaign
- Emphasize 20% discount on all products
- Mention free shipping on orders over $50
- Link to https://example.com/summer-sale landing page

def summarize_question_with_context(question: str, chat_history: List[Dict[str, str]], llm):
    # Extract last 4 user/assistant pairs
    recent_exchanges = []
    user_msgs = [msg for msg in chat_history if msg["role"] == "user"]
    assistant_msgs = [msg for msg in chat_history if msg["role"] == "assistant"]

    # Pair the last 4 exchanges
    for i in range(min(4, len(user_msgs), len(assistant_msgs))):
        u = user_msgs[-(i + 1)]["content"]
        a = assistant_msgs[-(i + 1)]["content"]
        recent_exchanges.insert(0, f"User: {u}\nAssistant: {a}")

    history_context = "\n\n".join(recent_exchanges)

    prompt = f"""
    You are an AI assistant helping to clarify user queries using recent chat context.

    Here is the recent conversation:
    {history_context}

    And here is the current user message:
    "{question}"

    Based on the above, rewrite the current user message as a standalone, clear, and context-rich question.
    Only return the rewritten question.
    """.strip()

    response = llm.invoke([HumanMessage(content=prompt)])
    enhanced_question = response.content.strip()
    return enhanced_question

User: "Do you offer cloud hosting?"
Assistant: "Yes, we offer cloud hosting with 99.9% uptime."
User: "What are the pricing options?"
Assistant: "We have Basic ($10/mo), Pro ($50/mo), and Enterprise (custom)."

DEFAULT_GUARDRAIL_USER_ID = "_system_default_user_"
DEFAULT_GUARDRAIL_PROJECT_ID = "_system_default_project_"

def getGuardrails(user_id: str, project_id: str):
    # Try user-specific guardrails first
    guardrails = guardrails_collection.find_one({
        "user_id": user_id,
        "project_id": project_id
    })

    if not guardrails:
        # Fallback to system default
        guardrails = guardrails_collection.find_one({
            "user_id": DEFAULT_GUARDRAIL_USER_ID,
            "project_id": DEFAULT_GUARDRAIL_PROJECT_ID
        })

    # Double fallback to empty guardrails
    if not guardrails:
        return {
            "document": {
                "categories": "No specific guardrails configured."
            }
        }

    return {"document": guardrails}

CONCURRENT_TASKS = os.cpu_count() or 4

LLM generates 20 text chunks
With CONCURRENT_TASKS=8:
- Processes chunks 0-7 in parallel
- Then chunks 8-15 in parallel
- Then chunks 16-19 in parallel
- Total: 3 batches instead of 20 sequential

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "session_id": "session-abc123",
    "question": "What are your pricing options?",
    "full_answer": "We offer three plans: Basic at $10/mo, Pro at $50/mo, and Enterprise with custom pricing...",
    "originating_url": "https://example.com/pricing?utm_source=google",

    // UTC Timestamps (ISO format)
    "timestamps_utc": {
        "request_arrival": "2025-12-26T13:10:25.123456+00:00",
        "rag_start": "2025-12-26T13:10:25.150000+00:00",
        "rag_end": "2025-12-26T13:10:25.890000+00:00",
        "llm_hit": "2025-12-26T13:10:25.900000+00:00",
        "first_text_yield": "2025-12-26T13:10:26.450000+00:00",
        "first_audio_yield": "2025-12-26T13:10:27.200000+00:00",
        "stream_end": "2025-12-26T13:10:32.100000+00:00"
    },

    // IST Timestamps (human-readable, auto-converted)
    "timestamps_ist": {
        "request_arrival": "2025-12-26 18:40:25.123 IST",
        "rag_start": "2025-12-26 18:40:25.150 IST",
        "rag_end": "2025-12-26 18:40:25.890 IST",
        "llm_hit": "2025-12-26 18:40:25.900 IST",
        "first_text_yield": "2025-12-26 18:40:26.450 IST",
        "first_audio_yield": "2025-12-26 18:40:27.200 IST",
        "stream_end": "2025-12-26 18:40:32.100 IST"
    },

    // Durations in SECONDS (Lines 2465-2478)
    "durations_sec": {
        "total_request_time": 6.977,      // request_arrival → stream_end
        "rag_processing": 0.740,          // rag_start → rag_end
        "llm_time_to_first_token": 0.550, // llm_hit → first_text_yield
        "time_to_first_text": 1.327,      // request_arrival → first_text_yield
        "time_to_first_audio": 2.077      // request_arrival → first_audio_yield
    },

    // Token Usage (Lines 2461-2463)
    "tokens": {
        "question": 5,    // Token count of question
        "answer": 45,     // Token count of full answer
        "total": 50       // question + answer
    }
}

import pytz
from datetime import datetime

async def save_performance_log(log_data: dict):
    # Define Indian Standard Timezone
    indian_tz = pytz.timezone("Asia/Kolkata")

    timestamps_ist = {}
    utc_timestamps = log_data.get("timestamps_utc", {})

    for key, value in utc_timestamps.items():
        if isinstance(value, datetime):
            # Convert UTC → IST
            ist_time = value.astimezone(indian_tz)
            timestamps_ist[key] = ist_time.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] + ' IST'

            # Convert UTC datetime → ISO string
            log_data["timestamps_utc"][key] = value.isoformat()

    log_data["timestamps_ist"] = timestamps_ist

    # Save to MongoDB
    await asyncio.to_thread(performance_logs_collection.insert_one, log_data)

# Lines 2465-2478
ts = performance_data["timestamps_utc"]
durations = performance_data["durations_sec"]

if ts.get("stream_end") and ts.get("request_arrival"):
    durations["total_request_time"] = (ts["stream_end"] - ts["request_arrival"]).total_seconds()

if ts.get("first_text_yield") and ts.get("request_arrival"):
    durations["time_to_first_text"] = (ts["first_text_yield"] - ts["request_arrival"]).total_seconds()

if ts.get("first_audio_yield") and ts.get("request_arrival"):
    durations["time_to_first_audio"] = (ts["first_audio_yield"] - ts["request_arrival"]).total_seconds()

if ts.get("rag_end") and ts.get("rag_start"):
    durations["rag_processing"] = (ts["rag_end"] - ts["rag_start"]).total_seconds()

if ts.get("first_text_yield") and ts.get("llm_hit"):
    durations["llm_time_to_first_token"] = (ts["first_text_yield"] - ts["llm_hit"]).total_seconds()

# Assumed implementation (not shown in file, but referenced in code)
def count_tokens(text: str) -> int:
    # Simple word-based approximation (actual may use tiktoken)
    return len(text.split())

performance_data["tokens"]["question"] = count_tokens(question)
performance_data["tokens"]["answer"] = count_tokens(full_answer)
performance_data["tokens"]["total"] = performance_data["tokens"]["question"] + performance_data["tokens"]["answer"]

{
    "lipsync": { ... },
    "audio": "base64..."
}

{
    "text": "...",
    "audio": "base64...",
    "lipsync": { ... }
}

async def _llm_stream_producer(stream, queue, delimiters):
    # Reads LLM stream, chunks, puts in queue
    async for chunk in stream:
        # ... chunk text ...
        await queue.put((chunk_index, text_chunk))
    await queue.put(None)  # Signal end

# Consumer
while not producer_finished or pending_audio_tasks:
    done, pending = await asyncio.wait(tasks, return_when=FIRST_COMPLETED)
    # ... process ...

// Text chunk (yielded immediately)
{"type": "text", "chunkId": 0, "text": "Hello! How can I help you?"}

// Audiovisual chunk (yielded later)
{
    "type": "audiovisual",
    "chunkId": 0,
    "audio": "base64...",
    "lipsync": {...},
    "facialExpression": "smile",
    "animation": "Idle"
}

// Completion
{"type": "control", "status": "end"}

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "session_id": "session_123",
    "originating_url": "https://example.com?utm_source=google",
    "question": "What are your pricing options?",
    "full_answer": "We offer three pricing tiers...",

    // UTC timestamps (ISO format)
    "timestamps_utc": {
        "request_arrival": "2025-01-15T14:00:00.000Z",
        "rag_start": "2025-01-15T14:00:00.050Z",
        "rag_end": "2025-01-15T14:00:00.200Z",
        "llm_hit": "2025-01-15T14:00:00.250Z",
        "first_text_yield": "2025-01-15T14:00:01.500Z",
        "first_audio_yield": "2025-01-15T14:00:03.200Z",
        "stream_end": "2025-01-15T14:00:05.800Z"
    },

    // IST timestamps (human-readable)
    "timestamps_ist": {
        "request_arrival": "2025-01-15 19:30:00.000 IST",
        "rag_start": "2025-01-15 19:30:00.050 IST",
        ...
    },

    // Durations in seconds (float)
    "durations_sec": {
        "total_request_time": 5.8,
        "time_to_first_text": 1.25,
        "time_to_first_audio": 2.95,
        "rag_processing": 0.15,
        "llm_time_to_first_token": 1.25
    },

    // Token usage
    "tokens": {
        "question": 25,
        "answer": 85,
        "total": 110
    }
}

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "trigger": true,
    "prompt_number": 4,  // Trigger after 4th message
    "message_display": "Could you please provide some details by filling the form?"
}

# Get current chat count
existing_session = history_collection.find_one({...})
existing_valid_chats = [chat for chat in existing_session["chat_data"] if ...]
current_chat_count = len(existing_valid_chats) + 1  # +1 for current

# Get form config
lead_config = lead_collection.find_one({...})

if lead_config and lead_config.get("trigger", False):
    prompt_number = lead_config.get("prompt_number", 0)

    if current_chat_count == prompt_number and prompt_number > 0:
        # Check if form already shown
        form_already_shown = False
        for chat in existing_session["chat_data"]:
            if " form" in chat["output_response"].lower() and "fill" in chat["output_response"].lower():
                form_already_shown = True
                break

        if not form_already_shown:
            form_message = lead_config.get("message_display", "Please fill out the form...")
            answer = f"{answer}\n\n{form_message}"

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "categories": [
        {
            "category": "Profanity",
            "action": "block",
            "keywords": ["bad", "word", "list"]
        },
        {
            "category": "Off-Topic",
            "action": "redirect",
            "allowed_topics": ["products", "services", "support"]
        },
        {
            "category": "Sensitive",
            "action": "escalate",
            "keywords": ["legal", "medical", "financial"]
        }
    ]
}

{
    "user_id": "_system_default_user_",
    "project_id": "_system_default_project_",
    "categories": [...]  // Fallback rules
}

Guardrails_data = guardrails_result.get('document', {}).get('categories', [])
# Injected into context
context_parts.append(f"Strict Guidelines (Guardrails):\n{Guardrails_data}")

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "model": "gpt-35-turbo-16k-0613",
    "chatbot_purpose": "Sales-Agent",
    "system_prompt": "Custom sales prompt..."
}

{
    "model": "gpt-35-turbo-16k-0613",
    "chatbot_purpose": "Sales-Agent",
    "content": "You are a highly skilled AI sales agent..."
}

male_avatars = {"Chris", "Jack"}
female_avatars = {"Eva", "Shayla", "Myra", "Anu"}

male_voices = {"Male_1", "Male_2", "Male_3", "Male_IND"}
female_voices = {"Female_1", "Female_2", "Female_3", "Female_4", "Female_IND", "Female_IND2"}

def assign_voice_by_avatar(avatar: str, voice: str) -> str:
    avatar = avatar.strip().capitalize()
    voice = voice.strip()

    if avatar in male_avatars:
        if voice not in male_voices:
            return "Male_2"  # Default male voice
    elif avatar in female_avatars:
        if voice not in female_voices:
            return "Female_2"  # Default female voice

    return voice  # Keep original if valid

with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
    future_system_prompt = executor.submit(get_system_prompt, ...)
    future_utm = executor.submit(process_utm_completely, ...)
    future_doc_retrieval = executor.submit(retrieve_relevant_documents, ...)
    future_guardrails = executor.submit(getGuardrails, ...)
    future_org_data = executor.submit(get_org_data)
    future_chat_history = executor.submit(get_chat_history)

    # Wait for all to complete
    concurrent.futures.wait([...])

    # Get results
    system_prompt = future_system_prompt.result()
    utm_result = future_utm.result()
    ...

if not chatbot_config or chatbot_config.get("isDeleted") is True:
    error_text = "This chatbot is no longer available."

    # Use chatbot's configured voice
    chatbot_voice_default = assign_voice_by_avatar(chatbot_avatar, chatbot_voice)
    azure_voice = SUPPORTED_VOICES.get(chatbot_voice_default)

    # Generate TTS + Lip-Sync for error
    base_filename = f"{user_id}_{session_id}_error"
    wav_file = await text_to_speech_azure1(error_text, azure_voice, wav_file)
    json_file = await generate_lip_sync1(wav_file, json_file)

    # Return SAME format as normal response
    return {
        "text": error_text,
        "facialExpression": "neutral",
        "animation": "Idle",
        "audio": audio_base64,
        "lipsync": lip_sync_data
    }

except Exception as e:
    logger.error(f"Unexpected error in sync_ask_gpt1: {e}")

    # Fallback: Minimal LLM call
    try:
        fallback_messages = [
            SystemMessage(content=system_prompts.get(chatbot_purpose, "You are a helpful AI assistant.")),
            HumanMessage(content=f"Question: {question}")
        ]
        response = llm.invoke(fallback_messages)
        answer = re.sub(r'(?i)^(answer:|response:|responds:| *-+ #?)\s*', '', response.content).strip()

        # Save fallback history
        save_chat_history(user_id, project_id, session_id, question, answer, originating_url)

        # Form trigger logic (duplicate from main flow)
        # ... check if form should be shown ...

        return {"text": answer}  # No audio/lipsync in fallback

    except Exception as fallback_error:
        raise HTTPException(status_code=500, detail=f"An error occurred: {e}")

async def text_to_speech_azure(text, voice, session_id):
    # Step 1: Clean the input text
    cleaned_text = remove_contact_numbers(text)  # Remove phones, URLs, emojis

    # Step 2: Define output file path
    wav_file = os.path.join(OUTPUT_DIR, f"{session_id}.wav")

    # Step 3: Configure Azure Speech
    speech_config = speechsdk.SpeechConfig(
        subscription="9N41NOfDyVDoduiD4EjlzmZU9CbUX3pPqWfLCORpl7cBf0l2lzVQJQQJ99BCACGhslBXJ3w3AAAYACOG2329",
        region="centralindia"
    )
    speech_config.speech_synthesis_voice_name = voice  # e.g., "en-US-JennyNeural"

    # Step 4: Configure audio output to file
    audio_config = speechsdk.audio.AudioOutputConfig(filename=wav_file)

    # Step 5: Create synthesizer
    speech_synthesizer = speechsdk.SpeechSynthesizer(
        speech_config=speech_config,
        audio_config=audio_config
    )

    # Step 6: Synthesize speech (blocking call)
    result = speech_synthesizer.speak_text_async(cleaned_text).get()

    # Step 7: Check result
    if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:
        logger.debug("✓ TTS synthesis completed")
        return wav_file
    else:
        error_details = result.cancellation_details
        raise Exception(f"Speech synthesis failed: {error_details.reason}")

def convert_wav_to_pcm(input_wav, output_wav):
    try:
        subprocess.run(
            ["ffmpeg", "-i", input_wav, "-acodec", "pcm_s16le", output_wav],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
        )
        return output_wav if os.path.exists(output_wav) else None
    except subprocess.CalledProcessError as e:
        logger.error(f"ffmpeg conversion failed: {e.stderr.decode()}")
        return None

def generate_lip_sync(wav_file, user_id):
    json_file = os.path.join(OUTPUT_DIR, f"{user_id}.json")

    try:
        result = subprocess.run(
            [rhubarbExePath, "-f", "json", "-o", json_file, wav_file, "-r", "phonetic"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        )
        return json_file if os.path.exists(json_file) else None
    except Exception as e:
        logger.error(f"Rhubarb lip sync failed: {e}")
        return None

def parse_lip_sync(json_file, sound_file):
    with open(json_file, "r") as file:
        lip_sync_data = json.load(file)

    lip_sync_data["metadata"]["soundFile"] = sound_file  # Add reference
    return lip_sync_data

{
    "metadata": {
        "soundFile": "tts_audio/session-abc123.wav",  // ‹‹‹ ADDED
        "duration": 3.5
    },
    "mouthCues": [/* ... */]
}

GET /lip-sync?message=Hello%2C%20welcome!&user_id=User-123&voice_selection=Female_2&project_id=User-123_Project_1

{
    "lipsync": {
        "metadata": {
            "soundFile": "tts_audio/User-123.wav",
            "duration": 2.5
        },
        "mouthCues": [
            {"start": 0.00, "end": 0.15, "value": "X"},
            {"start": 0.15, "end": 0.30, "value": "B"},
            // ...
        ]
    },
    "audio": "UklGRiQAAABXQVZFZm10..."  // Base64-encoded WAV
}

if project_id:
    sel = selection_collection.find_one({"user_id": user_id, "project_id": project_id})
    if sel:
        latest_voice = sel.get("selection_voice") or voice_selection
        latest_avatar = sel.get("selection_avatar") or ""
        voice_selection = assign_voice_by_avatar(str(latest_avatar), str(latest_voice))

def call_model_name(messages):
    try:
        # API call with model-specific configuration
        response = client.chat.completions.create(model="...", messages=messages, temperature=0.7)
        return response.choices[0].message.content
    except Exception as e:
        return f"Model API Error: {str(e)}"

# Configuration (Lines 715-724)
endpoint_gpt4 = "https://machineagentopenai.openai.azure.com/.../gpt-4-0613/..."
deployment_gpt4 = "gpt-4-0613"
subscription_key = "AZxDVMYB08AaUip0i5ed1sy73ZpUsqencYYxKDbm6nfWfG1AqPZ3JQQJ99BCACYeBjFXJ3w3AAABACOGVUo7"  # ⚠️ HARDCODED

client = AzureOpenAI(azure_endpoint=endpoint_gpt4, api_key=subscription_key, api_version="2024-02-15-preview")

def call_openai_4(messages):
    response = client.chat.completions.create(model="deployment_gpt4", messages=messages, temperature=0.7)
    return response.choices[0].message.content

endpoint_gpt35 = "https://machineagentopenai.openai.azure.com/.../gpt-35-turbo-16k-0613/..."
deployment_gpt35 = "gpt-35-turbo-16k-0613"
# Uses same subscription_key as GPT-4

client_gpt35 = AzureOpenAI(azure_endpoint=endpoint_gpt35, api_key=subscription_key, api_version="2024-02-15-preview")

endpoint_gpt4omini = "https://machineagentopenai.openai.azure.com/.../gpt-4o-mini/..."
deployment_gpt4omini = "gpt-4o-mini"

endpoint_gpto1mini = "https://machineagentopenai.openai.azure.com/.../o1-mini/..."
deployment_gpto1mini = "o1-mini"

AZURE_LLAMA_ENDPOINT = "https://Llama-3-3-70B-Instruct-ulmca.eastus.models.ai.azure.com/chat/completions"
AZURE_API_KEY = "JOfcw0VW0dS31Z8XgkNRSP9tUaBiwUYZ"  # ⚠️ HARDCODED

llama_HEADERS = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {AZURE_API_KEY}"
}

def call_llama(messages):
    payload = {"messages": messages, "temperature": 0.7, "max_tokens": 50}
    response = requests.post(AZURE_LLAMA_ENDPOINT, json=payload, headers=llama_HEADERS)
    return response.json()["choices"][0]["message"]["content"]

deepseek_api_url = "https://DeepSeek-R1-imalr.eastus2.models.ai.azure.com/chat/completions"
deepseek_api_key = "GwUcGzHhhUbvApfMR4aq1ZPFUic6lbWE"  # ⚠️ HARDCODED

def call_deepseek(messages):
    data = {"messages": messages, "max_tokens": 100}
    response = requests.post(deepseek_api_url, headers=deepseekheaders, json=data)
    answer = response.json()["choices"][0]["message"]["content"]
    # Remove reasoning tokens
    return re.sub(r"<think>.*?</think>", "", answer, flags=re.DOTALL).strip()

Ministral_api_url = "https://Ministral-3B-rvgab.eastus2.models.ai.azure.com/chat/completions"
Ministral_api_key = "Z7fNcdnw5Tht1xAz6VlgUlLOeZoVTkIf"  # ⚠️ HARDCODED

phi_api_url = "https://Phi-3-small-8k-instruct-qvlpq.eastus2.models.ai.azure.com/chat/completions"
phi_api_key = "T8I14He3lbMyAyUwNfffwG58e23EcXsU"  # ⚠️ HARDCODED

GEMINI_API_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=AIzaSyCxpr7wiyrPAEFRW-nz8kQjT2Y4tZXgv9g"
GEMINI_API_KEY = "AIzaSyCxpr7wiyrPAEFRW-nz8kQjT2Y4tZXgv9g"  # ⚠️ HARDCODED (also in URL)

def call_gemini(messages):
    payload = {
        "contents": [{"parts": [{"text": msg['content']}]} for msg in messages]
    }
    response = requests.post(GEMINI_API_ENDPOINT, json=payload, headers=HEADERS_GEMINI)
    return response.json()["candidates"][0]["content"]["parts"][0]["text"]

CLAUDE_API_ENDPOINT = "https://api.anthropic.com/v1/messages"
CLAUDE_API_KEY = "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"  # ⚠️ HARDCODED

HEADERS_CLAUDE = {"Content-Type": "application/json", "X-API-Key": f"{CLAUDE_API_KEY}"}

def call_claude(messages):
    payload = {
        "model": "claude-3-opus-20240229",
        "messages": messages,
        "max_tokens": 1024
    }
    response = requests.post(CLAUDE_API_ENDPOINT, json=payload, headers=HEADERS_CLAUDE)
    return response.json()["content"][0]["text"]

GROK_API_ENDPOINT = "https://machineagents-resource.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview"
GROK_API_KEY = "DZHPHaEk96KHbCgaD3fsaI7opVSL7pFjmLR94hYkw8w7nnTlz0SPJQQJ99BEACHYHv6XJ3w3AAAAACOGn1JK"  # ⚠️ HARDCODED

def call_grok(messages):
    payload = {"messages": messages, "max_tokens": 1024}
    response = requests.post(GROK_API_ENDPOINT, json=payload, headers=HEADERS_GROK)
    return response.json()["choices"][0]["message"]["content"]

model = selection.get("selection_model", "openai-35") if selection else "openai-4"

if model == "openai-4":
    answer = call_openai_4(messages)
elif model == "openai-35":
    answer = call_openai_35(messages)
elif model == "gpt-4o-mini":
    answer = call_gpt4o_mini(messages)
elif model == "o1-mini":
    answer = call_gpto1mini(messages)
elif model == "Azure-llama":
    answer = call_llama(messages)
elif model == "Azure-deepseek":
    answer = call_deepseek(messages)
elif model == "Azure-Ministral":
    answer = call_Ministral(messages)
elif model == "Azure-phi":
    answer = call_phi(messages)
elif model == "gemini":
    answer = call_gemini(messages)
elif model == "claude":
    answer = call_claude(messages)
elif model == "grok":
    answer = call_grok(messages)
else:
    answer = call_openai_35(messages)  # Fallback

def get_org_data():
    org_docs, org_source = db_manager.find_with_fallback(
        "organisation_data",
        user_id=user_id,
        project_id=project_id,
        query={"user_id": user_id, "project_id": project_id},
        limit=1
    )
    return org_docs[0] if org_docs else None

combined_text = org_data_doc.get("combined_text", "") if org_data_doc else ""

{
    "user_id": "User-123456",
    "project_id": "User-123456_Project_1",
    "combined_text": "Company: Acme Corp. Founded: 2010. Products: Widget A, Widget B, Widget C. Mission: Make the best widgets.",
    "company_name": "Acme Corp",
    "industry": "Manufacturing",
    "website": "https://acme.com"
}

context_parts.append(f"Organisation Info:\n{combined_text}")

phone_number_pattern = r"\+?[0-9]{1,4}[-.\ s]?[0-9]{1,3}[-.\ s]?[0-9]{3}[-.\ s]?[0-9]{3,4}"
# Replaced with: "the number provided in the chat"

url_pattern = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
# Replaced with: "via the url provided in the chat"

pattern = r"[#*]"
# Removed completely (avoid reading "hash" or "asterisk")

emoji_pattern = re.compile(
    "["
    "\U0001F600-\U0001F64F"  # emoticons
    "\U0001F300-\U0001F5FF"  # symbols & pictographs
    "\U0001F680-\U0001F6FF"  # transport & map symbols
    "\U0001F1E0-\U0001F1FF"  # flags (iOS)
    "\U00002700-\U000027BF"  # Dingbats
    "\U0001F900-\U0001F9FF"  # Supplemental Symbols
    "\U00002600-\U000026FF"  # Misc symbols
    "\U00002B00-\U00002BFF"  # Arrows
    "]+",
    flags=re.UNICODE
)
# Removed completely

# Input (LLM response)
"🎉 Call us at +1-555-123-4567 or visit https://example.com! **Great deals!** #sale"

# Output (for TTS)
"Call us at the number provided in the chat or visit via the url provided in the chat! Great deals! sale"

if not chatbot_config or chatbot_config.get("isDeleted") is True:
    error_text = "This chatbot is no longer available."

    # Use chatbot's configured voice
    chatbot_voice_default = assign_voice_by_avatar(chatbot_avatar, chatbot_voice)
    azure_voice = SUPPORTED_VOICES.get(chatbot_voice_default)

    # Generate TTS + Lip-Sync for error
    base_filename = f"{user_id}_{session_id}_error"
    wav_file = await text_to_speech_azure1(error_text, azure_voice, wav_file)
    json_file = await generate_lip_sync1(wav_file, json_file)

    # Return SAME format as normal response
    return {
        "text": error_text,
        "facialExpression": "neutral",
        "animation": "Idle",
        "audio": audio_base64,
        "lipsync": lip_sync_data
    }

except Exception as e:
    logger.error(f"Unexpected error in sync_ask_gpt1: {e}")

    # Fallback: Minimal LLM call
    try:
        fallback_messages = [
            SystemMessage(content=system_prompts.get(chatbot_purpose, "You are a helpful AI assistant.")),
            HumanMessage(content=f"Question: {question}")
        ]
        response = llm.invoke(fallback_messages)
        answer = re.sub(r'(?i)^(answer:|response:|responds:| *-+ #?)\s*', '', response.content).strip()

        # Save fallback history
        save_chat_history(user_id, project_id, session_id, question, answer, originating_url)

        # Form trigger logic (duplicate from main flow)
        # ... check if form should be shown ...

        return {"text": answer}  # No audio/lipsync in fallback

    except Exception as fallback_error:
        raise HTTPException(status_code=500, detail=f"An error occurred: {e}")

subscription="9N41NOfDyVDoduiD4EjlzmZU9CbUX3pPqWfLCORpl7cBf0l2lzVQJQQJ99BCACGhslBXJ3w3AAAYACOG2329"

subscription_key = "AZxDVMYB08AaUip0i5ed1sy73ZpUsqencYYxKDbm6nfWfG1AqPZ3JQQJ99BCACOGVUo7"

AZURE_API_KEY = "JOfcw0VW0dS31Z8XgkNRSP9tUaBiwUYZ"

deepseek_api_key = "GwUcGzHhhUbvApfMR4aq1ZPFUic6lbWE"

Ministral_api_key = "Z7fNcdnw5Tht1xAz6VlgUlLOeZoVTkIf"

phi_api_key = "T8I14He3lbMyAyUwNfffwG58e23EcXsU"

GEMINI_API_KEY = "AIzaSyCxpr7wiyrPAEFRW-nz8kQjT2Y4tZXgv9g"

CLAUDE_API_KEY = "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"

GROK_API_KEY = "DZHPHaEk96KHbCgaD3fsaI7opVSL7pFjmLR94hYkw8w7nnTlz0SPJQQJ99BEACHYHv6XJ3w3AAAAACOGn1JK"

import uuid
unique_id = f"{session_id}_{uuid.uuid4()}"
wav_file = os.path.join(OUTPUT_DIR, f"{unique_id}.wav")

FROM python:3.9-slim

WORKDIR /app

# Install system dependencies
RUN apt-get update && apt-get install -y \
    build-essential \
    ffmpeg \
    && rm -rf /var/lib/apt/lists/*

COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy Rhubarb executable
COPY Rhubarb/ ./Rhubarb/

# Copy shared modules
COPY shared/ ./shared/

COPY src/ .

# Create TTS output directory
RUN mkdir -p tts_audio

# Make Rhubarb executable
RUN chmod +x Rhubarb/rhubarb

EXPOSE 8011

CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8011"]

fastapi>=0.95.0
uvicorn[standard]>=0.22.0
pymongo>=4.3.3
python-multipart>=0.0.6
python-dotenv>=1.0.0

# Azure Services
azure-cognitiveservices-speech>=1.30.0
openai>=1.0.0

# LangChain
langchain>=0.1.0
langchain-openai>=0.0.5
langchain-core>=0.1.0

# Embeddings & ML
fastembed>=0.1.0
scikit-learn>=1.3.0
numpy>=1.24.0

# Utilities
tiktoken>=0.5.0
pytz>=2023.3
requests>=2.31.0

# Monitoring
ddtrace>=1.19.0

# Azure OpenAI
AZURE_OPENAI_API_KEY=<your-key>
ENDPOINT_URL=https://machineagentopenai.openai.azure.com/...
DEPLOYMENT_NAME=gpt-35-turbo-16k-0613

# MongoDB
MONGO_URI=mongodb://...
MONGO_DB_NAME=Machine_agent_demo

# Milvus
MILVUS_HOST=localhost
MILVUS_PORT=19530

# Alternative LLM APIs (optional)
GEMINI_API_KEY=<your-key>
CLAUDE_API_KEY=<your-key>
GROK_API_KEY=<your-key>

# DataDog
DD_SERVICE=response-3d-chatbot-service
DD_ENV=production