Voice

{
  type: "speech-to-text",
  features: {
    languages: ["en-US", "es-ES", "fr-FR", "de-DE", "ja-JP", ...],
    accuracy: "high",
    realtime: true,
    streaming: true,
    punctuation: true,
    profanityFilter: optional
  }
}

{
  type: "text-to-speech",
  voices: {
    languages: ["en-US", "es-ES", "fr-FR", ...],
    genders: ["male", "female", "neutral"],
    styles: ["conversational", "professional", "cheerful", "empathetic"],
    customization: {
      speed: 0.5 to 2.0,
      pitch: -20 to +20,
      volume: 0 to 100
    }
  }
}

const phoneAgent = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "en-US",
      profanityFilter: true
    },
    output: {
      voice: "en-US-Neural2-C",
      style: "friendly"
    }
  },
  system: `You are a customer service representative for TechCorp.

  Call Flow:
  1. Greet the customer warmly
  2. Ask how you can help
  3. Listen to their issue
  4. Provide solution or escalate if needed
  5. Confirm resolution
  6. Thank them for calling

  Guidelines:
  - Be empathetic and patient
  - Keep responses concise (2-3 sentences)
  - Confirm understanding before providing solution
  - Offer to escalate if you cannot help`,
  mcpServers: [
    {
      serverName: "crm-system",
      serverUrl: "https://api.company.com/crm"
    }
  ]
});

// Handle call flow
phoneAgent.on('transcript', async (text) => {
  if (text.toLowerCase().includes('speak to human')) {
    await phoneAgent.transfer({
      destination: "human-support",
      context: phoneAgent.getConversationContext()
    });
  }
});

const assistant = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "en-US",
      wakeWord: "Hey Assistant", // Activate on wake word
      vadEnabled: true
    },
    output: {
      voice: "en-US-Neural2-J",
      style: "conversational"
    }
  },
  memory: {
    namespace: `user_${userId}`,
    enabled: true
  },
  system: `You are a personal voice assistant.

  Capabilities:
  - Set reminders and alarms
  - Check calendar and schedule meetings
  - Send messages
  - Get weather and news
  - Control smart home devices
  - Answer questions

  Keep responses brief and natural.`,
  mcpServers: [
    { serverName: "calendar", serverUrl: "..." },
    { serverName: "messaging", serverUrl: "..." },
    { serverName: "smart-home", serverUrl: "..." }
  ]
});

const languageTutor = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "es-ES", // Student speaks Spanish
      pronunciationFeedback: true
    },
    output: {
      voice: "es-ES-Neural2-A",
      speed: 0.8, // Slower for learning
      style: "conversational"
    }
  },
  system: `You are a Spanish language tutor.

  Teaching Approach:
  - Listen to student's pronunciation
  - Provide gentle corrections
  - Use simple vocabulary
  - Repeat phrases for practice
  - Encourage and praise progress
  - Adapt to student's level

  Focus on conversational fluency.`,
  config: {
    pronunciationAnalysis: true,
    vocabularyLevel: "beginner"
  }
});

languageTutor.on('pronunciation', (analysis) => {
  console.log('Pronunciation score:', analysis.score);
  console.log('Suggestions:', analysis.improvements);
});

const healthcareAgent = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "en-US",
      medicalTerminology: true
    },
    output: {
      voice: "en-US-Neural2-C",
      style: "empathetic",
      speed: 0.9
    }
  },
  system: `You are a healthcare information assistant.

  Guidelines:
  - Use clear, simple language
  - Be empathetic and reassuring
  - Never provide medical diagnosis
  - Always recommend consulting healthcare provider
  - Maintain HIPAA compliance
  - Focus on general health information

  Remind users this is not medical advice.`,
  rules: [
    "Never provide specific medical diagnoses",
    "Always recommend consulting a healthcare provider for medical concerns",
    "Do not access or share patient health information without authorization"
  ]
});

const navigationAgent = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "en-US",
      noiseReduction: true, // Filter road noise
      vadEnabled: true
    },
    output: {
      voice: "en-US-Neural2-D",
      volume: "adaptive", // Adjust to ambient noise
      style: "clear"
    }
  },
  system: `You are a voice navigation assistant.

  Provide:
  - Clear, timely directions
  - Traffic updates
  - Alternate routes if needed
  - Nearby points of interest
  - Weather warnings

  Keep instructions brief and precise.
  Announce turns well in advance.`,
  mcpServers: [
    { serverName: "maps", serverUrl: "..." },
    { serverName: "traffic", serverUrl: "..." }
  ]
});

const conferenceAssistant = await agentbase.createVoiceStream({
  voice: {
    input: {
      language: "en-US",
      multiSpeaker: true, // Identify different speakers
      vadEnabled: true
    },
    output: {
      voice: "en-US-Neural2-F",
      style: "professional"
    }
  },
  system: `You are a conference room assistant.

  Functions:
  - Start/end meetings
  - Dial participants
  - Control AV equipment
  - Take meeting notes
  - Set reminders
  - Book follow-up meetings

  Commands should be clear and confirmed.`,
  mcpServers: [
    { serverName: "zoom", serverUrl: "..." },
    { serverName: "calendar", serverUrl: "..." },
    { serverName: "room-control", serverUrl: "..." }
  ]
});

// Customer service: Friendly, warm
voice: {
  output: {
    voice: "en-US-Neural2-C",
    style: "friendly",
    pitch: 2
  }
}

// Professional assistant: Clear, professional
voice: {
  output: {
    voice: "en-US-Neural2-J",
    style: "professional",
    pitch: 0
  }
}

// Educational content: Patient, clear
voice: {
  output: {
    voice: "en-US-Neural2-A",
    style: "conversational",
    speed: 0.85
  }
}

voiceStream.on('error', (error) => {
  switch (error.type) {
    case 'microphone_access_denied':
      speakText("I need microphone access to hear you. Please check your settings.");
      break;

    case 'audio_format_unsupported':
      console.error('Audio format not supported:', error.format);
      // Fallback to supported format
      break;

    case 'network_error':
      speakText("I'm having trouble connecting. Please check your internet connection.");
      break;

    case 'transcription_failed':
      speakText("Sorry, I didn't catch that. Could you repeat?");
      break;

    default:
      speakText("Sorry, I'm having technical difficulties. Please try again.");
  }
});

// Implement privacy controls
const voiceConfig = {
  voice: {
    input: {
      recording: {
        enabled: userConsent, // Only record with consent
        retention: "session", // Delete after session
        encryption: true
      },
      profanityFilter: true,
      sensitiveDataRedaction: true // Redact SSN, credit cards, etc.
    }
  },
  privacy: {
    storeAudio: false, // Don't store audio recordings
    storeTranscripts: userConsent,
    anonymizeData: true
  }
};

const voiceAgent = await agentbase.createVoiceStream({
  voice: {
    enabled: true,
    output: { voice: "en-US-Neural2-C" }
  },
  memory: {
    namespace: `user_${userId}`,
    enabled: true
  },
  system: `Remember user preferences and past conversations.
  Reference previous discussions when relevant.`
});

// Agent recalls: "Last time we spoke about your project deadline..."

const voiceAgent = await agentbase.createVoiceStream({
  voice: {
    enabled: true
  },
  mcpServers: [
    {
      serverName: "calendar",
      serverUrl: "https://api.company.com/calendar"
    }
  ],
  system: `You can:
  - Check calendar
  - Schedule meetings
  - Set reminders

  Use tools to access user's calendar.`
});

const result = await agentbase.createVoiceStream({
  voice: { enabled: true },
  agents: [
    {
      name: "Main Assistant",
      description: "General assistance and routing"
    },
    {
      name: "Technical Support",
      description: "Technical troubleshooting",
      voice: {
        output: {
          voice: "en-US-Neural2-J", // Different voice
          style: "professional"
        }
      }
    },
    {
      name: "Billing Support",
      description: "Billing and payments",
      voice: {
        output: {
          voice: "en-US-Neural2-C",
          style: "friendly"
        }
      }
    }
  ]
});

// Voice changes when transferring between agents

// Minimize latency for real-time conversations
const voiceStream = await agentbase.createVoiceStream({
  voice: {
    input: {
      streaming: true,
      vadEnabled: true, // Detect speech end faster
      endpointingDelay: 500 // 500ms after speech ends
    },
    output: {
      streaming: true,
      format: "opus", // Compressed for faster streaming
      quality: "high" // Balance quality and latency
    }
  },
  config: {
    responseMode: "streaming", // Stream response as generated
    thinkingTime: "minimal" // Start responding quickly
  }
});

// Adaptive quality based on network
function getVoiceConfig(networkSpeed: string) {
  const configs = {
    fast: {
      format: "opus",
      sampleRate: 48000,
      quality: "high"
    },
    medium: {
      format: "opus",
      sampleRate: 24000,
      quality: "medium"
    },
    slow: {
      format: "mulaw",
      sampleRate: 8000,
      quality: "low"
    }
  };

  return configs[networkSpeed];
}

const voiceConfig = getVoiceConfig(detectedNetworkSpeed);

// Optimize costs for voice
const voiceStream = await agentbase.createVoiceStream({
  voice: {
    input: {
      vadEnabled: true, // Only transcribe actual speech
      silenceDetection: true // Stop transcription during silence
    },
    output: {
      caching: true, // Cache common responses
      compression: true
    }
  },
  config: {
    sessionTimeout: 300000, // 5 minute timeout
    maxSessionDuration: 1800000 // 30 minute max
  }
});

const voiceStream = await agentbase.createVoiceStream({
  voice: {
    input: {
      emotionDetection: true,
      sentimentAnalysis: true
    }
  },
  system: `Adapt your tone based on user emotion.
  If user sounds frustrated, be more empathetic.
  If user sounds happy, match their energy.`
});

voiceStream.on('emotion', (emotion) => {
  console.log('Detected emotion:', emotion.type);
  console.log('Confidence:', emotion.confidence);
  // Adjust response accordingly
});

const voiceStream = await agentbase.createVoiceStream({
  voice: {
    input: {
      speakerIdentification: true,
      voiceprint: userVoiceprint // Pre-enrolled voiceprint
    }
  },
  security: {
    requireVoiceAuth: true
  }
});

voiceStream.on('speaker-identified', (speaker) => {
  if (speaker.verified) {
    console.log('Verified user:', speaker.userId);
  } else {
    console.log('Unknown speaker');
  }
});

const voiceStream = await agentbase.createVoiceStream({
  voice: {
    input: {
      multiSpeaker: true,
      speakerDiarization: true // Separate different speakers
    }
  },
  system: `You are moderating a group conversation.
  Address speakers by name when identified.
  Manage turn-taking and keep discussion on track.`
});

voiceStream.on('speaker-change', (event) => {
  console.log('Now speaking:', event.speakerId);
});