AI Voice Assistant API Integration in Flutter: A Practical End-to-End Guide

dependencies:
  flutter:
    sdk: flutter
  speech_to_text: any       # STT for quick prototyping (tap-to-talk)
  flutter_tts: any          # Speak responses
  http: any                 # Simple request/response API calls
  web_socket_channel: any   # For streaming upgrades
  permission_handler: any   # Microphone permission prompts

flutter pub get

<uses-permission android:name="android.permission.RECORD_AUDIO" />

<key>NSMicrophoneUsageDescription</key>
<string>We use the microphone to capture your voice commands.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>We use speech recognition to turn your voice into text.</string>

import 'dart:async';
import 'dart:convert';
import 'package:flutter/foundation.dart';
import 'package:http/http.dart' as http;
import 'package:speech_to_text/speech_to_text.dart' as stt;
import 'package:flutter_tts/flutter_tts.dart';
import 'package:permission_handler/permission_handler.dart';

class VoiceController with ChangeNotifier {
  final _stt = stt.SpeechToText();
  final _tts = FlutterTts();

  bool _isListening = false;
  bool _isSpeaking = false;
  String _partialTranscript = '';
  String _finalTranscript = '';
  String _assistantReply = '';
  String? conversationId; // maintain context server-side

  bool get isListening => _isListening;
  bool get isSpeaking => _isSpeaking;
  String get partialTranscript => _partialTranscript;
  String get finalTranscript => _finalTranscript;
  String get assistantReply => _assistantReply;

  Future<bool> _ensureMicPermission() async {
    final status = await Permission.microphone.request();
    return status.isGranted;
  }

  Future<void> startListening() async {
    if (!await _ensureMicPermission()) return;

    // If TTS is speaking, stop to allow barge-in
    if (_isSpeaking) {
      await _tts.stop();
      _isSpeaking = false;
    }

    final available = await _stt.initialize(
      onStatus: (status) {},
      onError: (err) {
        debugPrint('STT error: $err');
      },
    );

    if (!available) return;

    _isListening = true;
    _partialTranscript = '';
    _finalTranscript = '';
    notifyListeners();

    await _stt.listen(
      onResult: (res) {
        _partialTranscript = res.recognizedWords;
        if (res.finalResult) {
          _finalTranscript = res.recognizedWords;
          _isListening = false;
          notifyListeners();
          _stt.stop();
          _sendToAssistant(_finalTranscript);
        } else {
          notifyListeners();
        }
      },
      listenMode: stt.ListenMode.confirmation,
      partialResults: true,
      cancelOnError: true,
    );
  }

  Future<void> stopListening() async {
    if (_isListening) {
      await _stt.stop();
      _isListening = false;
      notifyListeners();
    }
  }

  Future<void> _sendToAssistant(String text) async {
    if (text.trim().isEmpty) return;
    try {
      final resp = await http.post(
        Uri.parse('https://api.your-assistant.example/v1/chat'),
        headers: {
          'Authorization': 'Bearer YOUR_APP_TOKEN',
          'Content-Type': 'application/json',
        },
        body: jsonEncode({
          'conversation_id': conversationId,
          'user': text,
          'mode': 'text', // or 'voice' if your API accepts audio
        }),
      );
      if (resp.statusCode == 200) {
        final data = jsonDecode(resp.body) as Map<String, dynamic>;
        conversationId = data['conversation_id'] as String? ?? conversationId;
        _assistantReply = data['assistant'] as String? ?? '';
        notifyListeners();
        await _speak(_assistantReply);
      } else {
        debugPrint('Assistant error: ${resp.statusCode} ${resp.body}');
      }
    } catch (e) {
      debugPrint('Assistant exception: $e');
    }
  }

  Future<void> _speak(String text) async {
    if (text.isEmpty) return;
    _isSpeaking = true;
    notifyListeners();
    await _tts.setLanguage('en-US');
    await _tts.setSpeechRate(0.95); // slightly slower for clarity
    await _tts.speak(text);
    _isSpeaking = false;
    notifyListeners();
  }
}

import 'package:flutter/material.dart';
import 'voice_controller.dart';
import 'package:provider/provider.dart';

class VoicePage extends StatelessWidget {
  const VoicePage({super.key});

  @override
  Widget build(BuildContext context) {
    return ChangeNotifierProvider(
      create: (_) => VoiceController(),
      child: Scaffold(
        appBar: AppBar(title: const Text('AI Voice Assistant')),
        body: const _Body(),
        floatingActionButton: const _MicButton(),
        floatingActionButtonLocation: FloatingActionButtonLocation.centerFloat,
      ),
    );
  }
}

class _Body extends StatelessWidget {
  const _Body();
  @override
  Widget build(BuildContext context) {
    final c = context.watch<VoiceController>();
    return Padding(
      padding: const EdgeInsets.all(16),
      child: Column(
        crossAxisAlignment: CrossAxisAlignment.start,
        children: [
          Text('You said:', style: Theme.of(context).textTheme.titleMedium),
          const SizedBox(height: 8),
          Text(c.partialTranscript.isNotEmpty ? c.partialTranscript : c.finalTranscript),
          const Divider(height: 32),
          Text('Assistant:', style: Theme.of(context).textTheme.titleMedium),
          const SizedBox(height: 8),
          Text(c.assistantReply),
          const Spacer(),
          if (c.isSpeaking) const LinearProgressIndicator(minHeight: 2),
        ],
      ),
    );
  }
}

class _MicButton extends StatelessWidget {
  const _MicButton();
  @override
  Widget build(BuildContext context) {
    final c = context.watch<VoiceController>();
    return FloatingActionButton.extended(
      onPressed: () => c.isListening ? c.stopListening() : c.startListening(),
      icon: Icon(c.isListening ? Icons.stop : Icons.mic),
      label: Text(c.isListening ? 'Stop' : 'Speak'),
    );
  }
}

{
  "conversation_id": "abc123",
  "user": "What are the top 3 things to do in Austin this weekend?",
  "mode": "text"
}

import 'dart:convert';
import 'package:web_socket_channel/web_socket_channel.dart';

late final WebSocketChannel channel;

void connect() {
  channel = WebSocketChannel.connect(Uri.parse('wss://api.your-assistant.example/v1/stream'));
  channel.sink.add(jsonEncode({
    'type': 'start',
    'token': 'EPHEMERAL_TOKEN',
    'config': {
      'language': 'en-US',
      'sample_rate_hz': 16000,
      'send_partial_results': true
    }
  }));

  channel.stream.listen((message) {
    final event = jsonDecode(message as String);
    switch (event['type']) {
      case 'partial_transcript':
        // update UI
        break;
      case 'final_transcript':
        // show final user text
        break;
      case 'assistant_delta':
        // stream assistant text or audio
        break;
      case 'end':
        // cleanup
        break;
      case 'error':
        // show error
        break;
    }
  });
}