409 lines
15 KiB
Dart
409 lines
15 KiB
Dart
import 'dart:async';
|
|
import 'dart:io';
|
|
import 'package:flutter/foundation.dart';
|
|
import 'package:mymuseum_visitapp/Helpers/translationHelper.dart';
|
|
import 'package:just_audio/just_audio.dart';
|
|
import 'package:mobile_scanner/mobile_scanner.dart';
|
|
import 'package:mymuseum_visitapp/Models/visitContext.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/llm_client.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/stt_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/tts_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/wake_word_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/meta_glasses_service.dart';
|
|
|
|
/// Instance active de l'orchestrateur, accessible globalement.
|
|
/// Gérée par VoiceController — ne pas modifier directement.
|
|
VoiceOrchestrator? activeVoiceOrchestrator;
|
|
|
|
/// Orchestre le pipeline complet mains-libres :
|
|
/// WakeWord → STT → dispatch → LLM ou QR scan → TTS
|
|
///
|
|
/// Toutes les dépendances sont injectées via les interfaces abstraites
|
|
/// pour pouvoir swapper chaque maillon indépendamment.
|
|
class VoiceOrchestrator {
|
|
final VisitAppContext visitAppContext;
|
|
final WakeWordEngine wakeWordEngine;
|
|
final SttEngine sttEngine;
|
|
final TtsEngine ttsEngine;
|
|
final LlmClient llmClient;
|
|
|
|
bool _running = false;
|
|
bool _inConversation = false;
|
|
|
|
final ValueNotifier<String> lastTranscription = ValueNotifier('');
|
|
final ValueNotifier<bool> isListeningForCommand = ValueNotifier(false);
|
|
final ValueNotifier<String> lastTtsText = ValueNotifier('');
|
|
final ValueNotifier<List<String>> visitPhotosNotifier = ValueNotifier([]);
|
|
final ValueNotifier<String?> lastQrScanPhoto = ValueNotifier(null);
|
|
|
|
final List<String> visitPhotos = [];
|
|
|
|
static const String _wakeSound = 'assets/sounds/wake_detected.mp3';
|
|
static const String _thinkingSound = 'assets/sounds/thinking.mp3';
|
|
static const String _doneSound = 'assets/sounds/done.mp3';
|
|
final AudioPlayer _soundPlayer = AudioPlayer();
|
|
final AudioPlayer _thinkingPlayer = AudioPlayer();
|
|
|
|
final Map<String, int> _lastQrTime = {};
|
|
static const int _qrCooldownMs = 10000;
|
|
|
|
static final RegExp _urlPattern1 =
|
|
RegExp(r'https://web\.mymuseum\.be/([^/]+)/([^/]+)/([^/\s]+)');
|
|
static final RegExp _urlPattern2 =
|
|
RegExp(r'https://web\.myinfomate\.be/([^/]+)/([^/]+)/([^/\s]+)');
|
|
|
|
VoiceOrchestrator({
|
|
required this.visitAppContext,
|
|
required this.wakeWordEngine,
|
|
required this.sttEngine,
|
|
required this.ttsEngine,
|
|
required this.llmClient,
|
|
});
|
|
|
|
Future<void> start() async {
|
|
if (_running) return;
|
|
_running = true;
|
|
await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
debugPrint('[VoiceOrchestrator] Started');
|
|
}
|
|
|
|
Future<void> stop() async {
|
|
await wakeWordEngine.stop();
|
|
await sttEngine.cancel();
|
|
await ttsEngine.stop();
|
|
_running = false;
|
|
debugPrint('[VoiceOrchestrator] Stopped');
|
|
}
|
|
|
|
bool get isRunning => _running;
|
|
bool get isInConversation => _inConversation;
|
|
bool get isListening => _running && !_inConversation;
|
|
|
|
Future<void> restartWakeWord() async {
|
|
if (!_running || _inConversation) return;
|
|
await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
|
|
Future<void> triggerConversation() => _handleConversation();
|
|
Future<void> dispatchCommand(String command) => _dispatch(command);
|
|
|
|
Future<void> triggerQrScan(String imagePath) async {
|
|
final qr = await _tryDecodeQr(imagePath);
|
|
if (qr != null) await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
}
|
|
|
|
// ── Wake word ──────────────────────────────────────────────────────────────
|
|
|
|
void _onWakeWord() async {
|
|
if (_inConversation) return;
|
|
_inConversation = true;
|
|
await wakeWordEngine.stop();
|
|
await _stopThinkingLoop();
|
|
_playWakeSound();
|
|
await Future.delayed(const Duration(milliseconds: 200));
|
|
try {
|
|
await _handleConversation();
|
|
} finally {
|
|
_inConversation = false;
|
|
if (_running) await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
}
|
|
|
|
void _onWakeWordWithCommand(String inlineCommand) async {
|
|
if (_inConversation) return;
|
|
_inConversation = true;
|
|
await wakeWordEngine.stop();
|
|
await _stopThinkingLoop();
|
|
_playWakeSound();
|
|
await Future.delayed(const Duration(milliseconds: 200));
|
|
try {
|
|
if (inlineCommand.isNotEmpty) {
|
|
debugPrint('[VoiceOrchestrator] Inline command: "$inlineCommand"');
|
|
await _dispatch(inlineCommand);
|
|
} else {
|
|
await _handleConversation();
|
|
}
|
|
} finally {
|
|
_inConversation = false;
|
|
if (_running) await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
}
|
|
|
|
Future<void> _playSound(String asset) async {
|
|
try {
|
|
await _soundPlayer.setAsset(asset);
|
|
await _soundPlayer.play();
|
|
} catch (_) {
|
|
debugPrint('[VoiceOrchestrator] Sound not found: $asset');
|
|
}
|
|
}
|
|
|
|
Future<void> _playWakeSound() => _playSound(_wakeSound);
|
|
Future<void> _playDoneSound() => _playSound(_doneSound);
|
|
|
|
Future<void> _startThinkingLoop() async {
|
|
try {
|
|
await _thinkingPlayer.setAsset(_thinkingSound);
|
|
await _thinkingPlayer.setLoopMode(LoopMode.one);
|
|
_thinkingPlayer.play();
|
|
} catch (_) {
|
|
debugPrint('[VoiceOrchestrator] Thinking sound not found');
|
|
}
|
|
}
|
|
|
|
Future<void> _stopThinkingLoop() async {
|
|
await _thinkingPlayer.stop();
|
|
}
|
|
|
|
// ── Conversation vocale ────────────────────────────────────────────────────
|
|
|
|
Future<void> _handleConversation() async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
isListeningForCommand.value = true;
|
|
final command = await sttEngine.transcribeOnce(languageCode: langCode);
|
|
isListeningForCommand.value = false;
|
|
debugPrint('[VoiceOrchestrator] Command: "$command"');
|
|
if (command.isEmpty) return;
|
|
lastTranscription.value = command;
|
|
await _dispatch(command);
|
|
}
|
|
|
|
Future<void> _dispatch(String command, {bool continueConversation = true}) async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
|
|
if (_isStopCommand(command)) {
|
|
debugPrint('[VoiceOrchestrator] Annulé: "$command"');
|
|
await _stopThinkingLoop();
|
|
return;
|
|
}
|
|
|
|
if (_isQrScanCommand(command)) { await _handleQrScan(); return; }
|
|
if (_isPhotoCommand(command)) { await _handlePhotoCapture(); return; }
|
|
|
|
if (_isRepeatCommand(command)) {
|
|
await ttsEngine.replay();
|
|
} else {
|
|
bool expectsReply = true;
|
|
try {
|
|
await _startThinkingLoop();
|
|
final result = await llmClient.chat(
|
|
command,
|
|
configurationId: visitAppContext.configuration?.id,
|
|
languageCode: lang,
|
|
);
|
|
expectsReply = result.expectsReply;
|
|
await _stopThinkingLoop();
|
|
if (result.reply.isNotEmpty) {
|
|
lastTtsText.value = result.reply;
|
|
await _playDoneSound();
|
|
await ttsEngine.speak(result.reply, languageCode: langCode);
|
|
}
|
|
} catch (e) {
|
|
await _stopThinkingLoop();
|
|
debugPrint('[VoiceOrchestrator] LLM error: $e');
|
|
return;
|
|
}
|
|
if (!expectsReply) return;
|
|
}
|
|
|
|
if (continueConversation) await _listenForFollowUp();
|
|
}
|
|
|
|
Future<void> _listenForFollowUp() async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
isListeningForCommand.value = true;
|
|
final followUp = await sttEngine.transcribeOnce(
|
|
languageCode: langCode,
|
|
timeout: const Duration(seconds: 5),
|
|
);
|
|
isListeningForCommand.value = false;
|
|
if (followUp.isNotEmpty) lastTranscription.value = followUp;
|
|
if (followUp.isEmpty || _isStopCommand(followUp)) return;
|
|
await _dispatch(followUp, continueConversation: true);
|
|
}
|
|
|
|
// ── QR scan ───────────────────────────────────────────────────────────────
|
|
|
|
Future<void> _handleQrScan() async {
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
final completer = Completer<String?>();
|
|
final prevCallback = MetaGlassesService.instance.onPhotoCaptured;
|
|
MetaGlassesService.instance.onPhotoCaptured = (path) {
|
|
if (!completer.isCompleted) completer.complete(path.isEmpty ? null : path);
|
|
prevCallback?.call(path);
|
|
};
|
|
Timer(const Duration(seconds: 25), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
unawaited(MetaGlassesService.instance.requestPhotoCapture());
|
|
final photoPath = await completer.future;
|
|
MetaGlassesService.instance.onPhotoCaptured = prevCallback;
|
|
|
|
if (photoPath == null) {
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.cameraUnavailable', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
return;
|
|
}
|
|
|
|
if (lastQrScanPhoto.value != null && lastQrScanPhoto.value != photoPath) {
|
|
try { File(lastQrScanPhoto.value!).deleteSync(); } catch (_) {}
|
|
}
|
|
lastQrScanPhoto.value = photoPath;
|
|
|
|
final qr = await _tryDecodeQr(photoPath);
|
|
if (qr != null) {
|
|
await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
} else {
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.noQrFound', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
}
|
|
}
|
|
|
|
Future<void> _handlePhotoCapture() async {
|
|
final completer = Completer<String?>();
|
|
final prevCallback = MetaGlassesService.instance.onPhotoCaptured;
|
|
MetaGlassesService.instance.onPhotoCaptured = (path) {
|
|
if (!completer.isCompleted) completer.complete(path);
|
|
prevCallback?.call(path);
|
|
};
|
|
Timer(const Duration(seconds: 25), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
unawaited(MetaGlassesService.instance.requestPhotoCapture());
|
|
final photoPath = await completer.future;
|
|
MetaGlassesService.instance.onPhotoCaptured = prevCallback;
|
|
|
|
if (photoPath == null || photoPath.isEmpty) {
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.photoFailed', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
return;
|
|
}
|
|
|
|
final qr = await _tryDecodeQr(photoPath);
|
|
if (qr != null) {
|
|
await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
} else {
|
|
visitPhotos.add(photoPath);
|
|
visitPhotosNotifier.value = List.unmodifiable(visitPhotos);
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.photoCaptured', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
}
|
|
}
|
|
|
|
Future<({String sectionId, String? configId})?> _tryDecodeQr(String imagePath) async {
|
|
final controller = MobileScannerController();
|
|
({String sectionId, String? configId})? result;
|
|
try {
|
|
final completer = Completer<({String sectionId, String? configId})?>();
|
|
final sub = controller.barcodes.listen((capture) {
|
|
for (final barcode in capture.barcodes) {
|
|
final raw = barcode.rawValue;
|
|
if (raw != null) {
|
|
final ids = _extractQrIds(raw);
|
|
if (ids != null && !completer.isCompleted) completer.complete(ids);
|
|
}
|
|
}
|
|
});
|
|
await controller.analyzeImage(imagePath);
|
|
Timer(const Duration(seconds: 2), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
result = await completer.future;
|
|
await sub.cancel();
|
|
} catch (e) {
|
|
debugPrint('[VoiceOrchestrator] QR decode error: $e');
|
|
} finally {
|
|
controller.dispose();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
({String sectionId, String? configId})? _extractQrIds(String raw) {
|
|
final m1 = _urlPattern1.firstMatch(raw);
|
|
if (m1 != null) return (sectionId: m1.group(3)!, configId: m1.group(2));
|
|
final m2 = _urlPattern2.firstMatch(raw);
|
|
if (m2 != null) return (sectionId: m2.group(3)!, configId: m2.group(2));
|
|
if (visitAppContext.sectionIds != null) {
|
|
return visitAppContext.sectionIds!.contains(raw)
|
|
? (sectionId: raw, configId: visitAppContext.configuration?.id)
|
|
: null;
|
|
}
|
|
return (sectionId: raw, configId: null);
|
|
}
|
|
|
|
Future<void> explainSection(String sectionId, {String? configurationId}) async {
|
|
final now = DateTime.now().millisecondsSinceEpoch;
|
|
if ((now - (_lastQrTime[sectionId] ?? 0)) < _qrCooldownMs) return;
|
|
_lastQrTime[sectionId] = now;
|
|
final cfgId = configurationId ?? visitAppContext.configuration?.id;
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
try {
|
|
final result = await llmClient.chat(
|
|
'Le visiteur vient de scanner le QR code de la section "$sectionId". '
|
|
'Appelle GetSectionDetail avec cet ID, puis présente le contenu de façon engageante en 2-3 phrases.',
|
|
configurationId: cfgId,
|
|
languageCode: lang,
|
|
);
|
|
if (result.reply.isNotEmpty) {
|
|
lastTtsText.value = result.reply;
|
|
await ttsEngine.speak(result.reply, languageCode: _toLangCode(lang));
|
|
}
|
|
} catch (e) {
|
|
debugPrint('[VoiceOrchestrator] explainSection error: $e');
|
|
}
|
|
}
|
|
|
|
// ── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
bool _isStopCommand(String text) {
|
|
final t = text.toLowerCase().trim();
|
|
return t == 'non' || t == 'rien' || t == 'non rien' || t == 'rien merci' ||
|
|
t == 'non merci' || t == 'laisse tomber' || t == 'annule' || t == 'annuler' ||
|
|
t.contains('stop') || t.contains('arrête') || t.contains('au revoir') ||
|
|
t.contains('c\'est bon') || t.contains('ok merci') || t.contains('laisse tomber') ||
|
|
(t.length < 10 && (t.contains('non') || t.contains('rien')));
|
|
}
|
|
|
|
bool _isQrScanCommand(String text) {
|
|
final t = text.toLowerCase();
|
|
return t.contains('scan') || t.contains('qr') || t.contains('code') || t.contains('regarde');
|
|
}
|
|
|
|
bool _isPhotoCommand(String text) {
|
|
final t = text.toLowerCase();
|
|
return t.contains('photo') || t.contains('prends') || t.contains('capture');
|
|
}
|
|
|
|
bool _isRepeatCommand(String text) {
|
|
final t = text.toLowerCase();
|
|
return t.contains('répète') || t.contains('repete') || t.contains('encore');
|
|
}
|
|
|
|
String _toLangCode(String lang) {
|
|
switch (lang.toUpperCase()) {
|
|
case 'FR': return 'fr-FR';
|
|
case 'NL': return 'nl-NL';
|
|
case 'EN': return 'en-US';
|
|
case 'DE': return 'de-DE';
|
|
default: return 'fr-FR';
|
|
}
|
|
}
|
|
}
|