import 'dart:async'; import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:mymuseum_visitapp/Helpers/translationHelper.dart'; import 'package:just_audio/just_audio.dart'; import 'package:mobile_scanner/mobile_scanner.dart'; import 'package:mymuseum_visitapp/Models/visitContext.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/llm_client.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/stt_engine.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/tts_engine.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/wake_word_engine.dart'; import 'package:mymuseum_visitapp/Services/meta_glasses_service.dart'; /// Instance active de l'orchestrateur, accessible globalement. /// Gérée par VoiceController — ne pas modifier directement. VoiceOrchestrator? activeVoiceOrchestrator; /// Orchestre le pipeline complet mains-libres : /// WakeWord → STT → dispatch → LLM ou QR scan → TTS /// /// Toutes les dépendances sont injectées via les interfaces abstraites /// pour pouvoir swapper chaque maillon indépendamment. class VoiceOrchestrator { final VisitAppContext visitAppContext; final WakeWordEngine wakeWordEngine; final SttEngine sttEngine; final TtsEngine ttsEngine; final LlmClient llmClient; bool _running = false; bool _inConversation = false; final ValueNotifier lastTranscription = ValueNotifier(''); final ValueNotifier isListeningForCommand = ValueNotifier(false); final ValueNotifier lastTtsText = ValueNotifier(''); final ValueNotifier> visitPhotosNotifier = ValueNotifier([]); final ValueNotifier lastQrScanPhoto = ValueNotifier(null); final List visitPhotos = []; static const String _wakeSound = 'assets/sounds/wake_detected.mp3'; static const String _thinkingSound = 'assets/sounds/thinking.mp3'; static const String _doneSound = 'assets/sounds/done.mp3'; final AudioPlayer _soundPlayer = AudioPlayer(); final AudioPlayer _thinkingPlayer = AudioPlayer(); final Map _lastQrTime = {}; static const int _qrCooldownMs = 10000; static final RegExp _urlPattern1 = RegExp(r'https://web\.mymuseum\.be/([^/]+)/([^/]+)/([^/\s]+)'); static final RegExp _urlPattern2 = RegExp(r'https://web\.myinfomate\.be/([^/]+)/([^/]+)/([^/\s]+)'); VoiceOrchestrator({ required this.visitAppContext, required this.wakeWordEngine, required this.sttEngine, required this.ttsEngine, required this.llmClient, }); Future start() async { if (_running) return; _running = true; await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); debugPrint('[VoiceOrchestrator] Started'); } Future stop() async { await wakeWordEngine.stop(); await sttEngine.cancel(); await ttsEngine.stop(); _running = false; debugPrint('[VoiceOrchestrator] Stopped'); } bool get isRunning => _running; bool get isInConversation => _inConversation; bool get isListening => _running && !_inConversation; Future restartWakeWord() async { if (!_running || _inConversation) return; await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } Future triggerConversation() => _handleConversation(); Future dispatchCommand(String command) => _dispatch(command); Future triggerQrScan(String imagePath) async { final qr = await _tryDecodeQr(imagePath); if (qr != null) await explainSection(qr.sectionId, configurationId: qr.configId); } // ── Wake word ────────────────────────────────────────────────────────────── void _onWakeWord() async { if (_inConversation) return; _inConversation = true; await wakeWordEngine.stop(); await _stopThinkingLoop(); _playWakeSound(); await Future.delayed(const Duration(milliseconds: 200)); try { await _handleConversation(); } finally { _inConversation = false; if (_running) await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } } void _onWakeWordWithCommand(String inlineCommand) async { if (_inConversation) return; _inConversation = true; await wakeWordEngine.stop(); await _stopThinkingLoop(); _playWakeSound(); await Future.delayed(const Duration(milliseconds: 200)); try { if (inlineCommand.isNotEmpty) { debugPrint('[VoiceOrchestrator] Inline command: "$inlineCommand"'); await _dispatch(inlineCommand); } else { await _handleConversation(); } } finally { _inConversation = false; if (_running) await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } } Future _playSound(String asset) async { try { await _soundPlayer.setAsset(asset); await _soundPlayer.play(); } catch (_) { debugPrint('[VoiceOrchestrator] Sound not found: $asset'); } } Future _playWakeSound() => _playSound(_wakeSound); Future _playDoneSound() => _playSound(_doneSound); Future _startThinkingLoop() async { try { await _thinkingPlayer.setAsset(_thinkingSound); await _thinkingPlayer.setLoopMode(LoopMode.one); _thinkingPlayer.play(); } catch (_) { debugPrint('[VoiceOrchestrator] Thinking sound not found'); } } Future _stopThinkingLoop() async { await _thinkingPlayer.stop(); } // ── Conversation vocale ──────────────────────────────────────────────────── Future _handleConversation() async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); isListeningForCommand.value = true; final command = await sttEngine.transcribeOnce(languageCode: langCode); isListeningForCommand.value = false; debugPrint('[VoiceOrchestrator] Command: "$command"'); if (command.isEmpty) return; lastTranscription.value = command; await _dispatch(command); } Future _dispatch(String command, {bool continueConversation = true}) async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); if (_isStopCommand(command)) { debugPrint('[VoiceOrchestrator] Annulé: "$command"'); await _stopThinkingLoop(); return; } if (_isQrScanCommand(command)) { await _handleQrScan(); return; } if (_isPhotoCommand(command)) { await _handlePhotoCapture(); return; } if (_isRepeatCommand(command)) { await ttsEngine.replay(); } else { bool expectsReply = true; try { await _startThinkingLoop(); final result = await llmClient.chat( command, configurationId: visitAppContext.configuration?.id, languageCode: lang, ); expectsReply = result.expectsReply; await _stopThinkingLoop(); if (result.reply.isNotEmpty) { lastTtsText.value = result.reply; await _playDoneSound(); await ttsEngine.speak(result.reply, languageCode: langCode); } } catch (e) { await _stopThinkingLoop(); debugPrint('[VoiceOrchestrator] LLM error: $e'); return; } if (!expectsReply) return; } if (continueConversation) await _listenForFollowUp(); } Future _listenForFollowUp() async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); isListeningForCommand.value = true; final followUp = await sttEngine.transcribeOnce( languageCode: langCode, timeout: const Duration(seconds: 5), ); isListeningForCommand.value = false; if (followUp.isNotEmpty) lastTranscription.value = followUp; if (followUp.isEmpty || _isStopCommand(followUp)) return; await _dispatch(followUp, continueConversation: true); } // ── QR scan ─────────────────────────────────────────────────────────────── Future _handleQrScan() async { final lang = _toLangCode(visitAppContext.language ?? 'FR'); final completer = Completer(); final prevCallback = MetaGlassesService.instance.onPhotoCaptured; MetaGlassesService.instance.onPhotoCaptured = (path) { if (!completer.isCompleted) completer.complete(path.isEmpty ? null : path); prevCallback?.call(path); }; Timer(const Duration(seconds: 25), () { if (!completer.isCompleted) completer.complete(null); }); unawaited(MetaGlassesService.instance.requestPhotoCapture()); final photoPath = await completer.future; MetaGlassesService.instance.onPhotoCaptured = prevCallback; if (photoPath == null) { lastTtsText.value = TranslationHelper.getFromLocale('voice.cameraUnavailable', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); return; } if (lastQrScanPhoto.value != null && lastQrScanPhoto.value != photoPath) { try { File(lastQrScanPhoto.value!).deleteSync(); } catch (_) {} } lastQrScanPhoto.value = photoPath; final qr = await _tryDecodeQr(photoPath); if (qr != null) { await explainSection(qr.sectionId, configurationId: qr.configId); } else { lastTtsText.value = TranslationHelper.getFromLocale('voice.noQrFound', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); } } Future _handlePhotoCapture() async { final completer = Completer(); final prevCallback = MetaGlassesService.instance.onPhotoCaptured; MetaGlassesService.instance.onPhotoCaptured = (path) { if (!completer.isCompleted) completer.complete(path); prevCallback?.call(path); }; Timer(const Duration(seconds: 25), () { if (!completer.isCompleted) completer.complete(null); }); unawaited(MetaGlassesService.instance.requestPhotoCapture()); final photoPath = await completer.future; MetaGlassesService.instance.onPhotoCaptured = prevCallback; if (photoPath == null || photoPath.isEmpty) { final lang = _toLangCode(visitAppContext.language ?? 'FR'); lastTtsText.value = TranslationHelper.getFromLocale('voice.photoFailed', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); return; } final qr = await _tryDecodeQr(photoPath); if (qr != null) { await explainSection(qr.sectionId, configurationId: qr.configId); } else { visitPhotos.add(photoPath); visitPhotosNotifier.value = List.unmodifiable(visitPhotos); final lang = _toLangCode(visitAppContext.language ?? 'FR'); lastTtsText.value = TranslationHelper.getFromLocale('voice.photoCaptured', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); } } Future<({String sectionId, String? configId})?> _tryDecodeQr(String imagePath) async { final controller = MobileScannerController(); ({String sectionId, String? configId})? result; try { final completer = Completer<({String sectionId, String? configId})?>(); final sub = controller.barcodes.listen((capture) { for (final barcode in capture.barcodes) { final raw = barcode.rawValue; if (raw != null) { final ids = _extractQrIds(raw); if (ids != null && !completer.isCompleted) completer.complete(ids); } } }); await controller.analyzeImage(imagePath); Timer(const Duration(seconds: 2), () { if (!completer.isCompleted) completer.complete(null); }); result = await completer.future; await sub.cancel(); } catch (e) { debugPrint('[VoiceOrchestrator] QR decode error: $e'); } finally { controller.dispose(); } return result; } ({String sectionId, String? configId})? _extractQrIds(String raw) { final m1 = _urlPattern1.firstMatch(raw); if (m1 != null) return (sectionId: m1.group(3)!, configId: m1.group(2)); final m2 = _urlPattern2.firstMatch(raw); if (m2 != null) return (sectionId: m2.group(3)!, configId: m2.group(2)); if (visitAppContext.sectionIds != null) { return visitAppContext.sectionIds!.contains(raw) ? (sectionId: raw, configId: visitAppContext.configuration?.id) : null; } return (sectionId: raw, configId: null); } Future explainSection(String sectionId, {String? configurationId}) async { final now = DateTime.now().millisecondsSinceEpoch; if ((now - (_lastQrTime[sectionId] ?? 0)) < _qrCooldownMs) return; _lastQrTime[sectionId] = now; final cfgId = configurationId ?? visitAppContext.configuration?.id; final lang = visitAppContext.language ?? 'FR'; try { final result = await llmClient.chat( 'Le visiteur vient de scanner le QR code de la section "$sectionId". ' 'Appelle GetSectionDetail avec cet ID, puis présente le contenu de façon engageante en 2-3 phrases.', configurationId: cfgId, languageCode: lang, ); if (result.reply.isNotEmpty) { lastTtsText.value = result.reply; await ttsEngine.speak(result.reply, languageCode: _toLangCode(lang)); } } catch (e) { debugPrint('[VoiceOrchestrator] explainSection error: $e'); } } // ── Helpers ──────────────────────────────────────────────────────────────── bool _isStopCommand(String text) { final t = text.toLowerCase().trim(); return t == 'non' || t == 'rien' || t == 'non rien' || t == 'rien merci' || t == 'non merci' || t == 'laisse tomber' || t == 'annule' || t == 'annuler' || t.contains('stop') || t.contains('arrête') || t.contains('au revoir') || t.contains('c\'est bon') || t.contains('ok merci') || t.contains('laisse tomber') || (t.length < 10 && (t.contains('non') || t.contains('rien'))); } bool _isQrScanCommand(String text) { final t = text.toLowerCase(); return t.contains('scan') || t.contains('qr') || t.contains('code') || t.contains('regarde'); } bool _isPhotoCommand(String text) { final t = text.toLowerCase(); return t.contains('photo') || t.contains('prends') || t.contains('capture'); } bool _isRepeatCommand(String text) { final t = text.toLowerCase(); return t.contains('répète') || t.contains('repete') || t.contains('encore'); } String _toLangCode(String lang) { switch (lang.toUpperCase()) { case 'FR': return 'fr-FR'; case 'NL': return 'nl-NL'; case 'EN': return 'en-US'; case 'DE': return 'de-DE'; default: return 'fr-FR'; } } }