import 'dart:async'; import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:mymuseum_visitapp/Helpers/translationHelper.dart'; import 'package:just_audio/just_audio.dart'; import 'package:mobile_scanner/mobile_scanner.dart'; import 'package:mymuseum_visitapp/Helpers/voiceLanguage.dart'; import 'package:mymuseum_visitapp/Models/visitContext.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/llm_client.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/stt_engine.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/tts_engine.dart'; import 'package:mymuseum_visitapp/Services/Glasses/engines/wake_word_engine.dart'; import 'package:mymuseum_visitapp/Services/meta_glasses_service.dart'; /// Instance active de l'orchestrateur, accessible globalement. /// Gérée par VoiceController — ne pas modifier directement. VoiceOrchestrator? activeVoiceOrchestrator; /// Orchestre le pipeline complet mains-libres : /// WakeWord → STT → dispatch → LLM ou QR scan → TTS /// /// Toutes les dépendances sont injectées via les interfaces abstraites /// pour pouvoir swapper chaque maillon indépendamment. class VoiceOrchestrator { final VisitAppContext visitAppContext; final WakeWordEngine wakeWordEngine; final SttEngine sttEngine; final TtsEngine ttsEngine; final LlmClient llmClient; bool _running = false; bool _inConversation = false; final ValueNotifier lastTranscription = ValueNotifier(''); final ValueNotifier isListeningForCommand = ValueNotifier(false); final ValueNotifier lastTtsText = ValueNotifier(''); final ValueNotifier> visitPhotosNotifier = ValueNotifier([]); final ValueNotifier lastQrScanPhoto = ValueNotifier(null); final List visitPhotos = []; static const String _wakeSound = 'assets/sounds/wake_detected.mp3'; static const String _thinkingSound = 'assets/sounds/thinking.mp3'; final AudioPlayer _soundPlayer = AudioPlayer(); final AudioPlayer _thinkingPlayer = AudioPlayer(); bool _soundsReady = false; /// Le son de réflexion n'apparaît qu'après ce délai : en dessous, le visiteur vient /// de finir de parler et n'attend pas encore de réponse — le silence s'y lit comme /// naturel, la nappe comme du bruit. static const Duration _thinkingDelay = Duration(milliseconds: 700); static const double _thinkingVolume = 0.45; Timer? _thinkingTimer; bool _thinkingWanted = false; final Map _lastQrTime = {}; static const int _qrCooldownMs = 10000; static final RegExp _urlPattern1 = RegExp(r'https://web\.mymuseum\.be/([^/]+)/([^/]+)/([^/\s]+)'); static final RegExp _urlPattern2 = RegExp(r'https://web\.myinfomate\.be/([^/]+)/([^/]+)/([^/\s]+)'); VoiceOrchestrator({ required this.visitAppContext, required this.wakeWordEngine, required this.sttEngine, required this.ttsEngine, required this.llmClient, }); Future start() async { if (_running) return; _running = true; await _preloadSounds(); await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); debugPrint('[VoiceOrchestrator] Started'); } Future stop() async { await wakeWordEngine.stop(); await sttEngine.cancel(); await ttsEngine.stop(); await _stopThinkingLoop(); _running = false; debugPrint('[VoiceOrchestrator] Stopped'); } bool get isRunning => _running; bool get isInConversation => _inConversation; bool get isListening => _running && !_inConversation; Future restartWakeWord() async { if (!_running || _inConversation) return; await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } Future triggerConversation() => _handleConversation(); Future dispatchCommand(String command) => _dispatch(command); Future triggerQrScan(String imagePath) async { final qr = await _tryDecodeQr(imagePath); if (qr != null) await explainSection(qr.sectionId, configurationId: qr.configId); } // ── Wake word ────────────────────────────────────────────────────────────── void _onWakeWord() async { if (_inConversation) return; _inConversation = true; await wakeWordEngine.stop(); await _stopThinkingLoop(); _playWakeSound(); await Future.delayed(const Duration(milliseconds: 200)); try { await _handleConversation(); } finally { _inConversation = false; if (_running) await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } } void _onWakeWordWithCommand(String inlineCommand) async { if (_inConversation) return; _inConversation = true; await wakeWordEngine.stop(); await _stopThinkingLoop(); _playWakeSound(); await Future.delayed(const Duration(milliseconds: 200)); try { if (inlineCommand.isNotEmpty) { debugPrint('[VoiceOrchestrator] Inline command: "$inlineCommand"'); await _dispatch(inlineCommand); } else { await _handleConversation(); } } finally { _inConversation = false; if (_running) await wakeWordEngine.start( onDetected: _onWakeWord, onDetectedWithCommand: _onWakeWordWithCommand, ); } } /// Charge les deux sons une fois pour toutes. Sans ça, `setAsset` décode à chaque /// wake word — sur le chemin le plus sensible à la latence de tout le pipeline. Future _preloadSounds() async { if (_soundsReady) return; try { await _soundPlayer.setAsset(_wakeSound); await _thinkingPlayer.setAsset(_thinkingSound); await _thinkingPlayer.setLoopMode(LoopMode.one); _soundsReady = true; } catch (_) { debugPrint('[VoiceOrchestrator] Sons introuvables — pipeline silencieux'); } } Future _playWakeSound() async { if (!_soundsReady) return; try { await _soundPlayer.seek(Duration.zero); _soundPlayer.play(); } catch (_) {} } /// Escalade plutôt que nappe permanente : rien pendant [_thinkingDelay], puis /// fondu d'entrée. Une réponse rapide ne déclenche aucun son. Future _startThinkingLoop() async { if (!_soundsReady) return; _thinkingWanted = true; _thinkingTimer?.cancel(); _thinkingTimer = Timer(_thinkingDelay, () async { if (!_thinkingWanted) return; try { await _thinkingPlayer.seek(Duration.zero); await _thinkingPlayer.setVolume(0); _thinkingPlayer.play(); for (var step = 1; step <= 6 && _thinkingWanted; step++) { await Future.delayed(const Duration(milliseconds: 100)); await _thinkingPlayer.setVolume(_thinkingVolume * step / 6); } } catch (_) {} }); } Future _stopThinkingLoop() async { _thinkingWanted = false; _thinkingTimer?.cancel(); _thinkingTimer = null; await _thinkingPlayer.stop(); } // ── Conversation vocale ──────────────────────────────────────────────────── Future _handleConversation() async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); isListeningForCommand.value = true; final command = await sttEngine.transcribeOnce(languageCode: langCode); isListeningForCommand.value = false; debugPrint('[VoiceOrchestrator] Command: "$command"'); if (command.isEmpty) return; lastTranscription.value = command; await _dispatch(command); } Future _dispatch(String command, {bool continueConversation = true}) async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); if (_isStopCommand(command)) { debugPrint('[VoiceOrchestrator] Annulé: "$command"'); await _stopThinkingLoop(); return; } if (_isQrScanCommand(command)) { await _handleQrScan(); return; } if (_isPhotoCommand(command)) { await _handlePhotoCapture(); return; } if (_isRepeatCommand(command)) { await ttsEngine.replay(); } else { bool expectsReply = true; try { await _startThinkingLoop(); final result = await llmClient.chat( command, configurationId: visitAppContext.configuration?.id, languageCode: lang, ); expectsReply = result.expectsReply; await _stopThinkingLoop(); if (result.reply.isNotEmpty) { lastTtsText.value = result.reply; await ttsEngine.speak(result.reply, languageCode: langCode); } } catch (e) { await _stopThinkingLoop(); debugPrint('[VoiceOrchestrator] LLM error: $e'); return; } if (!expectsReply) return; } if (continueConversation) await _listenForFollowUp(); } Future _listenForFollowUp() async { final lang = visitAppContext.language ?? 'FR'; final langCode = _toLangCode(lang); isListeningForCommand.value = true; final followUp = await sttEngine.transcribeOnce( languageCode: langCode, timeout: const Duration(seconds: 5), ); isListeningForCommand.value = false; if (followUp.isNotEmpty) lastTranscription.value = followUp; if (followUp.isEmpty || _isStopCommand(followUp)) return; await _dispatch(followUp, continueConversation: true); } // ── QR scan ─────────────────────────────────────────────────────────────── Future _handleQrScan() async { final lang = _toLangCode(visitAppContext.language ?? 'FR'); final completer = Completer(); final prevCallback = MetaGlassesService.instance.onPhotoCaptured; MetaGlassesService.instance.onPhotoCaptured = (path) { if (!completer.isCompleted) completer.complete(path.isEmpty ? null : path); prevCallback?.call(path); }; Timer(const Duration(seconds: 25), () { if (!completer.isCompleted) completer.complete(null); }); unawaited(MetaGlassesService.instance.requestPhotoCapture()); final photoPath = await completer.future; MetaGlassesService.instance.onPhotoCaptured = prevCallback; if (photoPath == null) { lastTtsText.value = TranslationHelper.getFromLocale('voice.cameraUnavailable', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); return; } if (lastQrScanPhoto.value != null && lastQrScanPhoto.value != photoPath) { try { File(lastQrScanPhoto.value!).deleteSync(); } catch (_) {} } lastQrScanPhoto.value = photoPath; final qr = await _tryDecodeQr(photoPath); if (qr != null) { await explainSection(qr.sectionId, configurationId: qr.configId); } else { lastTtsText.value = TranslationHelper.getFromLocale('voice.noQrFound', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); } } Future _handlePhotoCapture() async { final completer = Completer(); final prevCallback = MetaGlassesService.instance.onPhotoCaptured; MetaGlassesService.instance.onPhotoCaptured = (path) { if (!completer.isCompleted) completer.complete(path); prevCallback?.call(path); }; Timer(const Duration(seconds: 25), () { if (!completer.isCompleted) completer.complete(null); }); unawaited(MetaGlassesService.instance.requestPhotoCapture()); final photoPath = await completer.future; MetaGlassesService.instance.onPhotoCaptured = prevCallback; if (photoPath == null || photoPath.isEmpty) { final lang = _toLangCode(visitAppContext.language ?? 'FR'); lastTtsText.value = TranslationHelper.getFromLocale('voice.photoFailed', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); return; } final qr = await _tryDecodeQr(photoPath); if (qr != null) { await explainSection(qr.sectionId, configurationId: qr.configId); } else { visitPhotos.add(photoPath); visitPhotosNotifier.value = List.unmodifiable(visitPhotos); final lang = _toLangCode(visitAppContext.language ?? 'FR'); lastTtsText.value = TranslationHelper.getFromLocale('voice.photoCaptured', visitAppContext); await ttsEngine.speak(lastTtsText.value, languageCode: lang); } } Future<({String sectionId, String? configId})?> _tryDecodeQr(String imagePath) async { final controller = MobileScannerController(); ({String sectionId, String? configId})? result; try { final completer = Completer<({String sectionId, String? configId})?>(); final sub = controller.barcodes.listen((capture) { for (final barcode in capture.barcodes) { final raw = barcode.rawValue; if (raw != null) { final ids = _extractQrIds(raw); if (ids != null && !completer.isCompleted) completer.complete(ids); } } }); await controller.analyzeImage(imagePath); Timer(const Duration(seconds: 2), () { if (!completer.isCompleted) completer.complete(null); }); result = await completer.future; await sub.cancel(); } catch (e) { debugPrint('[VoiceOrchestrator] QR decode error: $e'); } finally { controller.dispose(); } return result; } ({String sectionId, String? configId})? _extractQrIds(String raw) { final m1 = _urlPattern1.firstMatch(raw); if (m1 != null) return (sectionId: m1.group(3)!, configId: m1.group(2)); final m2 = _urlPattern2.firstMatch(raw); if (m2 != null) return (sectionId: m2.group(3)!, configId: m2.group(2)); if (visitAppContext.sectionIds != null) { return visitAppContext.sectionIds!.contains(raw) ? (sectionId: raw, configId: visitAppContext.configuration?.id) : null; } return (sectionId: raw, configId: null); } Future explainSection(String sectionId, {String? configurationId}) async { final now = DateTime.now().millisecondsSinceEpoch; if ((now - (_lastQrTime[sectionId] ?? 0)) < _qrCooldownMs) return; _lastQrTime[sectionId] = now; final cfgId = configurationId ?? visitAppContext.configuration?.id; final lang = visitAppContext.language ?? 'FR'; try { final result = await llmClient.chat( 'Le visiteur vient de scanner le QR code de la section "$sectionId". ' 'Appelle GetSectionDetail avec cet ID, puis présente le contenu de façon engageante en 2-3 phrases.', configurationId: cfgId, languageCode: lang, ); if (result.reply.isNotEmpty) { lastTtsText.value = result.reply; await ttsEngine.speak(result.reply, languageCode: _toLangCode(lang)); } } catch (e) { debugPrint('[VoiceOrchestrator] explainSection error: $e'); } } // ── Helpers ──────────────────────────────────────────────────────────────── // Les quatre listes couvrent les quatre langues de l'assistant vocal (FR/NL/EN/DE). // Elles étaient en français seul — écrites pour tester, jamais reprises — ce qui rendait // les trois autres langues muettes sur les commandes. static const List _stopPhrases = [ 'non', 'rien', 'non merci', 'rien merci', 'laisse tomber', 'annule', 'annuler', 'arrête', 'arrete', 'arrêtez', 'au revoir', 'c\'est bon', 'ok merci', 'merci c\'est tout', 'no', 'nothing', 'no thanks', 'never mind', 'nevermind', 'cancel', 'goodbye', 'that\'s all', 'that\'s it', 'forget it', 'nee', 'niets', 'nee bedankt', 'laat maar', 'annuleer', 'tot ziens', 'het is goed', 'nein', 'nichts', 'nein danke', 'lass gut sein', 'abbrechen', 'tschüss', 'das war\'s', 'stop', 'stopp', ]; static const List _qrScanPhrases = [ 'scanne', 'scanner', 'scan', 'qr', 'qr code', 'code qr', 'qr-code', 'qrcode', 'regarde ce code', 'lis le code', ]; static const List _photoPhrases = [ 'photo', 'une photo', 'prends une photo', 'capture', 'picture', 'take a photo', 'take a picture', 'foto', 'neem een foto', 'maak een foto', 'mach ein foto', 'nimm ein foto', ]; static const List _repeatPhrases = [ // ⚠️ `encore` nu est volontairement absent : il matchait « raconte encore une // histoire », qui rejouait la réponse précédente au lieu d'en demander une nouvelle. 'répète', 'repete', 'répéter', 'répétez', 'encore une fois', 'redis', 'repeat', 'again', 'say that again', 'herhaal', 'opnieuw', 'nog eens', 'wiederhole', 'nochmal', 'noch einmal', ]; /// Match sur mot entier, pas sur sous-chaîne : `contains('prends')` reconnaissait /// « je ne com**prends** pas » comme une demande de photo. static bool _matchesAny(String text, List phrases) { final t = text.toLowerCase().trim(); return phrases.any( (p) => RegExp('(^|\\W)${RegExp.escape(p)}(\$|\\W)').hasMatch(t), ); } bool _isStopCommand(String text) => _matchesAny(text, _stopPhrases); /// ⚠️ Ne matche plus `code` nu : « what's the code of this painting » déclenchait /// un scan QR au lieu d'une réponse. bool _isQrScanCommand(String text) => _matchesAny(text, _qrScanPhrases); bool _isPhotoCommand(String text) => _matchesAny(text, _photoPhrases); bool _isRepeatCommand(String text) => _matchesAny(text, _repeatPhrases); /// Déléguée à `toVoiceLangCode` — la fonction était copiée ici, dans `AssistantChatSheet` /// et dans `GeoBeaconTriggerService`, et la décision « FR/NL/EN/DE seulement » n'était /// écrite que dans cette copie-ci. Voir `Helpers/voiceLanguage.dart`. String _toLangCode(String lang) => toVoiceLangCode(lang); }