0aff904ede
La fonction était recopiée dans VoiceOrchestrator, AssistantChatSheet et GeoBeaconTriggerService. La limite « FR/NL/EN/DE seulement, le reste retombe sur le français » est une décision assumée, mais elle n'était documentée que dans la première : les deux autres ressemblaient à un oubli qu'on aurait envie de « corriger » en ajoutant des langues, ce qui aurait produit un support partiel silencieux — les listes de commandes vocales, elles, ne couvrent que ces quatre langues. Une seule copie dans Helpers/voiceLanguage.dart, une seule décision. flutter analyze lib sans erreur, flutter build apk --debug --flavor dev vert. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
468 lines
18 KiB
Dart
468 lines
18 KiB
Dart
import 'dart:async';
|
|
import 'dart:io';
|
|
import 'package:flutter/foundation.dart';
|
|
import 'package:mymuseum_visitapp/Helpers/translationHelper.dart';
|
|
import 'package:just_audio/just_audio.dart';
|
|
import 'package:mobile_scanner/mobile_scanner.dart';
|
|
import 'package:mymuseum_visitapp/Helpers/voiceLanguage.dart';
|
|
import 'package:mymuseum_visitapp/Models/visitContext.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/llm_client.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/stt_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/tts_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/Glasses/engines/wake_word_engine.dart';
|
|
import 'package:mymuseum_visitapp/Services/meta_glasses_service.dart';
|
|
|
|
/// Instance active de l'orchestrateur, accessible globalement.
|
|
/// Gérée par VoiceController — ne pas modifier directement.
|
|
VoiceOrchestrator? activeVoiceOrchestrator;
|
|
|
|
/// Orchestre le pipeline complet mains-libres :
|
|
/// WakeWord → STT → dispatch → LLM ou QR scan → TTS
|
|
///
|
|
/// Toutes les dépendances sont injectées via les interfaces abstraites
|
|
/// pour pouvoir swapper chaque maillon indépendamment.
|
|
class VoiceOrchestrator {
|
|
final VisitAppContext visitAppContext;
|
|
final WakeWordEngine wakeWordEngine;
|
|
final SttEngine sttEngine;
|
|
final TtsEngine ttsEngine;
|
|
final LlmClient llmClient;
|
|
|
|
bool _running = false;
|
|
bool _inConversation = false;
|
|
|
|
final ValueNotifier<String> lastTranscription = ValueNotifier('');
|
|
final ValueNotifier<bool> isListeningForCommand = ValueNotifier(false);
|
|
final ValueNotifier<String> lastTtsText = ValueNotifier('');
|
|
final ValueNotifier<List<String>> visitPhotosNotifier = ValueNotifier([]);
|
|
final ValueNotifier<String?> lastQrScanPhoto = ValueNotifier(null);
|
|
|
|
final List<String> visitPhotos = [];
|
|
|
|
static const String _wakeSound = 'assets/sounds/wake_detected.mp3';
|
|
static const String _thinkingSound = 'assets/sounds/thinking.mp3';
|
|
final AudioPlayer _soundPlayer = AudioPlayer();
|
|
final AudioPlayer _thinkingPlayer = AudioPlayer();
|
|
bool _soundsReady = false;
|
|
|
|
/// Le son de réflexion n'apparaît qu'après ce délai : en dessous, le visiteur vient
|
|
/// de finir de parler et n'attend pas encore de réponse — le silence s'y lit comme
|
|
/// naturel, la nappe comme du bruit.
|
|
static const Duration _thinkingDelay = Duration(milliseconds: 700);
|
|
static const double _thinkingVolume = 0.45;
|
|
Timer? _thinkingTimer;
|
|
bool _thinkingWanted = false;
|
|
|
|
final Map<String, int> _lastQrTime = {};
|
|
static const int _qrCooldownMs = 10000;
|
|
|
|
static final RegExp _urlPattern1 =
|
|
RegExp(r'https://web\.mymuseum\.be/([^/]+)/([^/]+)/([^/\s]+)');
|
|
static final RegExp _urlPattern2 =
|
|
RegExp(r'https://web\.myinfomate\.be/([^/]+)/([^/]+)/([^/\s]+)');
|
|
|
|
VoiceOrchestrator({
|
|
required this.visitAppContext,
|
|
required this.wakeWordEngine,
|
|
required this.sttEngine,
|
|
required this.ttsEngine,
|
|
required this.llmClient,
|
|
});
|
|
|
|
Future<void> start() async {
|
|
if (_running) return;
|
|
_running = true;
|
|
await _preloadSounds();
|
|
await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
debugPrint('[VoiceOrchestrator] Started');
|
|
}
|
|
|
|
Future<void> stop() async {
|
|
await wakeWordEngine.stop();
|
|
await sttEngine.cancel();
|
|
await ttsEngine.stop();
|
|
await _stopThinkingLoop();
|
|
_running = false;
|
|
debugPrint('[VoiceOrchestrator] Stopped');
|
|
}
|
|
|
|
bool get isRunning => _running;
|
|
bool get isInConversation => _inConversation;
|
|
bool get isListening => _running && !_inConversation;
|
|
|
|
Future<void> restartWakeWord() async {
|
|
if (!_running || _inConversation) return;
|
|
await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
|
|
Future<void> triggerConversation() => _handleConversation();
|
|
Future<void> dispatchCommand(String command) => _dispatch(command);
|
|
|
|
Future<void> triggerQrScan(String imagePath) async {
|
|
final qr = await _tryDecodeQr(imagePath);
|
|
if (qr != null) await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
}
|
|
|
|
// ── Wake word ──────────────────────────────────────────────────────────────
|
|
|
|
void _onWakeWord() async {
|
|
if (_inConversation) return;
|
|
_inConversation = true;
|
|
await wakeWordEngine.stop();
|
|
await _stopThinkingLoop();
|
|
_playWakeSound();
|
|
await Future.delayed(const Duration(milliseconds: 200));
|
|
try {
|
|
await _handleConversation();
|
|
} finally {
|
|
_inConversation = false;
|
|
if (_running) await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
}
|
|
|
|
void _onWakeWordWithCommand(String inlineCommand) async {
|
|
if (_inConversation) return;
|
|
_inConversation = true;
|
|
await wakeWordEngine.stop();
|
|
await _stopThinkingLoop();
|
|
_playWakeSound();
|
|
await Future.delayed(const Duration(milliseconds: 200));
|
|
try {
|
|
if (inlineCommand.isNotEmpty) {
|
|
debugPrint('[VoiceOrchestrator] Inline command: "$inlineCommand"');
|
|
await _dispatch(inlineCommand);
|
|
} else {
|
|
await _handleConversation();
|
|
}
|
|
} finally {
|
|
_inConversation = false;
|
|
if (_running) await wakeWordEngine.start(
|
|
onDetected: _onWakeWord,
|
|
onDetectedWithCommand: _onWakeWordWithCommand,
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Charge les deux sons une fois pour toutes. Sans ça, `setAsset` décode à chaque
|
|
/// wake word — sur le chemin le plus sensible à la latence de tout le pipeline.
|
|
Future<void> _preloadSounds() async {
|
|
if (_soundsReady) return;
|
|
try {
|
|
await _soundPlayer.setAsset(_wakeSound);
|
|
await _thinkingPlayer.setAsset(_thinkingSound);
|
|
await _thinkingPlayer.setLoopMode(LoopMode.one);
|
|
_soundsReady = true;
|
|
} catch (_) {
|
|
debugPrint('[VoiceOrchestrator] Sons introuvables — pipeline silencieux');
|
|
}
|
|
}
|
|
|
|
Future<void> _playWakeSound() async {
|
|
if (!_soundsReady) return;
|
|
try {
|
|
await _soundPlayer.seek(Duration.zero);
|
|
_soundPlayer.play();
|
|
} catch (_) {}
|
|
}
|
|
|
|
/// Escalade plutôt que nappe permanente : rien pendant [_thinkingDelay], puis
|
|
/// fondu d'entrée. Une réponse rapide ne déclenche aucun son.
|
|
Future<void> _startThinkingLoop() async {
|
|
if (!_soundsReady) return;
|
|
_thinkingWanted = true;
|
|
_thinkingTimer?.cancel();
|
|
_thinkingTimer = Timer(_thinkingDelay, () async {
|
|
if (!_thinkingWanted) return;
|
|
try {
|
|
await _thinkingPlayer.seek(Duration.zero);
|
|
await _thinkingPlayer.setVolume(0);
|
|
_thinkingPlayer.play();
|
|
for (var step = 1; step <= 6 && _thinkingWanted; step++) {
|
|
await Future.delayed(const Duration(milliseconds: 100));
|
|
await _thinkingPlayer.setVolume(_thinkingVolume * step / 6);
|
|
}
|
|
} catch (_) {}
|
|
});
|
|
}
|
|
|
|
Future<void> _stopThinkingLoop() async {
|
|
_thinkingWanted = false;
|
|
_thinkingTimer?.cancel();
|
|
_thinkingTimer = null;
|
|
await _thinkingPlayer.stop();
|
|
}
|
|
|
|
// ── Conversation vocale ────────────────────────────────────────────────────
|
|
|
|
Future<void> _handleConversation() async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
isListeningForCommand.value = true;
|
|
final command = await sttEngine.transcribeOnce(languageCode: langCode);
|
|
isListeningForCommand.value = false;
|
|
debugPrint('[VoiceOrchestrator] Command: "$command"');
|
|
if (command.isEmpty) return;
|
|
lastTranscription.value = command;
|
|
await _dispatch(command);
|
|
}
|
|
|
|
Future<void> _dispatch(String command, {bool continueConversation = true}) async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
|
|
if (_isStopCommand(command)) {
|
|
debugPrint('[VoiceOrchestrator] Annulé: "$command"');
|
|
await _stopThinkingLoop();
|
|
return;
|
|
}
|
|
|
|
if (_isQrScanCommand(command)) { await _handleQrScan(); return; }
|
|
if (_isPhotoCommand(command)) { await _handlePhotoCapture(); return; }
|
|
|
|
if (_isRepeatCommand(command)) {
|
|
await ttsEngine.replay();
|
|
} else {
|
|
bool expectsReply = true;
|
|
try {
|
|
await _startThinkingLoop();
|
|
final result = await llmClient.chat(
|
|
command,
|
|
configurationId: visitAppContext.configuration?.id,
|
|
languageCode: lang,
|
|
);
|
|
expectsReply = result.expectsReply;
|
|
await _stopThinkingLoop();
|
|
if (result.reply.isNotEmpty) {
|
|
lastTtsText.value = result.reply;
|
|
await ttsEngine.speak(result.reply, languageCode: langCode);
|
|
}
|
|
} catch (e) {
|
|
await _stopThinkingLoop();
|
|
debugPrint('[VoiceOrchestrator] LLM error: $e');
|
|
return;
|
|
}
|
|
if (!expectsReply) return;
|
|
}
|
|
|
|
if (continueConversation) await _listenForFollowUp();
|
|
}
|
|
|
|
Future<void> _listenForFollowUp() async {
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
final langCode = _toLangCode(lang);
|
|
isListeningForCommand.value = true;
|
|
final followUp = await sttEngine.transcribeOnce(
|
|
languageCode: langCode,
|
|
timeout: const Duration(seconds: 5),
|
|
);
|
|
isListeningForCommand.value = false;
|
|
if (followUp.isNotEmpty) lastTranscription.value = followUp;
|
|
if (followUp.isEmpty || _isStopCommand(followUp)) return;
|
|
await _dispatch(followUp, continueConversation: true);
|
|
}
|
|
|
|
// ── QR scan ───────────────────────────────────────────────────────────────
|
|
|
|
Future<void> _handleQrScan() async {
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
final completer = Completer<String?>();
|
|
final prevCallback = MetaGlassesService.instance.onPhotoCaptured;
|
|
MetaGlassesService.instance.onPhotoCaptured = (path) {
|
|
if (!completer.isCompleted) completer.complete(path.isEmpty ? null : path);
|
|
prevCallback?.call(path);
|
|
};
|
|
Timer(const Duration(seconds: 25), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
unawaited(MetaGlassesService.instance.requestPhotoCapture());
|
|
final photoPath = await completer.future;
|
|
MetaGlassesService.instance.onPhotoCaptured = prevCallback;
|
|
|
|
if (photoPath == null) {
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.cameraUnavailable', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
return;
|
|
}
|
|
|
|
if (lastQrScanPhoto.value != null && lastQrScanPhoto.value != photoPath) {
|
|
try { File(lastQrScanPhoto.value!).deleteSync(); } catch (_) {}
|
|
}
|
|
lastQrScanPhoto.value = photoPath;
|
|
|
|
final qr = await _tryDecodeQr(photoPath);
|
|
if (qr != null) {
|
|
await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
} else {
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.noQrFound', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
}
|
|
}
|
|
|
|
Future<void> _handlePhotoCapture() async {
|
|
final completer = Completer<String?>();
|
|
final prevCallback = MetaGlassesService.instance.onPhotoCaptured;
|
|
MetaGlassesService.instance.onPhotoCaptured = (path) {
|
|
if (!completer.isCompleted) completer.complete(path);
|
|
prevCallback?.call(path);
|
|
};
|
|
Timer(const Duration(seconds: 25), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
unawaited(MetaGlassesService.instance.requestPhotoCapture());
|
|
final photoPath = await completer.future;
|
|
MetaGlassesService.instance.onPhotoCaptured = prevCallback;
|
|
|
|
if (photoPath == null || photoPath.isEmpty) {
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.photoFailed', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
return;
|
|
}
|
|
|
|
final qr = await _tryDecodeQr(photoPath);
|
|
if (qr != null) {
|
|
await explainSection(qr.sectionId, configurationId: qr.configId);
|
|
} else {
|
|
visitPhotos.add(photoPath);
|
|
visitPhotosNotifier.value = List.unmodifiable(visitPhotos);
|
|
final lang = _toLangCode(visitAppContext.language ?? 'FR');
|
|
lastTtsText.value = TranslationHelper.getFromLocale('voice.photoCaptured', visitAppContext);
|
|
await ttsEngine.speak(lastTtsText.value, languageCode: lang);
|
|
}
|
|
}
|
|
|
|
Future<({String sectionId, String? configId})?> _tryDecodeQr(String imagePath) async {
|
|
final controller = MobileScannerController();
|
|
({String sectionId, String? configId})? result;
|
|
try {
|
|
final completer = Completer<({String sectionId, String? configId})?>();
|
|
final sub = controller.barcodes.listen((capture) {
|
|
for (final barcode in capture.barcodes) {
|
|
final raw = barcode.rawValue;
|
|
if (raw != null) {
|
|
final ids = _extractQrIds(raw);
|
|
if (ids != null && !completer.isCompleted) completer.complete(ids);
|
|
}
|
|
}
|
|
});
|
|
await controller.analyzeImage(imagePath);
|
|
Timer(const Duration(seconds: 2), () {
|
|
if (!completer.isCompleted) completer.complete(null);
|
|
});
|
|
result = await completer.future;
|
|
await sub.cancel();
|
|
} catch (e) {
|
|
debugPrint('[VoiceOrchestrator] QR decode error: $e');
|
|
} finally {
|
|
controller.dispose();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
({String sectionId, String? configId})? _extractQrIds(String raw) {
|
|
final m1 = _urlPattern1.firstMatch(raw);
|
|
if (m1 != null) return (sectionId: m1.group(3)!, configId: m1.group(2));
|
|
final m2 = _urlPattern2.firstMatch(raw);
|
|
if (m2 != null) return (sectionId: m2.group(3)!, configId: m2.group(2));
|
|
if (visitAppContext.sectionIds != null) {
|
|
return visitAppContext.sectionIds!.contains(raw)
|
|
? (sectionId: raw, configId: visitAppContext.configuration?.id)
|
|
: null;
|
|
}
|
|
return (sectionId: raw, configId: null);
|
|
}
|
|
|
|
Future<void> explainSection(String sectionId, {String? configurationId}) async {
|
|
final now = DateTime.now().millisecondsSinceEpoch;
|
|
if ((now - (_lastQrTime[sectionId] ?? 0)) < _qrCooldownMs) return;
|
|
_lastQrTime[sectionId] = now;
|
|
final cfgId = configurationId ?? visitAppContext.configuration?.id;
|
|
final lang = visitAppContext.language ?? 'FR';
|
|
try {
|
|
final result = await llmClient.chat(
|
|
'Le visiteur vient de scanner le QR code de la section "$sectionId". '
|
|
'Appelle GetSectionDetail avec cet ID, puis présente le contenu de façon engageante en 2-3 phrases.',
|
|
configurationId: cfgId,
|
|
languageCode: lang,
|
|
);
|
|
if (result.reply.isNotEmpty) {
|
|
lastTtsText.value = result.reply;
|
|
await ttsEngine.speak(result.reply, languageCode: _toLangCode(lang));
|
|
}
|
|
} catch (e) {
|
|
debugPrint('[VoiceOrchestrator] explainSection error: $e');
|
|
}
|
|
}
|
|
|
|
// ── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
// Les quatre listes couvrent les quatre langues de l'assistant vocal (FR/NL/EN/DE).
|
|
// Elles étaient en français seul — écrites pour tester, jamais reprises — ce qui rendait
|
|
// les trois autres langues muettes sur les commandes.
|
|
|
|
static const List<String> _stopPhrases = [
|
|
'non', 'rien', 'non merci', 'rien merci', 'laisse tomber', 'annule', 'annuler',
|
|
'arrête', 'arrete', 'arrêtez', 'au revoir', 'c\'est bon', 'ok merci',
|
|
'merci c\'est tout',
|
|
'no', 'nothing', 'no thanks', 'never mind', 'nevermind', 'cancel', 'goodbye',
|
|
'that\'s all', 'that\'s it', 'forget it',
|
|
'nee', 'niets', 'nee bedankt', 'laat maar', 'annuleer', 'tot ziens', 'het is goed',
|
|
'nein', 'nichts', 'nein danke', 'lass gut sein', 'abbrechen', 'tschüss', 'das war\'s',
|
|
'stop', 'stopp',
|
|
];
|
|
|
|
static const List<String> _qrScanPhrases = [
|
|
'scanne', 'scanner', 'scan', 'qr', 'qr code', 'code qr', 'qr-code', 'qrcode',
|
|
'regarde ce code', 'lis le code',
|
|
];
|
|
|
|
static const List<String> _photoPhrases = [
|
|
'photo', 'une photo', 'prends une photo', 'capture',
|
|
'picture', 'take a photo', 'take a picture',
|
|
'foto', 'neem een foto', 'maak een foto',
|
|
'mach ein foto', 'nimm ein foto',
|
|
];
|
|
|
|
static const List<String> _repeatPhrases = [
|
|
// ⚠️ `encore` nu est volontairement absent : il matchait « raconte encore une
|
|
// histoire », qui rejouait la réponse précédente au lieu d'en demander une nouvelle.
|
|
'répète', 'repete', 'répéter', 'répétez', 'encore une fois', 'redis',
|
|
'repeat', 'again', 'say that again',
|
|
'herhaal', 'opnieuw', 'nog eens',
|
|
'wiederhole', 'nochmal', 'noch einmal',
|
|
];
|
|
|
|
/// Match sur mot entier, pas sur sous-chaîne : `contains('prends')` reconnaissait
|
|
/// « je ne com**prends** pas » comme une demande de photo.
|
|
static bool _matchesAny(String text, List<String> phrases) {
|
|
final t = text.toLowerCase().trim();
|
|
return phrases.any(
|
|
(p) => RegExp('(^|\\W)${RegExp.escape(p)}(\$|\\W)').hasMatch(t),
|
|
);
|
|
}
|
|
|
|
bool _isStopCommand(String text) => _matchesAny(text, _stopPhrases);
|
|
|
|
/// ⚠️ Ne matche plus `code` nu : « what's the code of this painting » déclenchait
|
|
/// un scan QR au lieu d'une réponse.
|
|
bool _isQrScanCommand(String text) => _matchesAny(text, _qrScanPhrases);
|
|
|
|
bool _isPhotoCommand(String text) => _matchesAny(text, _photoPhrases);
|
|
|
|
bool _isRepeatCommand(String text) => _matchesAny(text, _repeatPhrases);
|
|
|
|
/// Déléguée à `toVoiceLangCode` — la fonction était copiée ici, dans `AssistantChatSheet`
|
|
/// et dans `GeoBeaconTriggerService`, et la décision « FR/NL/EN/DE seulement » n'était
|
|
/// écrite que dans cette copie-ci. Voir `Helpers/voiceLanguage.dart`.
|
|
String _toLangCode(String lang) => toVoiceLangCode(lang);
|
|
}
|