|
|
@@ -22,8 +22,13 @@ from datetime import datetime, timezone, timedelta
|
|
|
from charset_normalizer import from_bytes
|
|
|
from discord.ext import tasks
|
|
|
from discord.ext import voice_recv
|
|
|
-from voice_stt import STTSink
|
|
|
-from voice_tts import speak
|
|
|
+import audioop
|
|
|
+import threading
|
|
|
+import time
|
|
|
+import wave
|
|
|
+import emoji
|
|
|
+from collections import Counter
|
|
|
+from davey import MediaType
|
|
|
|
|
|
# Charger les variables d'environnement depuis le fichier .env
|
|
|
load_dotenv()
|
|
|
@@ -54,7 +59,7 @@ if WEB_SEARCH_RESULT_COUNT < 1:
|
|
|
# Initialiser le client OpenAI asynchrone ici
|
|
|
openai_client = AsyncOpenAI(api_key=OPENAI_API_KEY, base_url=URL_OPENAI_API)
|
|
|
|
|
|
-BOT_VERSION = "3.4.1"
|
|
|
+BOT_VERSION = "3.4.2"
|
|
|
|
|
|
# Vérifier la version de Discord
|
|
|
if not sys.version_info[:2] >= (3, 15):
|
|
|
@@ -94,6 +99,396 @@ httpx_logger.setLevel(logging.DEBUG)
|
|
|
|
|
|
urllib3.disable_warnings()
|
|
|
|
|
|
+stt_logger = logging.getLogger("voice_stt")
|
|
|
+tts_logger = logging.getLogger("voice_tts")
|
|
|
+
|
|
|
+# Réception et transcription du vocal Discord
|
|
|
+class STTSink(voice_recv.AudioSink):
|
|
|
+ def __init__(self, on_phrase, stt_url, on_transcription=None, on_batch_status=None):
|
|
|
+ super().__init__()
|
|
|
+ self.phrase_silence = float(os.getenv('VOICE_PHRASE_SILENCE', '0.5'))
|
|
|
+ self.reply_silence = float(os.getenv('VOICE_REPLY_SILENCE', '1.0'))
|
|
|
+ if not (0.25 <= self.phrase_silence <= self.reply_silence <= 5):
|
|
|
+ raise ValueError('Silences vocaux attendus : 0.25 <= phrase <= réponse <= 5 secondes')
|
|
|
+ self.on_phrase = on_phrase
|
|
|
+ self.on_transcription = on_transcription
|
|
|
+ self.on_batch_status = on_batch_status
|
|
|
+ self.pending_receipts = []
|
|
|
+ self.pending = []
|
|
|
+ self.pending_event = asyncio.Event()
|
|
|
+ self.stt_url = stt_url
|
|
|
+ self.loop = asyncio.get_running_loop()
|
|
|
+ self.lock = threading.Lock()
|
|
|
+ self.buffers = {}
|
|
|
+ self.decoders = {}
|
|
|
+ self.closed = False
|
|
|
+ self.responding = False
|
|
|
+ self.last_stt_seconds = None
|
|
|
+ self.last_reply_seconds = None
|
|
|
+ self.last_voice = 0.0
|
|
|
+ self.last_error = 0
|
|
|
+ self.stats_lock = threading.Lock()
|
|
|
+ self.stats = Counter(audio=0, rtcp=0, pcm=0, audible=0, phrases=0, stt=0)
|
|
|
+ self.last_diagnostic = time.monotonic()
|
|
|
+ # File audio bornée si Whisper ne suit plus le rythme du vocal.
|
|
|
+ self.queue = asyncio.Queue(maxsize=32)
|
|
|
+ self.flush_task = self.loop.create_task(self.flush_phrases())
|
|
|
+ self.worker_task = self.loop.create_task(self.transcribe_phrases())
|
|
|
+ self.reply_task = self.loop.create_task(self.respond_batches())
|
|
|
+ self.reply_task.add_done_callback(self.task_finished)
|
|
|
+ self.flush_task.add_done_callback(self.task_finished)
|
|
|
+ self.worker_task.add_done_callback(self.task_finished)
|
|
|
+
|
|
|
+ def task_finished(self, task):
|
|
|
+ if not task.cancelled() and task.exception() is not None:
|
|
|
+ stt_logger.error("Tâche vocale arrêtée : %s erreur=%s",
|
|
|
+ task.get_coro().__name__, type(task.exception()).__name__)
|
|
|
+
|
|
|
+ def wants_opus(self):
|
|
|
+ # L'extension ne déchiffre pas encore DAVE : le faire avant Opus.
|
|
|
+ return True
|
|
|
+
|
|
|
+ def count(self, name):
|
|
|
+ with self.stats_lock:
|
|
|
+ self.stats[name] += 1
|
|
|
+
|
|
|
+ def log_diagnostic(self, reason='10s'):
|
|
|
+ with self.stats_lock:
|
|
|
+ stats = dict(self.stats)
|
|
|
+ voice = getattr(self, 'voice_client', None)
|
|
|
+ connection = getattr(voice, '_connection', None)
|
|
|
+ session = getattr(connection, 'dave_session', None)
|
|
|
+ stt_logger.info(
|
|
|
+ "Diagnostic vocal (%s, compteurs cumulés) : %s file=%d dave_version=%s dave_ready=%s",
|
|
|
+ reason, stats, self.queue.qsize(),
|
|
|
+ getattr(connection, 'dave_protocol_version', None), getattr(session, 'ready', False),
|
|
|
+ )
|
|
|
+
|
|
|
+ @voice_recv.AudioSink.listener()
|
|
|
+ def on_rtcp_packet(self, packet, guild):
|
|
|
+ self.count('rtcp')
|
|
|
+
|
|
|
+ @voice_recv.AudioSink.listener()
|
|
|
+ def on_voice_member_speaking_start(self, member):
|
|
|
+ self.count('speaking_start')
|
|
|
+
|
|
|
+ def receiver_stopped(self, error):
|
|
|
+ if error:
|
|
|
+ stt_logger.error("Thread de réception vocale arrêté : %s", type(error).__name__)
|
|
|
+ else:
|
|
|
+ stt_logger.info("Thread de réception vocale arrêté normalement")
|
|
|
+
|
|
|
+ def write(self, user, data):
|
|
|
+ if self.closed:
|
|
|
+ return
|
|
|
+ self.count('audio')
|
|
|
+ for ignored, reason in ((user is None, 'unknown_user'),
|
|
|
+ (getattr(user, 'bot', False), 'bot'),
|
|
|
+ (not data.opus, 'empty_opus'),
|
|
|
+ (data.packet.is_silence(), 'silence_packet')):
|
|
|
+ if ignored:
|
|
|
+ self.count(reason)
|
|
|
+ return
|
|
|
+ stage = 'DAVE'
|
|
|
+ try:
|
|
|
+ connection = self.voice_client._connection
|
|
|
+ opus = bytes(data.opus)
|
|
|
+ if connection.dave_protocol_version:
|
|
|
+ if connection.dave_session is None or not connection.dave_session.ready:
|
|
|
+ self.count('dave_not_ready')
|
|
|
+ return
|
|
|
+ opus = connection.dave_session.decrypt(user.id, MediaType.audio, opus)
|
|
|
+ stage = 'Opus'
|
|
|
+ with self.lock:
|
|
|
+ if self.closed:
|
|
|
+ return
|
|
|
+ decoder = self.decoders.get(user.id)
|
|
|
+ if decoder is None:
|
|
|
+ decoder = self.decoders[user.id] = discord.opus.Decoder()
|
|
|
+ pcm = decoder.decode(opus, fec=False)
|
|
|
+ self.count('pcm')
|
|
|
+ self.add_pcm(user, pcm, time.monotonic())
|
|
|
+ except Exception as exc:
|
|
|
+ self.count(stage + '_errors')
|
|
|
+ # Ne jamais journaliser les paquets audio ou les clés de session.
|
|
|
+ if time.monotonic() - self.last_error > 30:
|
|
|
+ stt_logger.warning("Réception vocale impossible à l'étape %s : %s", stage, type(exc).__name__)
|
|
|
+ self.last_error = time.monotonic()
|
|
|
+
|
|
|
+ def add_pcm(self, user, pcm, now):
|
|
|
+ """Appelé sous le verrou ; PCM Discord = 48 kHz, stéréo, 16 bits."""
|
|
|
+ audible = audioop.rms(pcm, 2) >= 200
|
|
|
+ self.count('audible' if audible else 'below_threshold')
|
|
|
+ if user.id not in self.buffers:
|
|
|
+ if not audible:
|
|
|
+ return
|
|
|
+ self.buffers[user.id] = [user, bytearray(), now]
|
|
|
+ buffer = self.buffers[user.id]
|
|
|
+ # Plafond de 20 secondes même si la boucle asyncio est occupée.
|
|
|
+ available = 48000 * 2 * 2 * 20 - len(buffer[1])
|
|
|
+ buffer[1].extend(pcm[:available])
|
|
|
+ if audible:
|
|
|
+ buffer[2] = now
|
|
|
+ self.last_voice = now
|
|
|
+
|
|
|
+ async def flush_phrases(self):
|
|
|
+ while True:
|
|
|
+ await asyncio.sleep(0.1)
|
|
|
+ now = time.monotonic()
|
|
|
+ if now - self.last_diagnostic >= 10:
|
|
|
+ self.log_diagnostic()
|
|
|
+ self.last_diagnostic = now
|
|
|
+ ready = []
|
|
|
+ with self.lock:
|
|
|
+ for user_id, (user, pcm, last_voice) in list(self.buffers.items()):
|
|
|
+ if now - last_voice >= self.phrase_silence or len(pcm) >= 48000 * 4 * 20:
|
|
|
+ del self.buffers[user_id]
|
|
|
+ if len(pcm) >= 48000 * 4 * 0.25:
|
|
|
+ ready.append((user, bytes(pcm)))
|
|
|
+ else:
|
|
|
+ self.count('too_short')
|
|
|
+ for item in ready:
|
|
|
+ try:
|
|
|
+ self.queue.put_nowait(item)
|
|
|
+ self.count('phrases')
|
|
|
+ stt_logger.info("Phrase vocale sélectionnée : durée=%.2fs transfert=%d",
|
|
|
+ len(item[1]) / (48000 * 4), self.queue.qsize())
|
|
|
+ except asyncio.QueueFull:
|
|
|
+ self.count('queue_full')
|
|
|
+ stt_logger.warning("File vocale pleine : une phrase a été ignorée")
|
|
|
+
|
|
|
+ async def transcribe_phrases(self):
|
|
|
+ # La transcription continue pendant les réponses du LLM.
|
|
|
+ async with httpx.AsyncClient(timeout=60) as http:
|
|
|
+ while not self.closed:
|
|
|
+ user, pcm = await self.queue.get()
|
|
|
+ stage = 'STT'
|
|
|
+ started = time.monotonic()
|
|
|
+ try:
|
|
|
+ audio = BytesIO()
|
|
|
+ with wave.open(audio, 'wb') as wav:
|
|
|
+ wav.setnchannels(2)
|
|
|
+ wav.setsampwidth(2)
|
|
|
+ wav.setframerate(48000)
|
|
|
+ wav.writeframes(pcm)
|
|
|
+ self.count('stt')
|
|
|
+ stt_logger.info("Envoi STT : durée audio=%.2fs", len(pcm) / (48000 * 4))
|
|
|
+ response = await http.post(
|
|
|
+ self.stt_url,
|
|
|
+ files={'file': ('phrase.wav', audio.getvalue(), 'audio/wav')},
|
|
|
+ )
|
|
|
+ response.raise_for_status()
|
|
|
+ text = response.json()['text'].strip()
|
|
|
+ self.last_stt_seconds = time.monotonic() - started
|
|
|
+ stt_logger.info("Retour STT : HTTP=%d durée=%.2fs caractères=%d",
|
|
|
+ response.status_code, time.monotonic() - started, len(text))
|
|
|
+ if not text:
|
|
|
+ self.count('empty_transcription')
|
|
|
+ if text and not self.closed:
|
|
|
+ receipt = None
|
|
|
+ try:
|
|
|
+ stage = 'affichage'
|
|
|
+ if self.on_transcription:
|
|
|
+ receipt = await self.on_transcription(user, text)
|
|
|
+ finally:
|
|
|
+ if not self.closed:
|
|
|
+ self.pending.append((user, text))
|
|
|
+ self.pending_receipts.append(receipt)
|
|
|
+ self.pending_event.set()
|
|
|
+ except asyncio.CancelledError:
|
|
|
+ raise
|
|
|
+ except Exception as exc:
|
|
|
+ self.count(stage + '_errors')
|
|
|
+ status = exc.response.status_code if isinstance(exc, httpx.HTTPStatusError) else None
|
|
|
+ stt_logger.warning("Traitement vocal impossible : étape=%s erreur=%s HTTP=%s durée=%.2fs",
|
|
|
+ stage, type(exc).__name__, status, time.monotonic() - started)
|
|
|
+ finally:
|
|
|
+ self.queue.task_done()
|
|
|
+
|
|
|
+ async def respond_batches(self):
|
|
|
+ while not self.closed:
|
|
|
+ await self.pending_event.wait()
|
|
|
+ # L'affichage STT continue ; seule la réponse attend le silence global.
|
|
|
+ while not self.closed:
|
|
|
+ await self.queue.join()
|
|
|
+ with self.lock:
|
|
|
+ quiet = time.monotonic() - self.last_voice >= self.reply_silence
|
|
|
+ buffered = bool(self.buffers)
|
|
|
+ if quiet and not buffered and self.queue.empty():
|
|
|
+ break
|
|
|
+ await asyncio.sleep(0.1)
|
|
|
+ if self.closed:
|
|
|
+ return
|
|
|
+ self.pending_event.clear()
|
|
|
+ batch, self.pending = self.pending, []
|
|
|
+ receipts, self.pending_receipts = self.pending_receipts, []
|
|
|
+ if not batch:
|
|
|
+ continue
|
|
|
+ started = time.monotonic()
|
|
|
+ self.responding = True
|
|
|
+ try:
|
|
|
+ await self.report_status(receipts, 'processing')
|
|
|
+ await self.on_phrase(batch)
|
|
|
+ await self.report_status(receipts, 'done')
|
|
|
+ except asyncio.CancelledError:
|
|
|
+ raise
|
|
|
+ except Exception as exc:
|
|
|
+ await self.report_status(receipts, 'error')
|
|
|
+ stt_logger.warning("Réponse au lot vocal impossible : %s", type(exc).__name__)
|
|
|
+ finally:
|
|
|
+ self.last_reply_seconds = time.monotonic() - started
|
|
|
+ self.responding = False
|
|
|
+
|
|
|
+ async def report_status(self, receipts, state):
|
|
|
+ if self.on_batch_status:
|
|
|
+ try:
|
|
|
+ await self.on_batch_status(receipts, state)
|
|
|
+ except Exception as exc:
|
|
|
+ stt_logger.warning('Mise à jour du statut impossible : %s', type(exc).__name__)
|
|
|
+
|
|
|
+ @voice_recv.AudioSink.listener()
|
|
|
+ def on_voice_member_disconnect(self, member, ssrc):
|
|
|
+ if member:
|
|
|
+ with self.lock:
|
|
|
+ self.buffers.pop(member.id, None)
|
|
|
+ self.decoders.pop(member.id, None)
|
|
|
+
|
|
|
+ def cleanup(self):
|
|
|
+ # La bibliothèque appelle cleanup depuis son thread de réception.
|
|
|
+ with self.lock:
|
|
|
+ if self.closed:
|
|
|
+ return
|
|
|
+ self.closed = True
|
|
|
+ self.buffers.clear()
|
|
|
+ self.decoders.clear()
|
|
|
+ self.pending.clear()
|
|
|
+ self.pending_receipts.clear()
|
|
|
+ self.log_diagnostic('arrêt')
|
|
|
+ if not self.loop.is_closed():
|
|
|
+ self.loop.call_soon_threadsafe(self.flush_task.cancel)
|
|
|
+ self.loop.call_soon_threadsafe(self.worker_task.cancel)
|
|
|
+ self.loop.call_soon_threadsafe(self.reply_task.cancel)
|
|
|
+
|
|
|
+ async def close(self):
|
|
|
+ self.cleanup()
|
|
|
+ self.flush_task.cancel()
|
|
|
+ self.worker_task.cancel()
|
|
|
+ self.reply_task.cancel()
|
|
|
+ await asyncio.gather(self.flush_task, self.worker_task, self.reply_task, return_exceptions=True)
|
|
|
+ while not self.queue.empty():
|
|
|
+ self.queue.get_nowait()
|
|
|
+ self.queue.task_done()
|
|
|
+
|
|
|
+# Synthèse vocale et lecture Discord
|
|
|
+def speech_chunks(text, limit=240, first_limit=None):
|
|
|
+ if limit < 1 or (first_limit is not None and first_limit < 1):
|
|
|
+ raise ValueError('La taille des morceaux doit être positive')
|
|
|
+ first = True
|
|
|
+ text = re.sub(r'```.*?```', ' Bloc de code omis. ', text, flags=re.S)
|
|
|
+ text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
|
|
|
+ text = re.sub(r'https?://\S+|<a?:\w+:\d+>|<[@#][!&]?\d+>', ' ', text)
|
|
|
+ # Le prompt peut émettre des alias personnalisés (:hap:, :desu:, etc.).
|
|
|
+ # Exiger une lettre évite de confondre ces alias avec une heure 12:30:45.
|
|
|
+ text = re.sub(r':[A-Za-z_][A-Za-z0-9_+\-]*:', ' ', text)
|
|
|
+ text = emoji.replace_emoji(text, replace=' ')
|
|
|
+ text = re.sub(r'[*_`~#>|]', '', text)
|
|
|
+ text = re.sub(r'[^\S\n]+', ' ', text)
|
|
|
+ for sentence in re.split(r'(?<=[.!?…])\s+|\n+', text.strip()):
|
|
|
+ while sentence:
|
|
|
+ budget = min(limit, first_limit) if first and first_limit else limit
|
|
|
+ end = len(sentence) if len(sentence) <= budget else sentence.rfind(' ', 0, budget + 1)
|
|
|
+ # Favoriser une pause naturelle, sans fabriquer un minuscule morceau.
|
|
|
+ if len(sentence) > budget:
|
|
|
+ pauses = list(re.finditer(r'[,;:]\s', sentence[:budget + 1]))
|
|
|
+ if pauses and pauses[-1].start() >= budget // 2:
|
|
|
+ end = pauses[-1].start() + 1
|
|
|
+ if end <= 0:
|
|
|
+ end = budget
|
|
|
+ chunk, sentence = sentence[:end].strip(), sentence[end:].strip()
|
|
|
+ if chunk:
|
|
|
+ first = False
|
|
|
+ yield chunk
|
|
|
+
|
|
|
+
|
|
|
+class WavSource(discord.AudioSource):
|
|
|
+ def __init__(self, payload):
|
|
|
+ with wave.open(BytesIO(payload), 'rb') as wav:
|
|
|
+ if wav.getsampwidth() != 2 or wav.getnchannels() not in (1, 2):
|
|
|
+ raise ValueError('Le TTS doit retourner du WAV PCM 16 bits mono/stéréo')
|
|
|
+ pcm = wav.readframes(wav.getnframes())
|
|
|
+ if wav.getnchannels() == 1:
|
|
|
+ pcm = audioop.tostereo(pcm, 2, 1, 1)
|
|
|
+ pcm, _ = audioop.ratecv(pcm, 2, 2, wav.getframerate(), 48000, None)
|
|
|
+ self.audio = BytesIO(pcm)
|
|
|
+
|
|
|
+ def read(self):
|
|
|
+ frame = self.audio.read(3840) # Discord : 20 ms, 48 kHz, stéréo, PCM16.
|
|
|
+ return frame.ljust(3840, b'\0') if frame else b''
|
|
|
+
|
|
|
+ def cleanup(self):
|
|
|
+ self.audio.close()
|
|
|
+
|
|
|
+
|
|
|
+async def play_wav(voice, payload):
|
|
|
+ loop = asyncio.get_running_loop()
|
|
|
+ done = loop.create_future()
|
|
|
+ source = WavSource(payload)
|
|
|
+
|
|
|
+ def complete(error):
|
|
|
+ if not done.done():
|
|
|
+ if error:
|
|
|
+ done.set_exception(error)
|
|
|
+ else:
|
|
|
+ done.set_result(None)
|
|
|
+
|
|
|
+ try:
|
|
|
+ voice.play(source, after=lambda error: loop.call_soon_threadsafe(complete, error))
|
|
|
+ await done
|
|
|
+ finally:
|
|
|
+ if not done.done() or done.cancelled():
|
|
|
+ voice.stop()
|
|
|
+ source.cleanup()
|
|
|
+
|
|
|
+
|
|
|
+async def speak(voice, text, url, voice_name='ff_siwis'):
|
|
|
+ first_chars = int(os.getenv('TTS_FIRST_CHUNK_CHARS', '48'))
|
|
|
+ chunk_chars = int(os.getenv('TTS_CHUNK_CHARS', '80'))
|
|
|
+ if not (16 <= first_chars <= chunk_chars <= 240):
|
|
|
+ raise ValueError('Tailles TTS attendues : 16 <= premier <= suivants <= 240')
|
|
|
+ chunks = iter(speech_chunks(text, limit=chunk_chars, first_limit=first_chars))
|
|
|
+ started = time.monotonic()
|
|
|
+ first_playback = True
|
|
|
+ async with httpx.AsyncClient(timeout=60) as http:
|
|
|
+ async def synthesize(chunk):
|
|
|
+ async with http.stream('POST', url, json={'input': chunk, 'voice': voice_name}) as response:
|
|
|
+ response.raise_for_status()
|
|
|
+ payload = bytearray()
|
|
|
+ async for data in response.aiter_bytes():
|
|
|
+ payload.extend(data)
|
|
|
+ if len(payload) > 8 * 1024 * 1024:
|
|
|
+ raise ValueError('Réponse audio trop volumineuse')
|
|
|
+ return bytes(payload)
|
|
|
+
|
|
|
+ first = next(chunks, None)
|
|
|
+ if first is None:
|
|
|
+ return # Une réponse uniquement composée d'emojis reste silencieuse.
|
|
|
+ pending = asyncio.create_task(synthesize(first))
|
|
|
+ try:
|
|
|
+ while pending is not None:
|
|
|
+ payload = await pending
|
|
|
+ following = next(chunks, None)
|
|
|
+ # Une seule phrase préchargée pendant la lecture de la précédente.
|
|
|
+ pending = asyncio.create_task(synthesize(following)) if following else None
|
|
|
+ if first_playback:
|
|
|
+ tts_logger.info('Premier audio prêt : délai TTS=%.3fs', time.monotonic() - started)
|
|
|
+ first_playback = False
|
|
|
+ await play_wav(voice, payload)
|
|
|
+ finally:
|
|
|
+ if pending is not None:
|
|
|
+ pending.cancel()
|
|
|
+ await asyncio.gather(pending, return_exceptions=True)
|
|
|
+
|
|
|
+
|
|
|
# Initialiser les intents
|
|
|
intents = discord.Intents.default()
|
|
|
intents.message_content = True # Activer l'intent pour les contenus de message
|
Andy Esnard