まだフリーズすることがあるがひとまず動く

This commit is contained in:
ntki72 2024-09-09 22:51:40 +09:00
parent af131ee53d
commit e5b62cf0ce
2 changed files with 73 additions and 63 deletions

135
bot.js
View File

@ -3,10 +3,13 @@ const { Client, GatewayIntentBits } = require('discord.js');
const { joinVoiceChannel, getVoiceConnection, VoiceConnectionStatus, EndBehaviorType, createAudioPlayer, createAudioResource, AudioPlayerStatus } = require('@discordjs/voice'); const { joinVoiceChannel, getVoiceConnection, VoiceConnectionStatus, EndBehaviorType, createAudioPlayer, createAudioResource, AudioPlayerStatus } = require('@discordjs/voice');
const fs = require('fs'); const fs = require('fs');
const { OpusEncoder } = require('@discordjs/opus'); const { OpusEncoder } = require('@discordjs/opus');
const { spawn } = require('child_process'); // execからspawnに変更 const { spawn } = require('child_process');
const OpenAI = require('openai'); const OpenAI = require('openai');
const axios = require('axios'); const axios = require('axios');
let isProcessing = false; // 処理中フラグ
let isPlaying = false; // 音声再生中フラグ
// OpenAI APIキーの確認 // OpenAI APIキーの確認
const apiKey = process.env.OPENAI_API_KEY || 'your-openai-api-key'; const apiKey = process.env.OPENAI_API_KEY || 'your-openai-api-key';
if (!apiKey || apiKey === 'your-openai-api-key') { if (!apiKey || apiKey === 'your-openai-api-key') {
@ -14,6 +17,9 @@ if (!apiKey || apiKey === 'your-openai-api-key') {
process.exit(1); // エラーコード1で終了 process.exit(1); // エラーコード1で終了
} }
// ChatGPTのモデル
const chatGptModel = process.env.CHAT_GPT_MODEL || 'gpt-4o';
const openai = new OpenAI({ apiKey }); const openai = new OpenAI({ apiKey });
sodium.ready.then(() => { sodium.ready.then(() => {
@ -34,6 +40,11 @@ client.once('ready', () => {
}); });
client.on('messageCreate', async message => { client.on('messageCreate', async message => {
if (isProcessing) {
message.reply('現在処理中です。少々お待ちください。');
return; // 処理中なら新しい処理を行わない
}
if (message.content === '!cs_join') { if (message.content === '!cs_join') {
if (message.member.voice.channel) { if (message.member.voice.channel) {
const connection = joinVoiceChannel({ const connection = joinVoiceChannel({
@ -57,13 +68,6 @@ client.on('messageCreate', async message => {
const connection = getVoiceConnection(message.guild.id); const connection = getVoiceConnection(message.guild.id);
if (connection) { if (connection) {
connection.destroy(); // ボイスチャンネルから切断 connection.destroy(); // ボイスチャンネルから切断
// 進行中のFFmpegなどのプロセスを強制終了
if (ffmpeg) {
ffmpeg.kill('SIGKILL');
console.log('FFmpeg process killed on leave');
}
message.reply('ボイスチャンネルを離れました。'); message.reply('ボイスチャンネルを離れました。');
} else { } else {
message.reply('ボイスチャンネルに接続していません。'); message.reply('ボイスチャンネルに接続していません。');
@ -71,9 +75,9 @@ client.on('messageCreate', async message => {
} }
}); });
function startRecording(connection, voiceChannel) { async function startRecording(connection, voiceChannel) {
const receiver = connection.receiver; const receiver = connection.receiver;
const activeStreams = new Map(); // ユーザーごとの音声ストリーム管理 const activeStreams = new Map();
receiver.speaking.on('start', (userId) => { receiver.speaking.on('start', (userId) => {
const user = voiceChannel.members.get(userId); const user = voiceChannel.members.get(userId);
@ -96,7 +100,7 @@ function startRecording(connection, voiceChannel) {
const writableStream = fs.createWriteStream(filePath); const writableStream = fs.createWriteStream(filePath);
activeStreams.set(user.id, { audioStream, writableStream, filePath }); activeStreams.set(user.id, { audioStream, writableStream, filePath });
const opusDecoder = new OpusEncoder(48000, 2); // Opusデコーダーを使用 const opusDecoder = new OpusEncoder(48000, 2);
audioStream.on('data', (chunk) => { audioStream.on('data', (chunk) => {
try { try {
const pcmData = opusDecoder.decode(chunk); const pcmData = opusDecoder.decode(chunk);
@ -129,28 +133,30 @@ function startRecording(connection, voiceChannel) {
return; return;
} }
if (stats.size > 0) { if (stats.size > 0) {
convertToWavAndTranscribe(filePath, connection); // バックグラウンドで変換と文字起こしを実行 convertToWavAndTranscribe(filePath, connection);
} else { } else {
console.log(`File ${filePath} is empty, skipping conversion.`); console.log(`File ${filePath} is empty, skipping conversion.`);
fs.unlinkSync(filePath); // 空のファイルを削除 fs.unlinkSync(filePath);
} }
}); });
}); });
}); });
} }
// PCMファイルをWAVファイルに変換し、文字起こしを実行 async function convertToWavAndTranscribe(pcmPath, connection) {
function convertToWavAndTranscribe(pcmPath, connection) { if (isProcessing) return; // 処理中ならスキップ
isProcessing = true; // 処理開始
const wavPath = pcmPath.replace('.pcm', '.wav'); const wavPath = pcmPath.replace('.pcm', '.wav');
console.log(`Converting ${pcmPath} to ${wavPath} and transcribing in the background`); console.log(`Converting ${pcmPath} to ${wavPath} and transcribing in the background`);
const ffmpeg = spawn('ffmpeg', ['-f', 's16le', '-ar', '48000', '-ac', '2', '-i', pcmPath, wavPath]); const ffmpeg = spawn('ffmpeg', ['-f', 's16le', '-ar', '48000', '-ac', '2', '-i', pcmPath, wavPath]);
// プロセスがフリーズした場合、タイムアウトして強制終了する
const ffmpegTimeout = setTimeout(() => { const ffmpegTimeout = setTimeout(() => {
ffmpeg.kill('SIGKILL'); // 強制終了 ffmpeg.kill('SIGKILL');
console.error('FFmpeg process timed out and was killed'); console.error('FFmpeg process timed out and was killed');
}, 10000); // 10秒でタイムアウト isProcessing = false; // タイムアウトで処理終了
}, 10000); // 10秒でタイムアウト
ffmpeg.stdout.on('data', (data) => { ffmpeg.stdout.on('data', (data) => {
console.log(`FFmpeg stdout: ${data}`); console.log(`FFmpeg stdout: ${data}`);
@ -161,90 +167,80 @@ function convertToWavAndTranscribe(pcmPath, connection) {
}); });
ffmpeg.on('close', (code) => { ffmpeg.on('close', (code) => {
clearTimeout(ffmpegTimeout); // 正常に終了したらタイムアウトをクリア clearTimeout(ffmpegTimeout);
if (code === 0) { if (code === 0) {
console.log(`Successfully converted ${pcmPath} to ${wavPath}`); console.log(`Successfully converted ${pcmPath} to ${wavPath}`);
fs.unlinkSync(pcmPath); // PCMファイル削除 fs.unlinkSync(pcmPath); // PCMファイル削除
// Whisper APIを使って文字起こしを実行 // Whisper APIで文字起こしと次の処理へ
transcribeAudio(wavPath, connection); transcribeAudio(wavPath, connection);
} else { } else {
console.error(`FFmpeg exited with code ${code}, conversion failed.`); console.error(`FFmpeg exited with code ${code}, conversion failed.`);
isProcessing = false; // エラーで処理終了
} }
}); });
ffmpeg.on('error', (err) => { ffmpeg.on('error', (err) => {
console.error('FFmpeg process error:', err); console.error('FFmpeg process error:', err);
isProcessing = false; // エラーで処理終了
}); });
} }
// Whisper APIを使って音声ファイルを文字起こしする関数
async function transcribeAudio(filePath, connection) { async function transcribeAudio(filePath, connection) {
try { try {
console.log(`Transcribing file: ${filePath}`); console.log(`Transcribing file: ${filePath}`);
const response = await openai.audio.transcriptions.create({ const response = await openai.audio.transcriptions.create({
file: fs.createReadStream(filePath), file: fs.createReadStream(filePath),
model: 'whisper-1', model: 'whisper-1',
response_format: 'json', // レスポンス形式をJSONに指定 response_format: 'json',
language: 'ja' // 言語を日本語に固定 language: 'ja'
}); });
// 文字起こし結果を取得
if (response && response.text) { if (response && response.text) {
const transcription = response.text; const transcription = response.text;
console.log('Transcription:', transcription); console.log('Transcription:', transcription);
// ChatGPTに反応すべきか判断させる
const shouldReact = await shouldReactToTranscription(transcription); const shouldReact = await shouldReactToTranscription(transcription);
if (!shouldReact) { if (!shouldReact) {
console.log('ChatGPT decided not to react to this transcription.'); console.log('ChatGPT decided not to react to this transcription.');
// 文字起こし完了後、WAVファイルを削除
fs.unlink(filePath, (err) => { fs.unlink(filePath, (err) => {
if (err) { if (err) console.error(`Error deleting recording file: ${err}`);
console.error(`Error deleting recording file: ${err}`); else console.log(`Deleted recording file: ${filePath}`);
} else {
console.log(`Deleted recording file: ${filePath}`);
}
}); });
return; return;
} }
// ChatGPTを使用して返答を生成
const reply = await generateChatResponse(transcription); const reply = await generateChatResponse(transcription);
console.log('ChatGPT response:', reply); console.log('ChatGPT response:', reply);
// 合成音声を生成して再生 if (reply) {
const voiceFilePath = await generateVoiceFromText(reply); const voiceFilePath = await generateVoiceFromText(reply);
if (voiceFilePath) { if (voiceFilePath) {
playVoiceInChannel(connection, voiceFilePath); // 音声を再生 await playVoiceInChannel(connection, voiceFilePath); // 音声再生が完了するまで待つ
}
} }
// 文字起こし完了後、WAVファイルを削除
fs.unlink(filePath, (err) => { fs.unlink(filePath, (err) => {
if (err) { if (err) console.error(`Error deleting recording file: ${err}`);
console.error(`Error deleting recording file: ${err}`); else console.log(`Deleted recording file: ${filePath}`);
} else {
console.log(`Deleted recording file: ${filePath}`);
}
}); });
} else { } else {
console.error('Error: Transcription response is missing "text" field.'); console.error('Error: Transcription response is missing "text" field.');
} }
} catch (error) { } catch (error) {
console.error('Error during transcription:', error.response ? error.response.data : error.message); console.error('Error during transcription:', error.response ? error.response.data : error.message);
} finally {
isProcessing = false; // 最後に処理フラグをリセット
} }
} }
// ChatGPTに反応すべきかどうかを判断させる関数
async function shouldReactToTranscription(transcription) { async function shouldReactToTranscription(transcription) {
try { try {
const systemPrompt = 'あなたは「セカイ」というキャラクターです。次のユーザーの発言に対して反応すべきかどうかを判断してください。反応すべき場合は「はい」、反応しない場合は「いいえ」と答えてください。'; const systemPrompt = 'あなたは「セカイ」というキャラクターです。次のユーザーの発言に対して反応すべきかどうかを判断してください。反応すべき場合は「はい」、反応しない場合は「いいえ」と答えてください。';
const response = await openai.chat.completions.create({ const response = await openai.chat.completions.create({
model: 'gpt-4', model: chatGptModel,
messages: [ messages: [
{ role: 'system', content: systemPrompt }, { role: 'system', content: systemPrompt },
{ role: 'user', content: transcription } { role: 'user', content: transcription }
@ -264,19 +260,20 @@ async function shouldReactToTranscription(transcription) {
} }
} }
// ChatGPTに応答を生成させる関数
async function generateChatResponse(transcription) { async function generateChatResponse(transcription) {
try { try {
const systemPrompt = 'あなたはこれから「セカイ」という名前の幼い少女を演じてユーザーとの会話を楽しんでください。回答メッセージは最大でも200字になるようにしてください。'; const systemPrompt = 'あなたはこれから「セカイ」という名前の少女を演じてユーザーとの会話を楽しんでください。回答メッセージは最大でも80字になるようにしてください。';
const chatResponse = await openai.chat.completions.create({ const chatResponse = await openai.chat.completions.create({
model: 'gpt-4', model: chatGptModel,
messages: [ messages: [
{ role: 'system', content: systemPrompt }, { role: 'system', content: systemPrompt },
{ role: 'user', content: transcription } { role: 'user', content: transcription }
] ]
}); });
console.log('ChatGPT full response:', chatResponse);
if (chatResponse && chatResponse.choices && chatResponse.choices.length > 0) { if (chatResponse && chatResponse.choices && chatResponse.choices.length > 0) {
return chatResponse.choices[0].message.content; return chatResponse.choices[0].message.content;
} else { } else {
@ -286,10 +283,11 @@ async function generateChatResponse(transcription) {
} catch (error) { } catch (error) {
console.error('Error during ChatGPT response generation:', error.response ? error.response.data : error.message); console.error('Error during ChatGPT response generation:', error.response ? error.response.data : error.message);
return null; return null;
} finally {
isProcessing = false; // 処理終了後にフラグを必ずリセット
} }
} }
// ChatGPTの返答を合成音声に変換する関数
async function generateVoiceFromText(text, retryCount = 10) { async function generateVoiceFromText(text, retryCount = 10) {
try { try {
const response = await axios.post('http://192.168.100.252:5000/generate_voice', { const response = await axios.post('http://192.168.100.252:5000/generate_voice', {
@ -301,21 +299,20 @@ async function generateVoiceFromText(text, retryCount = 10) {
emotion_fun: "50", emotion_fun: "50",
pitch: "1" pitch: "1"
}, { }, {
responseType: 'arraybuffer' // 音声ファイルをバイナリデータとして取得 responseType: 'arraybuffer'
}); });
const outputFilePath = `./voices/output-${Date.now()}.wav`; const outputFilePath = `./voices/output-${Date.now()}.wav`;
await fs.promises.writeFile(outputFilePath, response.data); // 非同期でファイルを書き込み await fs.promises.writeFile(outputFilePath, response.data);
console.log(`Voice file saved to ${outputFilePath}`); console.log(`Voice file saved to ${outputFilePath}`);
return outputFilePath; // 音声ファイルのパスを返す return outputFilePath;
} catch (error) { } catch (error) {
console.error('Error generating voice:', error.message); console.error('Error generating voice:', error.message);
// リトライ処理
if (retryCount > 0) { if (retryCount > 0) {
console.log(`Retrying to generate the voice... (${retryCount} retries left)`); console.log(`Retrying to generate the voice... (${retryCount} retries left)`);
await new Promise(resolve => setTimeout(resolve, 1000)); // 1秒待機 await new Promise(resolve => setTimeout(resolve, 1000));
return generateVoiceFromText(text, retryCount - 1); // リトライ return generateVoiceFromText(text, retryCount - 1);
} else { } else {
console.error('Max retries reached. Could not generate the voice.'); console.error('Max retries reached. Could not generate the voice.');
return null; return null;
@ -323,19 +320,27 @@ async function generateVoiceFromText(text, retryCount = 10) {
} }
} }
// ボイスチャンネルで音声を再生する関数 async function playVoiceInChannel(connection, filePath, retryCount = 3) {
function playVoiceInChannel(connection, filePath, retryCount = 3) { if (isPlaying) {
console.log('Another audio is still playing, waiting...');
return;
}
isPlaying = true;
const player = createAudioPlayer(); const player = createAudioPlayer();
const resource = createAudioResource(filePath); const resource = createAudioResource(filePath);
player.play(resource); player.play(resource);
connection.subscribe(player); connection.subscribe(player);
player.on(AudioPlayerStatus.Playing, () => {
console.log('Audio is playing...');
});
player.on(AudioPlayerStatus.Idle, () => { player.on(AudioPlayerStatus.Idle, () => {
console.log('Finished playing voice.'); console.log('Finished playing voice.');
// 再生が終了した後にWAVファイルを削除
setTimeout(() => { setTimeout(() => {
// 再生が終了した後にWAVファイルを削除
fs.unlink(filePath, (err) => { fs.unlink(filePath, (err) => {
if (err) { if (err) {
console.error(`Error deleting file: ${err}`); console.error(`Error deleting file: ${err}`);
@ -343,13 +348,15 @@ function playVoiceInChannel(connection, filePath, retryCount = 3) {
console.log(`Deleted file: ${filePath}`); console.log(`Deleted file: ${filePath}`);
} }
}); });
}, 1000); // 1秒の待機後にファイル削除 }, 1000);
isPlaying = false; // 音声再生終了
isProcessing = false; // 処理終了
}); });
player.on('error', error => { player.on('error', error => {
console.error('Error playing audio:', error.message); console.error('Error playing audio:', error.message);
// リトライ処理 // 再生エラー時にリトライ処理
if (retryCount > 0) { if (retryCount > 0) {
console.log(`Retrying to play the voice... (${retryCount} retries left)`); console.log(`Retrying to play the voice... (${retryCount} retries left)`);
setTimeout(() => { setTimeout(() => {
@ -357,6 +364,8 @@ function playVoiceInChannel(connection, filePath, retryCount = 3) {
}, 1000); // 1秒待機してリトライ }, 1000); // 1秒待機してリトライ
} else { } else {
console.error('Max retries reached. Could not play the audio.'); console.error('Max retries reached. Could not play the audio.');
isPlaying = false; // 音声再生失敗
isProcessing = false; // 処理終了
} }
}); });
} }

View File

@ -6,6 +6,7 @@ services:
environment: environment:
- BOT_TOKEN=MTI4MjM0OTkwMjM3ODMwNzY2Nw.G4AEby.jtFyPkoNbWH3OHPLNzbo0-QZ1odddtu-r9bzdo - BOT_TOKEN=MTI4MjM0OTkwMjM3ODMwNzY2Nw.G4AEby.jtFyPkoNbWH3OHPLNzbo0-QZ1odddtu-r9bzdo
- OPENAI_API_KEY=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 - OPENAI_API_KEY=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
- CHAT_GPT_MODEL=gpt-4o
volumes: volumes:
- ./recordings:/usr/src/app/recordings - ./recordings:/usr/src/app/recordings
- ./voices:/usr/src/app/voices - ./voices:/usr/src/app/voices