001project_wildgrowth/backend/scripts/test-chat-vs-completions.ts

/**
 * 对比测试：chat/completions vs completions 接口耗时
 * 使用书籍解析 Prompt + 同一段样本文本，各调用 3 次，记录耗时并汇总。
 */

import OpenAI from 'openai';
import * as dotenv from 'dotenv';
import { DEFAULT_BOOK_CONTENT_PROMPT } from '../src/services/bookPromptConfigService';

dotenv.config();

// 与 modelProvider 一致：无 env 时用同一 fallback（仅便于在服务器跑对比测试）
const DOUBAO_API_KEY = process.env.DOUBAO_API_KEY || 'a3e13a85-437f-448c-aaa9-14292cd5e0ab';
const DOUBAO_BASE_URL = 'https://ark.cn-beijing.volces.com/api/v3';
const DOUBAO_MODEL = 'doubao-seed-1-6-lite-251015';

// 与书籍解析一致的 system：把 {{sourceText}} 替换为「见下文」
const SYSTEM_PROMPT = DEFAULT_BOOK_CONTENT_PROMPT.replace(/\{\{sourceText\}\}/g, '见下文');

// 固定样本文本（单层结构，便于模型快速输出）
const SAMPLE_SOURCE_TEXT = `第一章 什么是有效沟通

有效沟通是人与人之间准确传递信息、情感与意图的过程。它包含清晰表达、积极倾听和及时反馈。

第二章 倾听的重要性

倾听不仅是听到对方说了什么，更要理解其背后的需求与情绪。主动倾听能让对方感到被尊重，减少误解。

第三章 反馈的技巧

反馈要具体、及时、对事不对人。用「我观察到…」「我的感受是…」等表述，能降低对方的防御心理。`;

const RUNS = 3;
const MAX_TOKENS = 2000;

async function run() {
  if (!DOUBAO_API_KEY) {
    console.error('请设置 DOUBAO_API_KEY（.env 或环境变量）');
    process.exit(1);
  }
  // 若使用 fallback key，仅打印提示，不暴露完整 key
  const keySource = process.env.DOUBAO_API_KEY ? 'env' : 'fallback';
  console.log('API Key 来源:', keySource);

  const client = new OpenAI({
    apiKey: DOUBAO_API_KEY,
    baseURL: DOUBAO_BASE_URL,
  });

  console.log('═══════════════════════════════════════════════════════════');
  console.log('  书籍解析 Prompt 对比：chat/completions vs completions');
  console.log('  同一文本，各调用 ' + RUNS + ' 次');
  console.log('═══════════════════════════════════════════════════════════');
  console.log('Base URL:', DOUBAO_BASE_URL);
  console.log('Model:', DOUBAO_MODEL);
  console.log('样本文本长度:', SAMPLE_SOURCE_TEXT.length, '字符');
  console.log('max_tokens:', MAX_TOKENS);
  console.log('');

  const chatDurations: number[] = [];
  const compDurations: number[] = [];

  // ---------- 1) Chat Completions，调用 RUNS 次 ----------
  console.log('--- /chat/completions 调用 ' + RUNS + ' 次 ---');
  for (let i = 0; i < RUNS; i++) {
    try {
      const t0 = Date.now();
      await client.chat.completions.create({
        model: DOUBAO_MODEL,
        messages: [
          { role: 'system', content: SYSTEM_PROMPT },
          { role: 'user', content: SAMPLE_SOURCE_TEXT },
        ],
        max_tokens: MAX_TOKENS,
        temperature: 0.7,
      });
      const ms = Date.now() - t0;
      chatDurations.push(ms);
      console.log('  第 ' + (i + 1) + ' 次: ' + ms + ' ms');
    } catch (e: any) {
      console.log('  第 ' + (i + 1) + ' 次: 失败 -', e?.message || e);
    }
  }
  console.log('');

  // ---------- 2) Completions（合并为单条 prompt），调用 RUNS 次 ----------
  const singlePrompt =
    SYSTEM_PROMPT + '\n\n--- 用户消息（原始材料）---\n\n' + SAMPLE_SOURCE_TEXT;
  console.log('--- /completions 调用 ' + RUNS + ' 次 ---');
  for (let i = 0; i < RUNS; i++) {
    try {
      const t0 = Date.now();
      await client.completions.create({
        model: DOUBAO_MODEL,
        prompt: singlePrompt,
        max_tokens: MAX_TOKENS,
        temperature: 0.7,
      });
      const ms = Date.now() - t0;
      compDurations.push(ms);
      console.log('  第 ' + (i + 1) + ' 次: ' + ms + ' ms');
    } catch (e: any) {
      console.log('  第 ' + (i + 1) + ' 次: 失败 -', e?.message || e);
    }
  }
  console.log('');

  // ---------- 汇总 ----------
  console.log('═══════════════════════════════════════════════════════════');
  console.log('  汇总');
  console.log('═══════════════════════════════════════════════════════════');
  if (chatDurations.length > 0) {
    const sum = chatDurations.reduce((a, b) => a + b, 0);
    const avg = Math.round(sum / chatDurations.length);
    console.log('chat/completions: 成功 ' + chatDurations.length + ' 次');
    console.log('  各次(ms): ' + chatDurations.join(', '));
    console.log('  平均(ms): ' + avg);
  } else {
    console.log('chat/completions: 无成功调用');
  }
  console.log('');
  if (compDurations.length > 0) {
    const sum = compDurations.reduce((a, b) => a + b, 0);
    const compAvg = sum / compDurations.length;
    console.log('completions: 成功 ' + compDurations.length + ' 次');
    console.log('  各次(ms): ' + compDurations.join(', '));
    console.log('  平均(ms): ' + Math.round(compAvg));
    if (chatDurations.length > 0) {
      const chatAvg =
        chatDurations.reduce((a, b) => a + b, 0) / chatDurations.length;
      const diff = Math.round(compAvg - chatAvg);
      const pct = chatAvg > 0 ? ((diff / chatAvg) * 100).toFixed(1) : '0';
      console.log(
        '与 chat 平均差异: ' + (diff >= 0 ? '+' : '') + diff + ' ms (' + pct + '%)'
      );
    }
  } else {
    console.log('completions: 无成功调用（豆包可能未开放此端点）');
  }
  console.log('═══════════════════════════════════════════════════════════');
}

run().catch((e) => {
  console.error(e);
  process.exit(1);
});