feat(API): 扩展聊天接口支持多模态消息

请求参数扩展:
- 新增 displayMessage 字段用于数据库存储原始用户输入
- 新增 images 字段支持用户上传的图片(Base64格式)
- 新增 uploadedImages 和 uploadedDocuments 用于持久化

多模态消息处理:
- Claude API 支持 image 类型内容块
- Codex API 支持 input_image 格式
- 用户消息保存时存储上传的图片和文档

系统提示词增强:
- 添加文档深度分析规范
- 定义七步文档分析框架
- 包含批判性评价和实践价值分析指导
This commit is contained in:
gaoziman 2025-12-20 12:14:18 +08:00
parent acf17557c2
commit 00b8589e03

View File

@ -8,9 +8,25 @@ import { executeTool } from '@/services/tools';
interface ChatRequest { interface ChatRequest {
conversationId: string; conversationId: string;
message: string; message: string;
displayMessage?: string; // 原始用户输入(用于数据库存储和显示)
model?: string; model?: string;
tools?: string[]; tools?: string[];
enableThinking?: boolean; enableThinking?: boolean;
// 用户上传的图片(发送给 AI
images?: {
type: 'image';
media_type: string;
data: string;
}[];
// 用户上传的图片 URL用于保存到数据库显示
uploadedImages?: string[];
// 用户上传的文档(用于保存到数据库)
uploadedDocuments?: {
name: string;
size: number;
type: string;
content: string;
}[];
} }
// 消息内容块类型Claude // 消息内容块类型Claude
@ -123,13 +139,78 @@ const DEFAULT_SYSTEM_PROMPT = `你是一个专业、友好的 AI 助手。请遵
2. **** 2. ****
3. **使** seaborn-whitegrid plt.style.use() 3. **使** seaborn-whitegrid plt.style.use()
4. ****Noto Sans SC使 4. ****Noto Sans SC使
5. ****使使(subplot)`; 5. ****使使(subplot)
##
/****
### 1. 📋
- /
-
-
### 2. 🏗
-
- /
-
### 3. 💡
- 5-10
-
-
### 4. 🔍
-
-
-
-
### 5. 🎯
-
- 使
-
### 6.
-
-
- /
### 7. 📝
-
- /
-
****
- 2-3
- 使
-
- 1500广
- 使便
- ****`;
// POST /api/chat - 发送消息并获取 AI 回复 // POST /api/chat - 发送消息并获取 AI 回复
export async function POST(request: Request) { export async function POST(request: Request) {
try { try {
const body: ChatRequest = await request.json(); const body: ChatRequest = await request.json();
const { conversationId, message, model, tools, enableThinking } = body; const { conversationId, message, displayMessage, model, tools, enableThinking, images, uploadedImages, uploadedDocuments } = body;
// 调试日志:确认接收到的图片数据
console.log('[API/chat] Received request with:', {
conversationId,
messageLength: message?.length,
displayMessageLength: displayMessage?.length,
model,
tools,
enableThinking,
imagesCount: images?.length || 0,
uploadedImagesCount: uploadedImages?.length || 0,
uploadedDocumentsCount: uploadedDocuments?.length || 0,
images: images ? images.map(img => ({
type: img.type,
media_type: img.media_type,
dataLength: img.data?.length || 0,
})) : undefined,
});
// 获取用户设置 // 获取用户设置
const settings = await db.query.userSettings.findFirst({ const settings = await db.query.userSettings.findFirst({
@ -161,14 +242,17 @@ export async function POST(request: Request) {
orderBy: (messages, { asc }) => [asc(messages.createdAt)], orderBy: (messages, { asc }) => [asc(messages.createdAt)],
}); });
// 保存用户消息 // 保存用户消息(包括上传的图片和文档)
// 使用 displayMessage原始用户输入作为显示内容如果没有则使用 message
const userMessageId = nanoid(); const userMessageId = nanoid();
await db.insert(messages).values({ await db.insert(messages).values({
messageId: userMessageId, messageId: userMessageId,
conversationId, conversationId,
role: 'user', role: 'user',
content: message, content: displayMessage || message, // 使用原始用户输入作为显示内容
status: 'completed', status: 'completed',
uploadedImages: uploadedImages && uploadedImages.length > 0 ? uploadedImages : null,
uploadedDocuments: uploadedDocuments && uploadedDocuments.length > 0 ? uploadedDocuments : null,
}); });
// 准备 AI 消息 ID // 准备 AI 消息 ID
@ -216,6 +300,7 @@ export async function POST(request: Request) {
tools: tools || (conversation.tools as string[]) || [], tools: tools || (conversation.tools as string[]) || [],
controller, controller,
encoder, encoder,
images, // 传递用户上传的图片
}); });
fullContent = result.fullContent; fullContent = result.fullContent;
@ -235,6 +320,7 @@ export async function POST(request: Request) {
enableThinking: enableThinking ?? conversation.enableThinking ?? false, enableThinking: enableThinking ?? conversation.enableThinking ?? false,
controller, controller,
encoder, encoder,
images, // 传递用户上传的图片
}); });
fullContent = result.fullContent; fullContent = result.fullContent;
@ -256,6 +342,7 @@ export async function POST(request: Request) {
}); });
// 更新对话信息 // 更新对话信息
const titleSource = displayMessage || message;
await db await db
.update(conversations) .update(conversations)
.set({ .set({
@ -264,7 +351,7 @@ export async function POST(request: Request) {
lastMessageAt: new Date(), lastMessageAt: new Date(),
updatedAt: new Date(), updatedAt: new Date(),
title: (conversation.messageCount || 0) === 0 title: (conversation.messageCount || 0) === 0
? message.slice(0, 50) + (message.length > 50 ? '...' : '') ? titleSource.slice(0, 50) + (titleSource.length > 50 ? '...' : '')
: conversation.title, : conversation.title,
}) })
.where(eq(conversations.conversationId, conversationId)); .where(eq(conversations.conversationId, conversationId));
@ -318,15 +405,26 @@ interface CodexChatParams {
tools: string[]; tools: string[];
controller: ReadableStreamDefaultController; controller: ReadableStreamDefaultController;
encoder: TextEncoder; encoder: TextEncoder;
// 用户上传的图片
images?: {
type: 'image';
media_type: string;
data: string;
}[];
} }
// Codex Response API 的输入项类型 // Codex Response API 的输入项类型
interface CodexInputItem { interface CodexInputItem {
type: 'message'; type: 'message';
role: 'user' | 'assistant' | 'system'; role: 'user' | 'assistant' | 'system';
content: string; content: string | CodexMultimodalContent[];
} }
// Codex 多模态内容类型
type CodexMultimodalContent =
| { type: 'input_text'; text: string }
| { type: 'input_image'; image_url: string };
// Codex Response API 的工具调用类型 // Codex Response API 的工具调用类型
interface CodexFunctionCall { interface CodexFunctionCall {
call_id: string; call_id: string;
@ -350,6 +448,7 @@ async function handleCodexChat(params: CodexChatParams): Promise<{
tools, tools,
controller, controller,
encoder, encoder,
images,
} = params; } = params;
// 构建 Codex Response API 格式的输入 // 构建 Codex Response API 格式的输入
@ -359,9 +458,53 @@ async function handleCodexChat(params: CodexChatParams): Promise<{
role: msg.role as 'user' | 'assistant', role: msg.role as 'user' | 'assistant',
content: msg.content, content: msg.content,
})), })),
{ type: 'message' as const, role: 'user' as const, content: message },
]; ];
// 添加当前用户消息(支持多模态内容)
if (images && images.length > 0) {
console.log('[handleCodexChat] Building multimodal message with', images.length, 'images');
// 如果有图片,构建多模态消息
const multimodalContent: CodexMultimodalContent[] = [];
// 先添加图片
for (const img of images) {
console.log('[handleCodexChat] Adding image:', {
type: img.type,
media_type: img.media_type,
dataLength: img.data?.length || 0,
});
// Codex/OpenAI 格式:使用 data URL
const dataUrl = `data:${img.media_type};base64,${img.data}`;
multimodalContent.push({
type: 'input_image',
image_url: dataUrl,
});
}
// 再添加文本
if (message) {
multimodalContent.push({
type: 'input_text',
text: message,
});
}
console.log('[handleCodexChat] Multimodal content blocks:', multimodalContent.length);
inputItems.push({
type: 'message' as const,
role: 'user' as const,
content: multimodalContent,
});
} else {
console.log('[handleCodexChat] No images, using simple text message');
// 没有图片,使用简单文本消息
inputItems.push({
type: 'message' as const,
role: 'user' as const,
content: message,
});
}
// 构建 Codex Response API 格式的工具定义 // 构建 Codex Response API 格式的工具定义
const codexTools = buildCodexToolDefinitions(tools); const codexTools = buildCodexToolDefinitions(tools);
@ -599,6 +742,12 @@ interface ClaudeChatParams {
enableThinking: boolean; enableThinking: boolean;
controller: ReadableStreamDefaultController; controller: ReadableStreamDefaultController;
encoder: TextEncoder; encoder: TextEncoder;
// 用户上传的图片
images?: {
type: 'image';
media_type: string;
data: string;
}[];
} }
async function handleClaudeChat(params: ClaudeChatParams): Promise<{ async function handleClaudeChat(params: ClaudeChatParams): Promise<{
@ -619,6 +768,7 @@ async function handleClaudeChat(params: ClaudeChatParams): Promise<{
enableThinking, enableThinking,
controller, controller,
encoder, encoder,
images,
} = params; } = params;
// 构建消息历史 // 构建消息历史
@ -627,11 +777,51 @@ async function handleClaudeChat(params: ClaudeChatParams): Promise<{
content: msg.content, content: msg.content,
})); }));
// 添加当前用户消息 // 添加当前用户消息(支持多模态内容)
if (images && images.length > 0) {
console.log('[handleClaudeChat] Building multimodal message with', images.length, 'images');
// 如果有图片,构建多模态消息
const multimodalContent: ContentBlock[] = [];
// 先添加图片
for (const img of images) {
console.log('[handleClaudeChat] Adding image:', {
type: img.type,
media_type: img.media_type,
dataLength: img.data?.length || 0,
});
multimodalContent.push({
type: 'image' as unknown as 'text',
// @ts-expect-error - Claude API 支持 image 类型但 TypeScript 类型定义不完整
source: {
type: 'base64',
media_type: img.media_type,
data: img.data,
},
});
}
// 再添加文本
if (message) {
multimodalContent.push({
type: 'text',
text: message,
});
}
console.log('[handleClaudeChat] Multimodal content blocks:', multimodalContent.length);
messageHistory.push({
role: 'user',
content: multimodalContent,
});
} else {
console.log('[handleClaudeChat] No images, using simple text message');
// 没有图片,使用简单文本消息
messageHistory.push({ messageHistory.push({
role: 'user', role: 'user',
content: message, content: message,
}); });
}
// 构建工具定义 // 构建工具定义
const toolDefinitions = buildClaudeToolDefinitions(tools); const toolDefinitions = buildClaudeToolDefinitions(tools);
@ -657,6 +847,20 @@ async function handleClaudeChat(params: ClaudeChatParams): Promise<{
messages: currentMessages, messages: currentMessages,
}; };
// 调试日志:查看发送给 Claude API 的消息内容
console.log('[handleClaudeChat] Sending to Claude API:', {
model,
messagesCount: currentMessages.length,
lastMessage: currentMessages.length > 0 ? {
role: currentMessages[currentMessages.length - 1].role,
contentType: typeof currentMessages[currentMessages.length - 1].content,
contentIsArray: Array.isArray(currentMessages[currentMessages.length - 1].content),
contentLength: Array.isArray(currentMessages[currentMessages.length - 1].content)
? (currentMessages[currentMessages.length - 1].content as unknown[]).length
: (currentMessages[currentMessages.length - 1].content as string).length,
} : null,
});
if (toolDefinitions.length > 0) { if (toolDefinitions.length > 0) {
requestBody.tools = toolDefinitions; requestBody.tools = toolDefinitions;
} }