feat(dingtalk): 小龙虾招募全语音/文字引导流程 + OAuth 一键授权卡片
## 功能说明
用户通过语音或文字说「帮我招募一只小龙虾」，iAgent 全程引导完成
OpenClaw 实例创建 + 钉钉 OAuth 一键授权绑定。
## 核心设计
- 语音场景 (claude_agent_sdk): Claude 通过 Bash/wget 调用内部 HTTP
端点触发 OAuth，绕开 ToolExecutor 限制，两引擎均兼容
- 文字场景 (claude_api): 使用 initiate_dingtalk_binding 自定义工具，
通过 uiEvent 机制传递 OAuth URL
## agent-service 变更
- agent-engine.port.ts: EngineStreamEvent 联合类型新增 oauth_prompt
- allowed-tools-resolver.service.ts: initiate_dingtalk_binding 加入
ALL_SDK_TOOLS / admin / operator 工具白名单
- tool-executor.ts: 新增 executeInitiateDingTalkBinding()，调用内部
oauth/init 端点获取 OAuth URL，返回 uiEvent
- claude-api-engine.ts: 在 tool_result 之后检查 result.uiEvent 并
yield 出去；buildToolDefinitions 注册 initiate_dingtalk_binding schema
- system-prompt-builder.ts:
- SystemPromptContext 新增 sessionId? 字段
- 语音 session (sessionId 存在) → Step 3 使用 wget 调用
POST /sessions/{sessionId}/dingtalk/oauth-trigger（两引擎通用）
- 文字 session (无 sessionId) → Step 3 调用 initiate_dingtalk_binding
工具（claude_api 专用）
- voice-session.controller.ts:
- 注入 AgentStreamGateway / DingTalkRouterService / AgentInstanceRepository
- startVoiceSession: 提前确定 sessionId，在 build() 前传入，使系统
提示能内嵌正确的端点 URL
- 新增 POST :sessionId/dingtalk/oauth-trigger — 无 JWT（内部端点，
由 Claude Bash 工具调用），sessionId 作为能力令牌；生成 OAuth URL
并通过 gateway.emitStreamEvent 直接推送 oauth_prompt 事件到 WS 流
## voice-agent 变更
- agent.py: 构造 AgentServiceLLM 时传入 room=ctx.room
- agent_llm.py:
- __init__ 增加 room 参数，存储为 self._room
- 新增 _publish_oauth_prompt(evt_data): null-safe，通过 LiveKit
publish_data(topic="oauth_prompt") 推送到 Flutter
- _do_inject_voice / _do_inject / _do_stream_voice / _do_stream:
处理 oauth_prompt 事件 → asyncio.create_task(_publish_oauth_prompt)
- 替换已弃用的 asyncio.ensure_future / get_event_loop().create_task
→ asyncio.create_task（Python 3.10+ 兼容）
## Flutter 变更
- agent_call_page.dart: DataReceivedEvent 监听 topic="oauth_prompt"，
解析 url/instanceName，弹出 _showOAuthBottomSheet（深色主题，🦞
图标，「立即授权」按钮 launchUrl externalApplication）
- stream_event.dart: 新增 OAuthPromptEvent(url, instanceId, instanceName)
- stream_event_model.dart: toEntity() 新增 'oauth_prompt' case
- chat_message.dart: MessageType 枚举新增 oauthPrompt
- chat_providers.dart: _handleStreamEvent 新增 OAuthPromptEvent case，
生成 type=oauthPrompt 的 ChatMessage（metadata 含 url/instanceName）
- chat_page.dart: 新增 oauthPrompt 时间线节点 + _OAuthPromptCard 组件
（「立即授权」按钮，launchUrl externalApplication）；import url_launcher
## 修复的关键 Bug
1. [严重] initiate_dingtalk_binding 只对 claude_api 有效，语音默认用
claude_agent_sdk → 新 wget 端点两引擎均可用
2. [严重] 文字聊天页面不处理 oauth_prompt 事件（静默丢弃）→ 补全
Flutter 4 处代码（entity/model/provider/page）
3. [中] _publish_oauth_prompt 缺 local_participant null 检查 → 已修复
4. [轻] asyncio.ensure_future / get_event_loop() 弃用警告 → 已修复
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3d626aebb5
commit
64499a5d86
|
|
@ -8,6 +8,7 @@ import 'package:it0_app/l10n/app_localizations.dart';
|
|||
import 'package:livekit_client/livekit_client.dart';
|
||||
import 'package:image_picker/image_picker.dart';
|
||||
import 'package:file_picker/file_picker.dart';
|
||||
import 'package:url_launcher/url_launcher.dart';
|
||||
import '../../../../core/config/api_endpoints.dart';
|
||||
import '../../../../core/config/app_config.dart';
|
||||
import '../../../../core/network/dio_client.dart';
|
||||
|
|
@ -190,6 +191,15 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
_scrollToBottom();
|
||||
}
|
||||
} catch (_) {}
|
||||
} else if (event.topic == 'oauth_prompt') {
|
||||
try {
|
||||
final payload = jsonDecode(utf8.decode(event.data));
|
||||
final url = payload['url'] as String? ?? '';
|
||||
final instanceName = payload['instanceName'] as String? ?? '小龙虾';
|
||||
if (url.isNotEmpty && mounted) {
|
||||
_showOAuthBottomSheet(url, instanceName);
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -297,6 +307,78 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
_endCall();
|
||||
}
|
||||
|
||||
/// Show DingTalk OAuth authorization card pushed by iAgent.
|
||||
void _showOAuthBottomSheet(String url, String instanceName) {
|
||||
showModalBottomSheet(
|
||||
context: context,
|
||||
backgroundColor: const Color(0xFF1A1D2E),
|
||||
shape: const RoundedRectangleBorder(
|
||||
borderRadius: BorderRadius.vertical(top: Radius.circular(20)),
|
||||
),
|
||||
builder: (ctx) => Padding(
|
||||
padding: const EdgeInsets.fromLTRB(24, 20, 24, 36),
|
||||
child: Column(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
Container(
|
||||
width: 40, height: 4,
|
||||
decoration: BoxDecoration(
|
||||
color: Colors.white24,
|
||||
borderRadius: BorderRadius.circular(2),
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 20),
|
||||
const Text('🦞', style: TextStyle(fontSize: 48)),
|
||||
const SizedBox(height: 12),
|
||||
Text(
|
||||
'授权「$instanceName」接入钉钉',
|
||||
style: const TextStyle(
|
||||
color: Colors.white,
|
||||
fontSize: 18,
|
||||
fontWeight: FontWeight.bold,
|
||||
),
|
||||
textAlign: TextAlign.center,
|
||||
),
|
||||
const SizedBox(height: 8),
|
||||
const Text(
|
||||
'点击下方按钮，在钉钉中一键完成授权绑定',
|
||||
style: TextStyle(color: Colors.white54, fontSize: 14),
|
||||
textAlign: TextAlign.center,
|
||||
),
|
||||
const SizedBox(height: 24),
|
||||
SizedBox(
|
||||
width: double.infinity,
|
||||
height: 52,
|
||||
child: ElevatedButton(
|
||||
style: ElevatedButton.styleFrom(
|
||||
backgroundColor: const Color(0xFF1677FF),
|
||||
shape: RoundedRectangleBorder(
|
||||
borderRadius: BorderRadius.circular(12),
|
||||
),
|
||||
),
|
||||
onPressed: () async {
|
||||
Navigator.of(ctx).pop();
|
||||
final uri = Uri.tryParse(url);
|
||||
if (uri != null) {
|
||||
await launchUrl(uri, mode: LaunchMode.externalApplication);
|
||||
}
|
||||
},
|
||||
child: const Text(
|
||||
'立即授权',
|
||||
style: TextStyle(
|
||||
color: Colors.white,
|
||||
fontSize: 16,
|
||||
fontWeight: FontWeight.w600,
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// Toggle microphone mute.
|
||||
void _toggleMute() {
|
||||
_isMuted = !_isMuted;
|
||||
|
|
|
|||
|
|
@ -88,6 +88,13 @@ class StreamEventModel {
|
|||
case 'cancelled':
|
||||
return CancelledEvent();
|
||||
|
||||
case 'oauth_prompt':
|
||||
return OAuthPromptEvent(
|
||||
data['url'] as String? ?? '',
|
||||
data['instanceId'] as String? ?? '',
|
||||
data['instanceName'] as String? ?? '小龙虾',
|
||||
);
|
||||
|
||||
default:
|
||||
// Fall back to text event for unknown types
|
||||
return TextEvent(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
enum MessageRole { user, assistant, system }
|
||||
|
||||
enum MessageType { text, toolUse, toolResult, approval, thinking, standingOrderDraft, interrupted }
|
||||
enum MessageType { text, toolUse, toolResult, approval, thinking, standingOrderDraft, interrupted, oauthPrompt }
|
||||
|
||||
enum ToolStatus { executing, completed, error, blocked, awaitingApproval }
|
||||
|
||||
|
|
|
|||
|
|
@ -67,3 +67,12 @@ class TaskInfoEvent extends StreamEvent {
|
|||
class CancelledEvent extends StreamEvent {
|
||||
CancelledEvent();
|
||||
}
|
||||
|
||||
/// Emitted when the agent wants to trigger DingTalk OAuth for an agent instance.
|
||||
/// The UI should show an authorization card so the user can tap "立即授权".
|
||||
class OAuthPromptEvent extends StreamEvent {
|
||||
final String url;
|
||||
final String instanceId;
|
||||
final String instanceName;
|
||||
OAuthPromptEvent(this.url, this.instanceId, this.instanceName);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import 'dart:io';
|
|||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
import 'package:it0_app/l10n/app_localizations.dart';
|
||||
import 'package:url_launcher/url_launcher.dart';
|
||||
import 'package:image_picker/image_picker.dart';
|
||||
import 'package:file_picker/file_picker.dart';
|
||||
import '../../../../core/theme/app_colors.dart';
|
||||
|
|
@ -483,6 +484,18 @@ class _ChatPageState extends ConsumerState<ChatPage> {
|
|||
icon: Icons.cancel_outlined,
|
||||
);
|
||||
|
||||
case MessageType.oauthPrompt:
|
||||
final url = message.metadata?['url'] as String? ?? '';
|
||||
final instanceName = message.metadata?['instanceName'] as String? ?? '小龙虾';
|
||||
return TimelineEventNode(
|
||||
status: NodeStatus.warning,
|
||||
label: '钉钉授权',
|
||||
isFirst: isFirst,
|
||||
isLast: isLast,
|
||||
icon: Icons.link,
|
||||
content: _OAuthPromptCard(url: url, instanceName: instanceName),
|
||||
);
|
||||
|
||||
case MessageType.text:
|
||||
default:
|
||||
return TimelineEventNode(
|
||||
|
|
@ -779,6 +792,50 @@ class _ChatPageState extends ConsumerState<ChatPage> {
|
|||
// Standing order content (embedded in timeline node)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OAuth prompt card — shown in timeline when agent triggers DingTalk binding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class _OAuthPromptCard extends StatelessWidget {
|
||||
final String url;
|
||||
final String instanceName;
|
||||
|
||||
const _OAuthPromptCard({required this.url, required this.instanceName});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
Text(
|
||||
'点击下方按钮，在钉钉中为「$instanceName」完成一键授权绑定。',
|
||||
style: const TextStyle(color: AppColors.textSecondary, fontSize: 13),
|
||||
),
|
||||
const SizedBox(height: 10),
|
||||
SizedBox(
|
||||
width: double.infinity,
|
||||
child: ElevatedButton.icon(
|
||||
icon: const Text('🦞', style: TextStyle(fontSize: 16)),
|
||||
label: const Text('立即授权'),
|
||||
style: ElevatedButton.styleFrom(
|
||||
backgroundColor: const Color(0xFF1677FF),
|
||||
foregroundColor: Colors.white,
|
||||
shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(10)),
|
||||
padding: const EdgeInsets.symmetric(vertical: 12),
|
||||
),
|
||||
onPressed: () async {
|
||||
final uri = Uri.tryParse(url);
|
||||
if (uri != null) {
|
||||
await launchUrl(uri, mode: LaunchMode.externalApplication);
|
||||
}
|
||||
},
|
||||
),
|
||||
),
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Collapsible code block for tool results – collapsed by default, tap to expand
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -348,6 +348,18 @@ class ChatNotifier extends StateNotifier<ChatState> {
|
|||
if (state.agentStatus != AgentStatus.idle) {
|
||||
state = state.copyWith(agentStatus: AgentStatus.idle, clearTaskId: true);
|
||||
}
|
||||
|
||||
case OAuthPromptEvent(:final url, :final instanceId, :final instanceName):
|
||||
// Add an authorization card to the timeline so the user can tap "立即授权"
|
||||
final msg = ChatMessage(
|
||||
id: DateTime.now().microsecondsSinceEpoch.toString(),
|
||||
role: MessageRole.assistant,
|
||||
content: '授权「$instanceName」接入钉钉',
|
||||
timestamp: DateTime.now(),
|
||||
type: MessageType.oauthPrompt,
|
||||
metadata: {'url': url, 'instanceId': instanceId, 'instanceName': instanceName},
|
||||
);
|
||||
state = state.copyWith(messages: [...state.messages, msg]);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,4 +47,5 @@ export type EngineStreamEvent =
|
|||
}
|
||||
| { type: 'error'; message: string; code: string }
|
||||
| { type: 'cancelled'; message: string; code: string }
|
||||
| { type: 'task_info'; taskId: string };
|
||||
| { type: 'task_info'; taskId: string }
|
||||
| { type: 'oauth_prompt'; url: string; instanceId: string; instanceName: string };
|
||||
|
|
|
|||
|
|
@ -16,11 +16,12 @@ import { TenantAgentConfig } from '../entities/tenant-agent-config.entity';
|
|||
const ALL_SDK_TOOLS = [
|
||||
'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep',
|
||||
'WebFetch', 'WebSearch', 'NotebookEdit', 'Task',
|
||||
'initiate_dingtalk_binding',
|
||||
];
|
||||
|
||||
const ROLE_TOOL_MAP: Record<string, string[]> = {
|
||||
admin: [...ALL_SDK_TOOLS],
|
||||
operator: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
operator: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'initiate_dingtalk_binding'],
|
||||
viewer: ['Read', 'Glob', 'Grep'],
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -203,6 +203,11 @@ export class ClaudeApiEngine implements AgentEnginePort {
|
|||
output: result.output,
|
||||
isError: result.isError,
|
||||
};
|
||||
|
||||
// Emit any UI side-effect events (e.g. oauth_prompt)
|
||||
if (result.uiEvent) {
|
||||
yield result.uiEvent as any;
|
||||
}
|
||||
}
|
||||
|
||||
// Add tool results as user message to continue the conversation
|
||||
|
|
@ -303,13 +308,28 @@ export class ClaudeApiEngine implements AgentEnginePort {
|
|||
): Array<{ name: string; description: string; input_schema: Record<string, unknown> }> {
|
||||
if (!allowedTools || allowedTools.length === 0) return [];
|
||||
|
||||
return allowedTools.map((toolName) => ({
|
||||
name: toolName,
|
||||
description: `Tool: ${toolName}`,
|
||||
input_schema: {
|
||||
type: 'object' as const,
|
||||
properties: {},
|
||||
const TOOL_SCHEMAS: Record<string, { description: string; properties: Record<string, unknown>; required?: string[] }> = {
|
||||
initiate_dingtalk_binding: {
|
||||
description: 'Generate a DingTalk OAuth authorization URL for an agent instance and push an authorization card to the user\'s screen. Call this after creating the instance.',
|
||||
properties: {
|
||||
instanceId: { type: 'string', description: 'The ID of the newly created agent instance' },
|
||||
instanceName: { type: 'string', description: 'The display name of the agent instance' },
|
||||
},
|
||||
required: ['instanceId', 'instanceName'],
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
return allowedTools.map((toolName) => {
|
||||
const schema = TOOL_SCHEMAS[toolName];
|
||||
return {
|
||||
name: toolName,
|
||||
description: schema?.description ?? `Tool: ${toolName}`,
|
||||
input_schema: {
|
||||
type: 'object' as const,
|
||||
properties: schema?.properties ?? {},
|
||||
...(schema?.required ? { required: schema.required } : {}),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ const exec = promisify(child_process.exec);
|
|||
export interface ToolExecutionResult {
|
||||
output: string;
|
||||
isError: boolean;
|
||||
/** Optional side-effect event to emit to the client UI (e.g. oauth_prompt). */
|
||||
uiEvent?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
|
|
@ -36,6 +38,8 @@ export class ToolExecutor {
|
|||
return this.executeGlob(input.pattern, input.path);
|
||||
case 'Grep':
|
||||
return this.executeGrep(input.pattern, input.path, input.include);
|
||||
case 'initiate_dingtalk_binding':
|
||||
return this.executeInitiateDingTalkBinding(input.instanceId as string, input.instanceName as string);
|
||||
default:
|
||||
return { output: `Unknown tool: ${toolName}`, isError: true };
|
||||
}
|
||||
|
|
@ -121,6 +125,35 @@ export class ToolExecutor {
|
|||
return { output: matches.join('\n') || '(no matches)', isError: false };
|
||||
}
|
||||
|
||||
private async executeInitiateDingTalkBinding(
|
||||
instanceId: string,
|
||||
instanceName: string,
|
||||
): Promise<ToolExecutionResult> {
|
||||
if (!instanceId) {
|
||||
return { output: 'instanceId is required', isError: true };
|
||||
}
|
||||
try {
|
||||
const url = `http://localhost:3002/api/v1/agent/channels/dingtalk/oauth/init?instanceId=${encodeURIComponent(instanceId)}`;
|
||||
const resp = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
||||
if (!resp.ok) {
|
||||
return { output: `Failed to generate OAuth URL: HTTP ${resp.status}`, isError: true };
|
||||
}
|
||||
const data = await resp.json() as { oauthUrl: string };
|
||||
return {
|
||||
output: 'DingTalk OAuth authorization URL generated. The app will show a one-tap authorization card to the user.',
|
||||
isError: false,
|
||||
uiEvent: {
|
||||
type: 'oauth_prompt',
|
||||
url: data.oauthUrl,
|
||||
instanceId,
|
||||
instanceName: instanceName || instanceId,
|
||||
},
|
||||
};
|
||||
} catch (err: any) {
|
||||
return { output: `Failed to generate OAuth URL: ${err.message}`, isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
private async executeGrep(
|
||||
pattern: string,
|
||||
searchPath?: string,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ export interface SystemPromptContext {
|
|||
skills?: string[];
|
||||
riskBoundary?: string;
|
||||
additionalInstructions?: string;
|
||||
/** Voice session ID — when set, uses the internal wget-based OAuth trigger endpoint
|
||||
* instead of the initiate_dingtalk_binding tool, so both engines work. */
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -29,6 +32,23 @@ export class SystemPromptBuilder {
|
|||
*/
|
||||
build(context: SystemPromptContext): string {
|
||||
const parts: string[] = [];
|
||||
// Build the DingTalk OAuth trigger instruction once, reused in the prompt below.
|
||||
// Voice sessions (sessionId present): use Bash/wget to call the internal endpoint —
|
||||
// works for BOTH claude_api and claude_agent_sdk engines.
|
||||
// Text sessions (no sessionId): use the initiate_dingtalk_binding tool (claude_api only).
|
||||
const dingtalkOauthStep = context.sessionId
|
||||
? ` Step 3 — Trigger DingTalk OAuth (Bash/wget — works for all engines):\n` +
|
||||
` wget -q -O- --post-data='{"instanceId":"<id>","instanceName":"<name>"}' \\\n` +
|
||||
` --header="Content-Type: application/json" \\\n` +
|
||||
` http://localhost:3002/api/v1/agent/sessions/${context.sessionId}/dingtalk/oauth-trigger\n` +
|
||||
` This automatically pushes an OAuth authorization card to the user's screen.\n` +
|
||||
` Say: "请查看屏幕上弹出的授权卡片，点击「立即授权」按钮，在钉钉里完成一键授权就好了。"\n` +
|
||||
` Do NOT read out the URL — the card handles it automatically.`
|
||||
: ` Step 3 — Trigger DingTalk OAuth (initiate_dingtalk_binding tool):\n` +
|
||||
` Call: initiate_dingtalk_binding(instanceId="<id>", instanceName="<name>")\n` +
|
||||
` This automatically pushes an OAuth authorization card to the user's screen.\n` +
|
||||
` Say: "请查看屏幕上弹出的授权卡片，点击「立即授权」按钮，在钉钉里完成一键授权就好了。"\n` +
|
||||
` Do NOT read out the URL — the card handles it automatically.`;
|
||||
|
||||
// Base instruction
|
||||
parts.push(
|
||||
|
|
@ -45,25 +65,22 @@ export class SystemPromptBuilder {
|
|||
'1. Ask for a name if not given\n' +
|
||||
'2. Use the Current User ID from this prompt as userId\n' +
|
||||
'3. Call the create API with Bash and report the result (id, status, containerName)\n\n' +
|
||||
'## DingTalk Channel Binding (钉钉接入 — 语音/对话绑定)\n' +
|
||||
'iAgent has a centralized DingTalk bot. Users bind their OpenClaw instance to it using a one-time code.\n' +
|
||||
'When a user asks to bind DingTalk (e.g. "帮我绑定钉钉", "bind DingTalk", "连接钉钉"), follow these steps:\n\n' +
|
||||
' Step 1 — Find the user\'s active instance:\n' +
|
||||
' wget -q -O- http://localhost:3002/api/v1/agent/instances/user/<userId>\n' +
|
||||
' Pick the first instance with status "running". If none running, pick the first one.\n' +
|
||||
' If the user has no instances, tell them to create one first.\n\n' +
|
||||
' Step 2 — Generate a binding code for the instance:\n' +
|
||||
' wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/bind/<instanceId>\n' +
|
||||
' This returns JSON: { "code": "A3F9C2", "expiresAt": "..." }\n\n' +
|
||||
' Step 3 — Tell the user clearly (voice-friendly):\n' +
|
||||
' Speak the code letter-by-letter with pauses: "验证码是 A-3-F-9-C-2"\n' +
|
||||
' Instructions: "请在钉钉中找到 iAgent 机器人，向它发送这6位验证码。发送后绑定会自动完成，有效期15分钟。"\n' +
|
||||
' If speaking via voice: spell each character slowly for the user to type easily.\n\n' +
|
||||
' Check binding status (optional, if user asks):\n' +
|
||||
'## 招募小龙虾 + 钉钉绑定 (全语音/文字引导)\n' +
|
||||
'When a user says "帮我招募一只小龙虾" / "创建小龙虾" / "recruit agent" / "bind DingTalk":\n\n' +
|
||||
' Step 1 — Ask for a name if not provided:\n' +
|
||||
' "你想给这只小龙虾取个什么名字？"\n\n' +
|
||||
' Step 2 — Create the instance (Bash tool):\n' +
|
||||
' wget -q -O- --post-data=\'{"name":"<name>","userId":"<userId>","usePool":true}\' --header="Content-Type: application/json" http://localhost:3002/api/v1/agent/instances\n' +
|
||||
' Parse JSON to get "id" (instanceId) and "name".\n' +
|
||||
' Say: "好的，小龙虾「<name>」已经创建好了！"\n\n' +
|
||||
`${dingtalkOauthStep}\n\n` +
|
||||
' Step 4 — Confirm (after user says they authorized):\n' +
|
||||
' wget -q -O- http://localhost:3002/api/v1/agent/channels/dingtalk/status/<instanceId>\n' +
|
||||
' Returns { "bound": true/false }\n\n' +
|
||||
' Unbind DingTalk:\n' +
|
||||
' wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/unbind/<instanceId>',
|
||||
' If { "bound": true }: "太好了！小龙虾「<name>」已成功绑定钉钉，现在可以在钉钉里和它对话了！"\n' +
|
||||
' If still false: "授权还没完成，请确认是否点击了钉钉里的授权按钮。"\n\n' +
|
||||
' Check/unbind:\n' +
|
||||
' Status: wget -q -O- http://localhost:3002/api/v1/agent/channels/dingtalk/status/<instanceId>\n' +
|
||||
' Unbind: wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/unbind/<instanceId>',
|
||||
);
|
||||
|
||||
// Tenant + user context
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ import { TenantId } from '@it0/common';
|
|||
import { VoiceSessionManager } from '../../../domain/services/voice-session-manager.service';
|
||||
import { SessionRepository } from '../../../infrastructure/repositories/session.repository';
|
||||
import { SystemPromptBuilder } from '../../../infrastructure/engines/claude-code-cli/system-prompt-builder';
|
||||
import { AgentStreamGateway } from '../../ws/agent-stream.gateway';
|
||||
import { DingTalkRouterService } from '../../../infrastructure/dingtalk/dingtalk-router.service';
|
||||
import { AgentInstanceRepository } from '../../../infrastructure/repositories/agent-instance.repository';
|
||||
import { AgentEngineType } from '../../../domain/value-objects/agent-engine-type.vo';
|
||||
import { AgentSession } from '../../../domain/entities/agent-session.entity';
|
||||
import * as crypto from 'crypto';
|
||||
|
|
@ -32,6 +35,9 @@ export class VoiceSessionController {
|
|||
private readonly voiceSessionManager: VoiceSessionManager,
|
||||
private readonly sessionRepository: SessionRepository,
|
||||
private readonly systemPromptBuilder: SystemPromptBuilder,
|
||||
private readonly gateway: AgentStreamGateway,
|
||||
private readonly dingTalkRouter: DingTalkRouterService,
|
||||
private readonly instanceRepo: AgentInstanceRepository,
|
||||
) {}
|
||||
|
||||
/**
|
||||
|
|
@ -57,6 +63,10 @@ export class VoiceSessionController {
|
|||
}
|
||||
}
|
||||
|
||||
// Determine sessionId early — needed for embedding in the system prompt
|
||||
// so Claude can call the DingTalk OAuth trigger endpoint with the correct session.
|
||||
const sessionId = session?.id ?? crypto.randomUUID();
|
||||
|
||||
// Extract user identity from JWT for context-aware system prompt
|
||||
const jwtUser = (req as any).user;
|
||||
const userId: string | undefined = jwtUser?.sub ?? jwtUser?.userId;
|
||||
|
|
@ -68,12 +78,13 @@ export class VoiceSessionController {
|
|||
tenantId,
|
||||
userId,
|
||||
userEmail,
|
||||
sessionId,
|
||||
});
|
||||
|
||||
if (!session) {
|
||||
// Create a fresh session pre-marked as voice mode
|
||||
session = new AgentSession();
|
||||
session.id = crypto.randomUUID();
|
||||
session.id = sessionId;
|
||||
session.tenantId = tenantId;
|
||||
session.engineType = AgentEngineType.CLAUDE_AGENT_SDK;
|
||||
session.status = 'active';
|
||||
|
|
@ -162,4 +173,58 @@ export class VoiceSessionController {
|
|||
this.logger.log(`Voice session terminated: ${sessionId}`);
|
||||
return { sessionId, terminated: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal DingTalk OAuth trigger — called by Claude via Bash/wget.
|
||||
*
|
||||
* Works for BOTH claude_api and claude_agent_sdk engines because Claude
|
||||
* calls it with `wget` (a Bash tool call), not a custom tool.
|
||||
*
|
||||
* Flow:
|
||||
* Claude (Bash) → POST /sessions/:sessionId/dingtalk/oauth-trigger
|
||||
* → generateOAuthUrl() → emit oauth_prompt via WS gateway
|
||||
* → voice-agent picks up → publish_data to Flutter → bottom sheet opens
|
||||
*
|
||||
* This endpoint is intentionally unauthenticated (no JWT guard) because
|
||||
* it is called from localhost inside the container by Claude's Bash tool.
|
||||
* The sessionId acts as a capability token — only the agent running the
|
||||
* session knows it.
|
||||
*/
|
||||
@Post(':sessionId/dingtalk/oauth-trigger')
|
||||
async triggerDingTalkOAuth(
|
||||
@Param('sessionId') sessionId: string,
|
||||
@Body() body: { instanceId: string; instanceName?: string },
|
||||
) {
|
||||
if (!body.instanceId) {
|
||||
throw new BadRequestException('instanceId is required');
|
||||
}
|
||||
|
||||
if (!this.dingTalkRouter.isEnabled()) {
|
||||
return { triggered: false, reason: 'DingTalk integration not configured' };
|
||||
}
|
||||
|
||||
const inst = await this.instanceRepo.findById(body.instanceId);
|
||||
if (!inst) {
|
||||
throw new NotFoundException(`Instance ${body.instanceId} not found`);
|
||||
}
|
||||
|
||||
const instanceName = body.instanceName || inst.name || body.instanceId;
|
||||
const { oauthUrl } = this.dingTalkRouter.generateOAuthUrl(body.instanceId);
|
||||
|
||||
// Emit the oauth_prompt event to the session's WebSocket stream.
|
||||
// voice-agent subscribes to this stream and forwards it to Flutter via
|
||||
// LiveKit publish_data (topic="oauth_prompt"), which triggers the bottom sheet.
|
||||
this.gateway.emitStreamEvent(sessionId, {
|
||||
type: 'oauth_prompt',
|
||||
url: oauthUrl,
|
||||
instanceId: body.instanceId,
|
||||
instanceName,
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`[VoiceSession ${sessionId}] DingTalk OAuth triggered for instance=${body.instanceId}`,
|
||||
);
|
||||
|
||||
return { triggered: true, instanceId: body.instanceId, instanceName };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -339,6 +339,7 @@ async def entrypoint(ctx: JobContext) -> None:
|
|||
agent_service_url=settings.agent_service_url,
|
||||
auth_header=auth_header,
|
||||
engine_type=engine_type,
|
||||
room=ctx.room,
|
||||
)
|
||||
|
||||
# Create and start AgentSession with the full pipeline
|
||||
|
|
|
|||
|
|
@ -56,12 +56,14 @@ class AgentServiceLLM(llm.LLM):
|
|||
agent_service_url: str = "http://agent-service:3002",
|
||||
auth_header: str = "",
|
||||
engine_type: str = "claude_agent_sdk",
|
||||
room=None,
|
||||
):
|
||||
super().__init__()
|
||||
self._agent_service_url = agent_service_url
|
||||
self._auth_header = auth_header
|
||||
self._engine_type = engine_type
|
||||
self._agent_session_id: str | None = None
|
||||
self._room = room # LiveKit Room — used to publish oauth_prompt events to Flutter
|
||||
|
||||
# Voice session mode: long-lived agent process tied to the call duration.
|
||||
# True once start_voice_session() completes successfully.
|
||||
|
|
@ -231,6 +233,8 @@ class AgentServiceLLM(llm.LLM):
|
|||
|
||||
if evt_type == "text":
|
||||
collected_text += evt_data.get("content", "")
|
||||
elif evt_type == "oauth_prompt":
|
||||
await self._publish_oauth_prompt(evt_data)
|
||||
elif evt_type in ("completed", "error"):
|
||||
break
|
||||
|
||||
|
|
@ -325,6 +329,8 @@ class AgentServiceLLM(llm.LLM):
|
|||
|
||||
if evt_type == "text":
|
||||
collected_text += evt_data.get("content", "")
|
||||
elif evt_type == "oauth_prompt":
|
||||
await self._publish_oauth_prompt(evt_data)
|
||||
elif evt_type == "completed":
|
||||
return collected_text
|
||||
elif evt_type == "error":
|
||||
|
|
@ -332,6 +338,22 @@ class AgentServiceLLM(llm.LLM):
|
|||
|
||||
return collected_text
|
||||
|
||||
async def _publish_oauth_prompt(self, evt_data: dict) -> None:
|
||||
"""Publish an oauth_prompt event to Flutter via LiveKit data channel."""
|
||||
if not self._room or not self._room.local_participant:
|
||||
logger.warning("Cannot publish oauth_prompt: room or local_participant not available")
|
||||
return
|
||||
try:
|
||||
payload = json.dumps(evt_data).encode("utf-8")
|
||||
await self._room.local_participant.publish_data(
|
||||
payload,
|
||||
reliable=True,
|
||||
topic="oauth_prompt",
|
||||
)
|
||||
logger.info("Published oauth_prompt to Flutter: instanceId=%s", evt_data.get("instanceId"))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to publish oauth_prompt: %s", exc)
|
||||
|
||||
async def terminate_voice_session(self) -> None:
|
||||
"""
|
||||
Terminate the long-lived voice agent session.
|
||||
|
|
@ -614,6 +636,11 @@ class AgentServiceLLMStream(llm.LLMStream):
|
|||
)
|
||||
)
|
||||
|
||||
elif evt_type == "oauth_prompt":
|
||||
asyncio.create_task(
|
||||
self._llm_instance._publish_oauth_prompt(evt_data)
|
||||
)
|
||||
|
||||
elif evt_type == "completed":
|
||||
logger.info("Voice inject stream completed")
|
||||
return
|
||||
|
|
@ -775,6 +802,11 @@ class AgentServiceLLMStream(llm.LLMStream):
|
|||
)
|
||||
)
|
||||
|
||||
elif evt_type == "oauth_prompt":
|
||||
asyncio.create_task(
|
||||
self._llm_instance._publish_oauth_prompt(evt_data)
|
||||
)
|
||||
|
||||
elif evt_type == "completed":
|
||||
logger.info("Agent stream completed")
|
||||
return
|
||||
|
|
|
|||
Loading…
Reference in New Issue