feat(dingtalk): 小龙虾招募全语音/文字引导流程 + OAuth 一键授权卡片
## 功能说明
用户通过语音或文字说「帮我招募一只小龙虾」,iAgent 全程引导完成
OpenClaw 实例创建 + 钉钉 OAuth 一键授权绑定。
## 核心设计
- 语音场景 (claude_agent_sdk): Claude 通过 Bash/wget 调用内部 HTTP
端点触发 OAuth,绕开 ToolExecutor 限制,两引擎均兼容
- 文字场景 (claude_api): 使用 initiate_dingtalk_binding 自定义工具,
通过 uiEvent 机制传递 OAuth URL
## agent-service 变更
- agent-engine.port.ts: EngineStreamEvent 联合类型新增 oauth_prompt
- allowed-tools-resolver.service.ts: initiate_dingtalk_binding 加入
ALL_SDK_TOOLS / admin / operator 工具白名单
- tool-executor.ts: 新增 executeInitiateDingTalkBinding(),调用内部
oauth/init 端点获取 OAuth URL,返回 uiEvent
- claude-api-engine.ts: 在 tool_result 之后检查 result.uiEvent 并
yield 出去;buildToolDefinitions 注册 initiate_dingtalk_binding schema
- system-prompt-builder.ts:
- SystemPromptContext 新增 sessionId? 字段
- 语音 session (sessionId 存在) → Step 3 使用 wget 调用
POST /sessions/{sessionId}/dingtalk/oauth-trigger(两引擎通用)
- 文字 session (无 sessionId) → Step 3 调用 initiate_dingtalk_binding
工具(claude_api 专用)
- voice-session.controller.ts:
- 注入 AgentStreamGateway / DingTalkRouterService / AgentInstanceRepository
- startVoiceSession: 提前确定 sessionId,在 build() 前传入,使系统
提示能内嵌正确的端点 URL
- 新增 POST :sessionId/dingtalk/oauth-trigger — 无 JWT(内部端点,
由 Claude Bash 工具调用),sessionId 作为能力令牌;生成 OAuth URL
并通过 gateway.emitStreamEvent 直接推送 oauth_prompt 事件到 WS 流
## voice-agent 变更
- agent.py: 构造 AgentServiceLLM 时传入 room=ctx.room
- agent_llm.py:
- __init__ 增加 room 参数,存储为 self._room
- 新增 _publish_oauth_prompt(evt_data): null-safe,通过 LiveKit
publish_data(topic="oauth_prompt") 推送到 Flutter
- _do_inject_voice / _do_inject / _do_stream_voice / _do_stream:
处理 oauth_prompt 事件 → asyncio.create_task(_publish_oauth_prompt)
- 替换已弃用的 asyncio.ensure_future / get_event_loop().create_task
→ asyncio.create_task(Python 3.10+ 兼容)
## Flutter 变更
- agent_call_page.dart: DataReceivedEvent 监听 topic="oauth_prompt",
解析 url/instanceName,弹出 _showOAuthBottomSheet(深色主题,🦞
图标,「立即授权」按钮 launchUrl externalApplication)
- stream_event.dart: 新增 OAuthPromptEvent(url, instanceId, instanceName)
- stream_event_model.dart: toEntity() 新增 'oauth_prompt' case
- chat_message.dart: MessageType 枚举新增 oauthPrompt
- chat_providers.dart: _handleStreamEvent 新增 OAuthPromptEvent case,
生成 type=oauthPrompt 的 ChatMessage(metadata 含 url/instanceName)
- chat_page.dart: 新增 oauthPrompt 时间线节点 + _OAuthPromptCard 组件
(「立即授权」按钮,launchUrl externalApplication);import url_launcher
## 修复的关键 Bug
1. [严重] initiate_dingtalk_binding 只对 claude_api 有效,语音默认用
claude_agent_sdk → 新 wget 端点两引擎均可用
2. [严重] 文字聊天页面不处理 oauth_prompt 事件(静默丢弃)→ 补全
Flutter 4 处代码(entity/model/provider/page)
3. [中] _publish_oauth_prompt 缺 local_participant null 检查 → 已修复
4. [轻] asyncio.ensure_future / get_event_loop() 弃用警告 → 已修复
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3d626aebb5
commit
64499a5d86
|
|
@ -8,6 +8,7 @@ import 'package:it0_app/l10n/app_localizations.dart';
|
||||||
import 'package:livekit_client/livekit_client.dart';
|
import 'package:livekit_client/livekit_client.dart';
|
||||||
import 'package:image_picker/image_picker.dart';
|
import 'package:image_picker/image_picker.dart';
|
||||||
import 'package:file_picker/file_picker.dart';
|
import 'package:file_picker/file_picker.dart';
|
||||||
|
import 'package:url_launcher/url_launcher.dart';
|
||||||
import '../../../../core/config/api_endpoints.dart';
|
import '../../../../core/config/api_endpoints.dart';
|
||||||
import '../../../../core/config/app_config.dart';
|
import '../../../../core/config/app_config.dart';
|
||||||
import '../../../../core/network/dio_client.dart';
|
import '../../../../core/network/dio_client.dart';
|
||||||
|
|
@ -190,6 +191,15 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
||||||
_scrollToBottom();
|
_scrollToBottom();
|
||||||
}
|
}
|
||||||
} catch (_) {}
|
} catch (_) {}
|
||||||
|
} else if (event.topic == 'oauth_prompt') {
|
||||||
|
try {
|
||||||
|
final payload = jsonDecode(utf8.decode(event.data));
|
||||||
|
final url = payload['url'] as String? ?? '';
|
||||||
|
final instanceName = payload['instanceName'] as String? ?? '小龙虾';
|
||||||
|
if (url.isNotEmpty && mounted) {
|
||||||
|
_showOAuthBottomSheet(url, instanceName);
|
||||||
|
}
|
||||||
|
} catch (_) {}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -297,6 +307,78 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
||||||
_endCall();
|
_endCall();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Show DingTalk OAuth authorization card pushed by iAgent.
|
||||||
|
void _showOAuthBottomSheet(String url, String instanceName) {
|
||||||
|
showModalBottomSheet(
|
||||||
|
context: context,
|
||||||
|
backgroundColor: const Color(0xFF1A1D2E),
|
||||||
|
shape: const RoundedRectangleBorder(
|
||||||
|
borderRadius: BorderRadius.vertical(top: Radius.circular(20)),
|
||||||
|
),
|
||||||
|
builder: (ctx) => Padding(
|
||||||
|
padding: const EdgeInsets.fromLTRB(24, 20, 24, 36),
|
||||||
|
child: Column(
|
||||||
|
mainAxisSize: MainAxisSize.min,
|
||||||
|
children: [
|
||||||
|
Container(
|
||||||
|
width: 40, height: 4,
|
||||||
|
decoration: BoxDecoration(
|
||||||
|
color: Colors.white24,
|
||||||
|
borderRadius: BorderRadius.circular(2),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
const SizedBox(height: 20),
|
||||||
|
const Text('🦞', style: TextStyle(fontSize: 48)),
|
||||||
|
const SizedBox(height: 12),
|
||||||
|
Text(
|
||||||
|
'授权「$instanceName」接入钉钉',
|
||||||
|
style: const TextStyle(
|
||||||
|
color: Colors.white,
|
||||||
|
fontSize: 18,
|
||||||
|
fontWeight: FontWeight.bold,
|
||||||
|
),
|
||||||
|
textAlign: TextAlign.center,
|
||||||
|
),
|
||||||
|
const SizedBox(height: 8),
|
||||||
|
const Text(
|
||||||
|
'点击下方按钮,在钉钉中一键完成授权绑定',
|
||||||
|
style: TextStyle(color: Colors.white54, fontSize: 14),
|
||||||
|
textAlign: TextAlign.center,
|
||||||
|
),
|
||||||
|
const SizedBox(height: 24),
|
||||||
|
SizedBox(
|
||||||
|
width: double.infinity,
|
||||||
|
height: 52,
|
||||||
|
child: ElevatedButton(
|
||||||
|
style: ElevatedButton.styleFrom(
|
||||||
|
backgroundColor: const Color(0xFF1677FF),
|
||||||
|
shape: RoundedRectangleBorder(
|
||||||
|
borderRadius: BorderRadius.circular(12),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
onPressed: () async {
|
||||||
|
Navigator.of(ctx).pop();
|
||||||
|
final uri = Uri.tryParse(url);
|
||||||
|
if (uri != null) {
|
||||||
|
await launchUrl(uri, mode: LaunchMode.externalApplication);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
child: const Text(
|
||||||
|
'立即授权',
|
||||||
|
style: TextStyle(
|
||||||
|
color: Colors.white,
|
||||||
|
fontSize: 16,
|
||||||
|
fontWeight: FontWeight.w600,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Toggle microphone mute.
|
/// Toggle microphone mute.
|
||||||
void _toggleMute() {
|
void _toggleMute() {
|
||||||
_isMuted = !_isMuted;
|
_isMuted = !_isMuted;
|
||||||
|
|
|
||||||
|
|
@ -88,6 +88,13 @@ class StreamEventModel {
|
||||||
case 'cancelled':
|
case 'cancelled':
|
||||||
return CancelledEvent();
|
return CancelledEvent();
|
||||||
|
|
||||||
|
case 'oauth_prompt':
|
||||||
|
return OAuthPromptEvent(
|
||||||
|
data['url'] as String? ?? '',
|
||||||
|
data['instanceId'] as String? ?? '',
|
||||||
|
data['instanceName'] as String? ?? '小龙虾',
|
||||||
|
);
|
||||||
|
|
||||||
default:
|
default:
|
||||||
// Fall back to text event for unknown types
|
// Fall back to text event for unknown types
|
||||||
return TextEvent(
|
return TextEvent(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
enum MessageRole { user, assistant, system }
|
enum MessageRole { user, assistant, system }
|
||||||
|
|
||||||
enum MessageType { text, toolUse, toolResult, approval, thinking, standingOrderDraft, interrupted }
|
enum MessageType { text, toolUse, toolResult, approval, thinking, standingOrderDraft, interrupted, oauthPrompt }
|
||||||
|
|
||||||
enum ToolStatus { executing, completed, error, blocked, awaitingApproval }
|
enum ToolStatus { executing, completed, error, blocked, awaitingApproval }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -67,3 +67,12 @@ class TaskInfoEvent extends StreamEvent {
|
||||||
class CancelledEvent extends StreamEvent {
|
class CancelledEvent extends StreamEvent {
|
||||||
CancelledEvent();
|
CancelledEvent();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Emitted when the agent wants to trigger DingTalk OAuth for an agent instance.
|
||||||
|
/// The UI should show an authorization card so the user can tap "立即授权".
|
||||||
|
class OAuthPromptEvent extends StreamEvent {
|
||||||
|
final String url;
|
||||||
|
final String instanceId;
|
||||||
|
final String instanceName;
|
||||||
|
OAuthPromptEvent(this.url, this.instanceId, this.instanceName);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import 'dart:io';
|
||||||
import 'package:flutter/material.dart';
|
import 'package:flutter/material.dart';
|
||||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||||
import 'package:it0_app/l10n/app_localizations.dart';
|
import 'package:it0_app/l10n/app_localizations.dart';
|
||||||
|
import 'package:url_launcher/url_launcher.dart';
|
||||||
import 'package:image_picker/image_picker.dart';
|
import 'package:image_picker/image_picker.dart';
|
||||||
import 'package:file_picker/file_picker.dart';
|
import 'package:file_picker/file_picker.dart';
|
||||||
import '../../../../core/theme/app_colors.dart';
|
import '../../../../core/theme/app_colors.dart';
|
||||||
|
|
@ -483,6 +484,18 @@ class _ChatPageState extends ConsumerState<ChatPage> {
|
||||||
icon: Icons.cancel_outlined,
|
icon: Icons.cancel_outlined,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
case MessageType.oauthPrompt:
|
||||||
|
final url = message.metadata?['url'] as String? ?? '';
|
||||||
|
final instanceName = message.metadata?['instanceName'] as String? ?? '小龙虾';
|
||||||
|
return TimelineEventNode(
|
||||||
|
status: NodeStatus.warning,
|
||||||
|
label: '钉钉授权',
|
||||||
|
isFirst: isFirst,
|
||||||
|
isLast: isLast,
|
||||||
|
icon: Icons.link,
|
||||||
|
content: _OAuthPromptCard(url: url, instanceName: instanceName),
|
||||||
|
);
|
||||||
|
|
||||||
case MessageType.text:
|
case MessageType.text:
|
||||||
default:
|
default:
|
||||||
return TimelineEventNode(
|
return TimelineEventNode(
|
||||||
|
|
@ -779,6 +792,50 @@ class _ChatPageState extends ConsumerState<ChatPage> {
|
||||||
// Standing order content (embedded in timeline node)
|
// Standing order content (embedded in timeline node)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// OAuth prompt card — shown in timeline when agent triggers DingTalk binding
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class _OAuthPromptCard extends StatelessWidget {
|
||||||
|
final String url;
|
||||||
|
final String instanceName;
|
||||||
|
|
||||||
|
const _OAuthPromptCard({required this.url, required this.instanceName});
|
||||||
|
|
||||||
|
@override
|
||||||
|
Widget build(BuildContext context) {
|
||||||
|
return Column(
|
||||||
|
crossAxisAlignment: CrossAxisAlignment.start,
|
||||||
|
children: [
|
||||||
|
Text(
|
||||||
|
'点击下方按钮,在钉钉中为「$instanceName」完成一键授权绑定。',
|
||||||
|
style: const TextStyle(color: AppColors.textSecondary, fontSize: 13),
|
||||||
|
),
|
||||||
|
const SizedBox(height: 10),
|
||||||
|
SizedBox(
|
||||||
|
width: double.infinity,
|
||||||
|
child: ElevatedButton.icon(
|
||||||
|
icon: const Text('🦞', style: TextStyle(fontSize: 16)),
|
||||||
|
label: const Text('立即授权'),
|
||||||
|
style: ElevatedButton.styleFrom(
|
||||||
|
backgroundColor: const Color(0xFF1677FF),
|
||||||
|
foregroundColor: Colors.white,
|
||||||
|
shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(10)),
|
||||||
|
padding: const EdgeInsets.symmetric(vertical: 12),
|
||||||
|
),
|
||||||
|
onPressed: () async {
|
||||||
|
final uri = Uri.tryParse(url);
|
||||||
|
if (uri != null) {
|
||||||
|
await launchUrl(uri, mode: LaunchMode.externalApplication);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Collapsible code block for tool results – collapsed by default, tap to expand
|
// Collapsible code block for tool results – collapsed by default, tap to expand
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
|
@ -348,6 +348,18 @@ class ChatNotifier extends StateNotifier<ChatState> {
|
||||||
if (state.agentStatus != AgentStatus.idle) {
|
if (state.agentStatus != AgentStatus.idle) {
|
||||||
state = state.copyWith(agentStatus: AgentStatus.idle, clearTaskId: true);
|
state = state.copyWith(agentStatus: AgentStatus.idle, clearTaskId: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case OAuthPromptEvent(:final url, :final instanceId, :final instanceName):
|
||||||
|
// Add an authorization card to the timeline so the user can tap "立即授权"
|
||||||
|
final msg = ChatMessage(
|
||||||
|
id: DateTime.now().microsecondsSinceEpoch.toString(),
|
||||||
|
role: MessageRole.assistant,
|
||||||
|
content: '授权「$instanceName」接入钉钉',
|
||||||
|
timestamp: DateTime.now(),
|
||||||
|
type: MessageType.oauthPrompt,
|
||||||
|
metadata: {'url': url, 'instanceId': instanceId, 'instanceName': instanceName},
|
||||||
|
);
|
||||||
|
state = state.copyWith(messages: [...state.messages, msg]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -47,4 +47,5 @@ export type EngineStreamEvent =
|
||||||
}
|
}
|
||||||
| { type: 'error'; message: string; code: string }
|
| { type: 'error'; message: string; code: string }
|
||||||
| { type: 'cancelled'; message: string; code: string }
|
| { type: 'cancelled'; message: string; code: string }
|
||||||
| { type: 'task_info'; taskId: string };
|
| { type: 'task_info'; taskId: string }
|
||||||
|
| { type: 'oauth_prompt'; url: string; instanceId: string; instanceName: string };
|
||||||
|
|
|
||||||
|
|
@ -16,11 +16,12 @@ import { TenantAgentConfig } from '../entities/tenant-agent-config.entity';
|
||||||
const ALL_SDK_TOOLS = [
|
const ALL_SDK_TOOLS = [
|
||||||
'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep',
|
'Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep',
|
||||||
'WebFetch', 'WebSearch', 'NotebookEdit', 'Task',
|
'WebFetch', 'WebSearch', 'NotebookEdit', 'Task',
|
||||||
|
'initiate_dingtalk_binding',
|
||||||
];
|
];
|
||||||
|
|
||||||
const ROLE_TOOL_MAP: Record<string, string[]> = {
|
const ROLE_TOOL_MAP: Record<string, string[]> = {
|
||||||
admin: [...ALL_SDK_TOOLS],
|
admin: [...ALL_SDK_TOOLS],
|
||||||
operator: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
operator: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'initiate_dingtalk_binding'],
|
||||||
viewer: ['Read', 'Glob', 'Grep'],
|
viewer: ['Read', 'Glob', 'Grep'],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -203,6 +203,11 @@ export class ClaudeApiEngine implements AgentEnginePort {
|
||||||
output: result.output,
|
output: result.output,
|
||||||
isError: result.isError,
|
isError: result.isError,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Emit any UI side-effect events (e.g. oauth_prompt)
|
||||||
|
if (result.uiEvent) {
|
||||||
|
yield result.uiEvent as any;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add tool results as user message to continue the conversation
|
// Add tool results as user message to continue the conversation
|
||||||
|
|
@ -303,13 +308,28 @@ export class ClaudeApiEngine implements AgentEnginePort {
|
||||||
): Array<{ name: string; description: string; input_schema: Record<string, unknown> }> {
|
): Array<{ name: string; description: string; input_schema: Record<string, unknown> }> {
|
||||||
if (!allowedTools || allowedTools.length === 0) return [];
|
if (!allowedTools || allowedTools.length === 0) return [];
|
||||||
|
|
||||||
return allowedTools.map((toolName) => ({
|
const TOOL_SCHEMAS: Record<string, { description: string; properties: Record<string, unknown>; required?: string[] }> = {
|
||||||
name: toolName,
|
initiate_dingtalk_binding: {
|
||||||
description: `Tool: ${toolName}`,
|
description: 'Generate a DingTalk OAuth authorization URL for an agent instance and push an authorization card to the user\'s screen. Call this after creating the instance.',
|
||||||
input_schema: {
|
properties: {
|
||||||
type: 'object' as const,
|
instanceId: { type: 'string', description: 'The ID of the newly created agent instance' },
|
||||||
properties: {},
|
instanceName: { type: 'string', description: 'The display name of the agent instance' },
|
||||||
|
},
|
||||||
|
required: ['instanceId', 'instanceName'],
|
||||||
},
|
},
|
||||||
}));
|
};
|
||||||
|
|
||||||
|
return allowedTools.map((toolName) => {
|
||||||
|
const schema = TOOL_SCHEMAS[toolName];
|
||||||
|
return {
|
||||||
|
name: toolName,
|
||||||
|
description: schema?.description ?? `Tool: ${toolName}`,
|
||||||
|
input_schema: {
|
||||||
|
type: 'object' as const,
|
||||||
|
properties: schema?.properties ?? {},
|
||||||
|
...(schema?.required ? { required: schema.required } : {}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,8 @@ const exec = promisify(child_process.exec);
|
||||||
export interface ToolExecutionResult {
|
export interface ToolExecutionResult {
|
||||||
output: string;
|
output: string;
|
||||||
isError: boolean;
|
isError: boolean;
|
||||||
|
/** Optional side-effect event to emit to the client UI (e.g. oauth_prompt). */
|
||||||
|
uiEvent?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
|
|
@ -36,6 +38,8 @@ export class ToolExecutor {
|
||||||
return this.executeGlob(input.pattern, input.path);
|
return this.executeGlob(input.pattern, input.path);
|
||||||
case 'Grep':
|
case 'Grep':
|
||||||
return this.executeGrep(input.pattern, input.path, input.include);
|
return this.executeGrep(input.pattern, input.path, input.include);
|
||||||
|
case 'initiate_dingtalk_binding':
|
||||||
|
return this.executeInitiateDingTalkBinding(input.instanceId as string, input.instanceName as string);
|
||||||
default:
|
default:
|
||||||
return { output: `Unknown tool: ${toolName}`, isError: true };
|
return { output: `Unknown tool: ${toolName}`, isError: true };
|
||||||
}
|
}
|
||||||
|
|
@ -121,6 +125,35 @@ export class ToolExecutor {
|
||||||
return { output: matches.join('\n') || '(no matches)', isError: false };
|
return { output: matches.join('\n') || '(no matches)', isError: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async executeInitiateDingTalkBinding(
|
||||||
|
instanceId: string,
|
||||||
|
instanceName: string,
|
||||||
|
): Promise<ToolExecutionResult> {
|
||||||
|
if (!instanceId) {
|
||||||
|
return { output: 'instanceId is required', isError: true };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const url = `http://localhost:3002/api/v1/agent/channels/dingtalk/oauth/init?instanceId=${encodeURIComponent(instanceId)}`;
|
||||||
|
const resp = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
||||||
|
if (!resp.ok) {
|
||||||
|
return { output: `Failed to generate OAuth URL: HTTP ${resp.status}`, isError: true };
|
||||||
|
}
|
||||||
|
const data = await resp.json() as { oauthUrl: string };
|
||||||
|
return {
|
||||||
|
output: 'DingTalk OAuth authorization URL generated. The app will show a one-tap authorization card to the user.',
|
||||||
|
isError: false,
|
||||||
|
uiEvent: {
|
||||||
|
type: 'oauth_prompt',
|
||||||
|
url: data.oauthUrl,
|
||||||
|
instanceId,
|
||||||
|
instanceName: instanceName || instanceId,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (err: any) {
|
||||||
|
return { output: `Failed to generate OAuth URL: ${err.message}`, isError: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async executeGrep(
|
private async executeGrep(
|
||||||
pattern: string,
|
pattern: string,
|
||||||
searchPath?: string,
|
searchPath?: string,
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,9 @@ export interface SystemPromptContext {
|
||||||
skills?: string[];
|
skills?: string[];
|
||||||
riskBoundary?: string;
|
riskBoundary?: string;
|
||||||
additionalInstructions?: string;
|
additionalInstructions?: string;
|
||||||
|
/** Voice session ID — when set, uses the internal wget-based OAuth trigger endpoint
|
||||||
|
* instead of the initiate_dingtalk_binding tool, so both engines work. */
|
||||||
|
sessionId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -29,6 +32,23 @@ export class SystemPromptBuilder {
|
||||||
*/
|
*/
|
||||||
build(context: SystemPromptContext): string {
|
build(context: SystemPromptContext): string {
|
||||||
const parts: string[] = [];
|
const parts: string[] = [];
|
||||||
|
// Build the DingTalk OAuth trigger instruction once, reused in the prompt below.
|
||||||
|
// Voice sessions (sessionId present): use Bash/wget to call the internal endpoint —
|
||||||
|
// works for BOTH claude_api and claude_agent_sdk engines.
|
||||||
|
// Text sessions (no sessionId): use the initiate_dingtalk_binding tool (claude_api only).
|
||||||
|
const dingtalkOauthStep = context.sessionId
|
||||||
|
? ` Step 3 — Trigger DingTalk OAuth (Bash/wget — works for all engines):\n` +
|
||||||
|
` wget -q -O- --post-data='{"instanceId":"<id>","instanceName":"<name>"}' \\\n` +
|
||||||
|
` --header="Content-Type: application/json" \\\n` +
|
||||||
|
` http://localhost:3002/api/v1/agent/sessions/${context.sessionId}/dingtalk/oauth-trigger\n` +
|
||||||
|
` This automatically pushes an OAuth authorization card to the user's screen.\n` +
|
||||||
|
` Say: "请查看屏幕上弹出的授权卡片,点击「立即授权」按钮,在钉钉里完成一键授权就好了。"\n` +
|
||||||
|
` Do NOT read out the URL — the card handles it automatically.`
|
||||||
|
: ` Step 3 — Trigger DingTalk OAuth (initiate_dingtalk_binding tool):\n` +
|
||||||
|
` Call: initiate_dingtalk_binding(instanceId="<id>", instanceName="<name>")\n` +
|
||||||
|
` This automatically pushes an OAuth authorization card to the user's screen.\n` +
|
||||||
|
` Say: "请查看屏幕上弹出的授权卡片,点击「立即授权」按钮,在钉钉里完成一键授权就好了。"\n` +
|
||||||
|
` Do NOT read out the URL — the card handles it automatically.`;
|
||||||
|
|
||||||
// Base instruction
|
// Base instruction
|
||||||
parts.push(
|
parts.push(
|
||||||
|
|
@ -45,25 +65,22 @@ export class SystemPromptBuilder {
|
||||||
'1. Ask for a name if not given\n' +
|
'1. Ask for a name if not given\n' +
|
||||||
'2. Use the Current User ID from this prompt as userId\n' +
|
'2. Use the Current User ID from this prompt as userId\n' +
|
||||||
'3. Call the create API with Bash and report the result (id, status, containerName)\n\n' +
|
'3. Call the create API with Bash and report the result (id, status, containerName)\n\n' +
|
||||||
'## DingTalk Channel Binding (钉钉接入 — 语音/对话绑定)\n' +
|
'## 招募小龙虾 + 钉钉绑定 (全语音/文字引导)\n' +
|
||||||
'iAgent has a centralized DingTalk bot. Users bind their OpenClaw instance to it using a one-time code.\n' +
|
'When a user says "帮我招募一只小龙虾" / "创建小龙虾" / "recruit agent" / "bind DingTalk":\n\n' +
|
||||||
'When a user asks to bind DingTalk (e.g. "帮我绑定钉钉", "bind DingTalk", "连接钉钉"), follow these steps:\n\n' +
|
' Step 1 — Ask for a name if not provided:\n' +
|
||||||
' Step 1 — Find the user\'s active instance:\n' +
|
' "你想给这只小龙虾取个什么名字?"\n\n' +
|
||||||
' wget -q -O- http://localhost:3002/api/v1/agent/instances/user/<userId>\n' +
|
' Step 2 — Create the instance (Bash tool):\n' +
|
||||||
' Pick the first instance with status "running". If none running, pick the first one.\n' +
|
' wget -q -O- --post-data=\'{"name":"<name>","userId":"<userId>","usePool":true}\' --header="Content-Type: application/json" http://localhost:3002/api/v1/agent/instances\n' +
|
||||||
' If the user has no instances, tell them to create one first.\n\n' +
|
' Parse JSON to get "id" (instanceId) and "name".\n' +
|
||||||
' Step 2 — Generate a binding code for the instance:\n' +
|
' Say: "好的,小龙虾「<name>」已经创建好了!"\n\n' +
|
||||||
' wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/bind/<instanceId>\n' +
|
`${dingtalkOauthStep}\n\n` +
|
||||||
' This returns JSON: { "code": "A3F9C2", "expiresAt": "..." }\n\n' +
|
' Step 4 — Confirm (after user says they authorized):\n' +
|
||||||
' Step 3 — Tell the user clearly (voice-friendly):\n' +
|
|
||||||
' Speak the code letter-by-letter with pauses: "验证码是 A-3-F-9-C-2"\n' +
|
|
||||||
' Instructions: "请在钉钉中找到 iAgent 机器人,向它发送这6位验证码。发送后绑定会自动完成,有效期15分钟。"\n' +
|
|
||||||
' If speaking via voice: spell each character slowly for the user to type easily.\n\n' +
|
|
||||||
' Check binding status (optional, if user asks):\n' +
|
|
||||||
' wget -q -O- http://localhost:3002/api/v1/agent/channels/dingtalk/status/<instanceId>\n' +
|
' wget -q -O- http://localhost:3002/api/v1/agent/channels/dingtalk/status/<instanceId>\n' +
|
||||||
' Returns { "bound": true/false }\n\n' +
|
' If { "bound": true }: "太好了!小龙虾「<name>」已成功绑定钉钉,现在可以在钉钉里和它对话了!"\n' +
|
||||||
' Unbind DingTalk:\n' +
|
' If still false: "授权还没完成,请确认是否点击了钉钉里的授权按钮。"\n\n' +
|
||||||
' wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/unbind/<instanceId>',
|
' Check/unbind:\n' +
|
||||||
|
' Status: wget -q -O- http://localhost:3002/api/v1/agent/channels/dingtalk/status/<instanceId>\n' +
|
||||||
|
' Unbind: wget -q -O- --post-data="" http://localhost:3002/api/v1/agent/channels/dingtalk/unbind/<instanceId>',
|
||||||
);
|
);
|
||||||
|
|
||||||
// Tenant + user context
|
// Tenant + user context
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,9 @@ import { TenantId } from '@it0/common';
|
||||||
import { VoiceSessionManager } from '../../../domain/services/voice-session-manager.service';
|
import { VoiceSessionManager } from '../../../domain/services/voice-session-manager.service';
|
||||||
import { SessionRepository } from '../../../infrastructure/repositories/session.repository';
|
import { SessionRepository } from '../../../infrastructure/repositories/session.repository';
|
||||||
import { SystemPromptBuilder } from '../../../infrastructure/engines/claude-code-cli/system-prompt-builder';
|
import { SystemPromptBuilder } from '../../../infrastructure/engines/claude-code-cli/system-prompt-builder';
|
||||||
|
import { AgentStreamGateway } from '../../ws/agent-stream.gateway';
|
||||||
|
import { DingTalkRouterService } from '../../../infrastructure/dingtalk/dingtalk-router.service';
|
||||||
|
import { AgentInstanceRepository } from '../../../infrastructure/repositories/agent-instance.repository';
|
||||||
import { AgentEngineType } from '../../../domain/value-objects/agent-engine-type.vo';
|
import { AgentEngineType } from '../../../domain/value-objects/agent-engine-type.vo';
|
||||||
import { AgentSession } from '../../../domain/entities/agent-session.entity';
|
import { AgentSession } from '../../../domain/entities/agent-session.entity';
|
||||||
import * as crypto from 'crypto';
|
import * as crypto from 'crypto';
|
||||||
|
|
@ -32,6 +35,9 @@ export class VoiceSessionController {
|
||||||
private readonly voiceSessionManager: VoiceSessionManager,
|
private readonly voiceSessionManager: VoiceSessionManager,
|
||||||
private readonly sessionRepository: SessionRepository,
|
private readonly sessionRepository: SessionRepository,
|
||||||
private readonly systemPromptBuilder: SystemPromptBuilder,
|
private readonly systemPromptBuilder: SystemPromptBuilder,
|
||||||
|
private readonly gateway: AgentStreamGateway,
|
||||||
|
private readonly dingTalkRouter: DingTalkRouterService,
|
||||||
|
private readonly instanceRepo: AgentInstanceRepository,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -57,6 +63,10 @@ export class VoiceSessionController {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Determine sessionId early — needed for embedding in the system prompt
|
||||||
|
// so Claude can call the DingTalk OAuth trigger endpoint with the correct session.
|
||||||
|
const sessionId = session?.id ?? crypto.randomUUID();
|
||||||
|
|
||||||
// Extract user identity from JWT for context-aware system prompt
|
// Extract user identity from JWT for context-aware system prompt
|
||||||
const jwtUser = (req as any).user;
|
const jwtUser = (req as any).user;
|
||||||
const userId: string | undefined = jwtUser?.sub ?? jwtUser?.userId;
|
const userId: string | undefined = jwtUser?.sub ?? jwtUser?.userId;
|
||||||
|
|
@ -68,12 +78,13 @@ export class VoiceSessionController {
|
||||||
tenantId,
|
tenantId,
|
||||||
userId,
|
userId,
|
||||||
userEmail,
|
userEmail,
|
||||||
|
sessionId,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!session) {
|
if (!session) {
|
||||||
// Create a fresh session pre-marked as voice mode
|
// Create a fresh session pre-marked as voice mode
|
||||||
session = new AgentSession();
|
session = new AgentSession();
|
||||||
session.id = crypto.randomUUID();
|
session.id = sessionId;
|
||||||
session.tenantId = tenantId;
|
session.tenantId = tenantId;
|
||||||
session.engineType = AgentEngineType.CLAUDE_AGENT_SDK;
|
session.engineType = AgentEngineType.CLAUDE_AGENT_SDK;
|
||||||
session.status = 'active';
|
session.status = 'active';
|
||||||
|
|
@ -162,4 +173,58 @@ export class VoiceSessionController {
|
||||||
this.logger.log(`Voice session terminated: ${sessionId}`);
|
this.logger.log(`Voice session terminated: ${sessionId}`);
|
||||||
return { sessionId, terminated: true };
|
return { sessionId, terminated: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal DingTalk OAuth trigger — called by Claude via Bash/wget.
|
||||||
|
*
|
||||||
|
* Works for BOTH claude_api and claude_agent_sdk engines because Claude
|
||||||
|
* calls it with `wget` (a Bash tool call), not a custom tool.
|
||||||
|
*
|
||||||
|
* Flow:
|
||||||
|
* Claude (Bash) → POST /sessions/:sessionId/dingtalk/oauth-trigger
|
||||||
|
* → generateOAuthUrl() → emit oauth_prompt via WS gateway
|
||||||
|
* → voice-agent picks up → publish_data to Flutter → bottom sheet opens
|
||||||
|
*
|
||||||
|
* This endpoint is intentionally unauthenticated (no JWT guard) because
|
||||||
|
* it is called from localhost inside the container by Claude's Bash tool.
|
||||||
|
* The sessionId acts as a capability token — only the agent running the
|
||||||
|
* session knows it.
|
||||||
|
*/
|
||||||
|
@Post(':sessionId/dingtalk/oauth-trigger')
|
||||||
|
async triggerDingTalkOAuth(
|
||||||
|
@Param('sessionId') sessionId: string,
|
||||||
|
@Body() body: { instanceId: string; instanceName?: string },
|
||||||
|
) {
|
||||||
|
if (!body.instanceId) {
|
||||||
|
throw new BadRequestException('instanceId is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.dingTalkRouter.isEnabled()) {
|
||||||
|
return { triggered: false, reason: 'DingTalk integration not configured' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const inst = await this.instanceRepo.findById(body.instanceId);
|
||||||
|
if (!inst) {
|
||||||
|
throw new NotFoundException(`Instance ${body.instanceId} not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const instanceName = body.instanceName || inst.name || body.instanceId;
|
||||||
|
const { oauthUrl } = this.dingTalkRouter.generateOAuthUrl(body.instanceId);
|
||||||
|
|
||||||
|
// Emit the oauth_prompt event to the session's WebSocket stream.
|
||||||
|
// voice-agent subscribes to this stream and forwards it to Flutter via
|
||||||
|
// LiveKit publish_data (topic="oauth_prompt"), which triggers the bottom sheet.
|
||||||
|
this.gateway.emitStreamEvent(sessionId, {
|
||||||
|
type: 'oauth_prompt',
|
||||||
|
url: oauthUrl,
|
||||||
|
instanceId: body.instanceId,
|
||||||
|
instanceName,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`[VoiceSession ${sessionId}] DingTalk OAuth triggered for instance=${body.instanceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return { triggered: true, instanceId: body.instanceId, instanceName };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -339,6 +339,7 @@ async def entrypoint(ctx: JobContext) -> None:
|
||||||
agent_service_url=settings.agent_service_url,
|
agent_service_url=settings.agent_service_url,
|
||||||
auth_header=auth_header,
|
auth_header=auth_header,
|
||||||
engine_type=engine_type,
|
engine_type=engine_type,
|
||||||
|
room=ctx.room,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create and start AgentSession with the full pipeline
|
# Create and start AgentSession with the full pipeline
|
||||||
|
|
|
||||||
|
|
@ -56,12 +56,14 @@ class AgentServiceLLM(llm.LLM):
|
||||||
agent_service_url: str = "http://agent-service:3002",
|
agent_service_url: str = "http://agent-service:3002",
|
||||||
auth_header: str = "",
|
auth_header: str = "",
|
||||||
engine_type: str = "claude_agent_sdk",
|
engine_type: str = "claude_agent_sdk",
|
||||||
|
room=None,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._agent_service_url = agent_service_url
|
self._agent_service_url = agent_service_url
|
||||||
self._auth_header = auth_header
|
self._auth_header = auth_header
|
||||||
self._engine_type = engine_type
|
self._engine_type = engine_type
|
||||||
self._agent_session_id: str | None = None
|
self._agent_session_id: str | None = None
|
||||||
|
self._room = room # LiveKit Room — used to publish oauth_prompt events to Flutter
|
||||||
|
|
||||||
# Voice session mode: long-lived agent process tied to the call duration.
|
# Voice session mode: long-lived agent process tied to the call duration.
|
||||||
# True once start_voice_session() completes successfully.
|
# True once start_voice_session() completes successfully.
|
||||||
|
|
@ -231,6 +233,8 @@ class AgentServiceLLM(llm.LLM):
|
||||||
|
|
||||||
if evt_type == "text":
|
if evt_type == "text":
|
||||||
collected_text += evt_data.get("content", "")
|
collected_text += evt_data.get("content", "")
|
||||||
|
elif evt_type == "oauth_prompt":
|
||||||
|
await self._publish_oauth_prompt(evt_data)
|
||||||
elif evt_type in ("completed", "error"):
|
elif evt_type in ("completed", "error"):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
@ -325,6 +329,8 @@ class AgentServiceLLM(llm.LLM):
|
||||||
|
|
||||||
if evt_type == "text":
|
if evt_type == "text":
|
||||||
collected_text += evt_data.get("content", "")
|
collected_text += evt_data.get("content", "")
|
||||||
|
elif evt_type == "oauth_prompt":
|
||||||
|
await self._publish_oauth_prompt(evt_data)
|
||||||
elif evt_type == "completed":
|
elif evt_type == "completed":
|
||||||
return collected_text
|
return collected_text
|
||||||
elif evt_type == "error":
|
elif evt_type == "error":
|
||||||
|
|
@ -332,6 +338,22 @@ class AgentServiceLLM(llm.LLM):
|
||||||
|
|
||||||
return collected_text
|
return collected_text
|
||||||
|
|
||||||
|
async def _publish_oauth_prompt(self, evt_data: dict) -> None:
|
||||||
|
"""Publish an oauth_prompt event to Flutter via LiveKit data channel."""
|
||||||
|
if not self._room or not self._room.local_participant:
|
||||||
|
logger.warning("Cannot publish oauth_prompt: room or local_participant not available")
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
payload = json.dumps(evt_data).encode("utf-8")
|
||||||
|
await self._room.local_participant.publish_data(
|
||||||
|
payload,
|
||||||
|
reliable=True,
|
||||||
|
topic="oauth_prompt",
|
||||||
|
)
|
||||||
|
logger.info("Published oauth_prompt to Flutter: instanceId=%s", evt_data.get("instanceId"))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to publish oauth_prompt: %s", exc)
|
||||||
|
|
||||||
async def terminate_voice_session(self) -> None:
|
async def terminate_voice_session(self) -> None:
|
||||||
"""
|
"""
|
||||||
Terminate the long-lived voice agent session.
|
Terminate the long-lived voice agent session.
|
||||||
|
|
@ -614,6 +636,11 @@ class AgentServiceLLMStream(llm.LLMStream):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
elif evt_type == "oauth_prompt":
|
||||||
|
asyncio.create_task(
|
||||||
|
self._llm_instance._publish_oauth_prompt(evt_data)
|
||||||
|
)
|
||||||
|
|
||||||
elif evt_type == "completed":
|
elif evt_type == "completed":
|
||||||
logger.info("Voice inject stream completed")
|
logger.info("Voice inject stream completed")
|
||||||
return
|
return
|
||||||
|
|
@ -775,6 +802,11 @@ class AgentServiceLLMStream(llm.LLMStream):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
elif evt_type == "oauth_prompt":
|
||||||
|
asyncio.create_task(
|
||||||
|
self._llm_instance._publish_oauth_prompt(evt_data)
|
||||||
|
)
|
||||||
|
|
||||||
elif evt_type == "completed":
|
elif evt_type == "completed":
|
||||||
logger.info("Agent stream completed")
|
logger.info("Agent stream completed")
|
||||||
return
|
return
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue