hailin
/
it0
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: redesign voice call mixed-mode input with dual-layout architecture 

Problem:
- Text input area caused BOTTOM OVERFLOWED BY 135 PIXELS when keyboard opened
- Input bar overlapped with call control buttons
- Sent messages were not displayed on screen (only SnackBar feedback)

Solution — split into two distinct layouts:

1. Call Mode (default):
   - Full-screen call UI: avatar, waveform, duration, large control buttons
   - Keyboard button in controls toggles to chat mode
   - No text input elements — clean voice-only interface

2. Chat Mode (tap keyboard button):
   - Compact call header: green status dot + "iAgent" + duration + inline
     mute/end/speaker/collapse controls
   - Scrollable message list (Expanded widget — properly handles keyboard)
   - User messages: right-aligned blue bubbles with attachment thumbnails
   - Agent responses: left-aligned gray bubbles with robot avatar
   - Input bar at bottom: attachment picker + text field + send button

Message display:
- User-sent text/attachments tracked in _messages list, shown as bubbles
- Agent responses sent back via LiveKit data channel (topic='text_reply')
  from voice-agent → Flutter, displayed as assistant bubbles
- Auto-scroll to latest message

Voice-agent change (agent.py):
- After session.say(response), publish response text back to Flutter via
  ctx.room.local_participant.publish_data() with topic='text_reply'
- Flutter listens for DataReceivedEvent to display agent responses

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-03-02 06:11:07 -08:00
parent ce63ece340
commit 63b986fced
2 changed files with 370 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

455
it0_app/lib/features/agent_call/presentation/pages/agent_call_page.dart
View File

@ -59,6 +59,8 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
bool _isInputExpanded = false;
bool _isSending = false;
final List<ChatAttachment> _pendingAttachments = [];
final List<_CallMessage> _messages = [];
final ScrollController _scrollController = ScrollController();
@override
void initState() {
@ -138,6 +140,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
if (_phase != _CallPhase.ended && !_userEndedCall) {
_onCallEnded();
}
})
..on<DataReceivedEvent>((event) {
if (event.topic == 'text_reply') {
try {
final payload = jsonDecode(utf8.decode(event.data));
final text = payload['text'] as String? ?? '';
if (text.isNotEmpty && mounted) {
setState(() {
_messages.add(_CallMessage(
text: text,
isUser: false,
timestamp: DateTime.now(),
));
});
_scrollToBottom();
}
} catch (_) {}
}
});
// 4. Connect to LiveKit room (with timeout)
@ -258,6 +278,18 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
setState(() {});
}
void _scrollToBottom() {
WidgetsBinding.instance.addPostFrameCallback((_) {
if (_scrollController.hasClients) {
_scrollController.animateTo(
_scrollController.position.maxScrollExtent,
duration: const Duration(milliseconds: 200),
curve: Curves.easeOut,
);
}
});
}
// ---------------------------------------------------------------------------
// Text input + attachments (mixed-mode during voice call)
// ---------------------------------------------------------------------------
@ -297,17 +329,20 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
topic: 'text_inject',
);
// Track message for display
setState(() {
_messages.add(_CallMessage(
text: text,
isUser: true,
timestamp: DateTime.now(),
attachments: _pendingAttachments.isNotEmpty
? List<ChatAttachment>.from(_pendingAttachments)
: null,
));
_pendingAttachments.clear();
});
_textController.clear();
setState(() => _pendingAttachments.clear());
if (mounted) {
ScaffoldMessenger.of(context).showSnackBar(
const SnackBar(
content: Text('已发送'),
duration: Duration(seconds: 1),
),
);
}
_scrollToBottom();
} catch (e) {
if (mounted) {
ScaffoldMessenger.of(context).showSnackBar(
@ -499,103 +534,306 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
return Scaffold(
backgroundColor: AppColors.background,
body: SafeArea(
child: Center(
child: Column(
children: [
const Spacer(flex: 2),
_buildAvatar(),
const SizedBox(height: 24),
Text(
_statusText,
textAlign: TextAlign.center,
style: const TextStyle(
fontSize: 24, fontWeight: FontWeight.bold),
),
const SizedBox(height: 8),
Text(
_subtitleText,
textAlign: TextAlign.center,
style: const TextStyle(
color: AppColors.textSecondary, fontSize: 15),
),
// Inline error message
if (_errorMessage != null) ...[
const SizedBox(height: 16),
Padding(
padding: const EdgeInsets.symmetric(horizontal: 32),
child: Container(
padding: const EdgeInsets.symmetric(
horizontal: 16, vertical: 12),
decoration: BoxDecoration(
color: AppColors.error.withOpacity(0.1),
borderRadius: BorderRadius.circular(12),
border: Border.all(
color: AppColors.error.withOpacity(0.3)),
),
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
const Icon(Icons.error_outline,
size: 18, color: AppColors.error),
const SizedBox(width: 8),
Flexible(
child: Text(
_errorMessage!,
textAlign: TextAlign.center,
style: const TextStyle(
color: AppColors.error,
fontSize: 13,
),
),
),
],
),
),
child: _isInputExpanded && _phase == _CallPhase.active
? _buildChatModeLayout()
: _buildCallModeLayout(),
),
);
}
/// Full-screen call layout (avatar, waveform, big controls).
Widget _buildCallModeLayout() {
return Center(
child: Column(
children: [
const Spacer(flex: 2),
_buildAvatar(),
const SizedBox(height: 24),
Text(
_statusText,
textAlign: TextAlign.center,
style: const TextStyle(
fontSize: 24, fontWeight: FontWeight.bold),
),
const SizedBox(height: 8),
Text(
_subtitleText,
textAlign: TextAlign.center,
style: const TextStyle(
color: AppColors.textSecondary, fontSize: 15),
),
if (_errorMessage != null) ...[
const SizedBox(height: 16),
Padding(
padding: const EdgeInsets.symmetric(horizontal: 32),
child: Container(
padding: const EdgeInsets.symmetric(
horizontal: 16, vertical: 12),
decoration: BoxDecoration(
color: AppColors.error.withOpacity(0.1),
borderRadius: BorderRadius.circular(12),
border: Border.all(
color: AppColors.error.withOpacity(0.3)),
),
],
const SizedBox(height: 32),
if (_phase == _CallPhase.active)
Text(
_durationLabel,
style: const TextStyle(
fontSize: 40,
fontWeight: FontWeight.w300,
color: AppColors.textPrimary,
letterSpacing: 4,
),
),
const SizedBox(height: 24),
if (_phase == _CallPhase.active && _isReconnecting)
Padding(
padding: const EdgeInsets.only(bottom: 12),
child: Row(
mainAxisAlignment: MainAxisAlignment.center,
children: [
const SizedBox(
width: 14,
height: 14,
child: CircularProgressIndicator(strokeWidth: 2),
),
const SizedBox(width: 8),
Text(
'网络重连中${_reconnectAttempt > 0 ? ' ($_reconnectAttempt)' : ''}...',
style: TextStyle(
color: AppColors.warning,
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
const Icon(Icons.error_outline,
size: 18, color: AppColors.error),
const SizedBox(width: 8),
Flexible(
child: Text(
_errorMessage!,
textAlign: TextAlign.center,
style: const TextStyle(
color: AppColors.error,
fontSize: 13,
),
),
),
],
),
),
),
],
const SizedBox(height: 32),
if (_phase == _CallPhase.active)
Text(
_durationLabel,
style: const TextStyle(
fontSize: 40,
fontWeight: FontWeight.w300,
color: AppColors.textPrimary,
letterSpacing: 4,
),
),
const SizedBox(height: 24),
if (_phase == _CallPhase.active && _isReconnecting)
Padding(
padding: const EdgeInsets.only(bottom: 12),
child: Row(
mainAxisAlignment: MainAxisAlignment.center,
children: [
const SizedBox(
width: 14,
height: 14,
child: CircularProgressIndicator(strokeWidth: 2),
),
const SizedBox(width: 8),
Text(
'网络重连中${_reconnectAttempt > 0 ? ' ($_reconnectAttempt)' : ''}...',
style: TextStyle(
color: AppColors.warning,
fontSize: 13,
),
),
],
),
),
if (_phase == _CallPhase.active) _buildWaveform(),
const Spacer(flex: 3),
_buildControls(),
const SizedBox(height: 48),
],
),
);
}
/// Chat-mode layout: compact header + message list + input bar.
Widget _buildChatModeLayout() {
return Column(
children: [
_buildCompactCallHeader(),
const Divider(height: 1, color: AppColors.surfaceLight),
Expanded(
child: _messages.isEmpty
? Center(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Icon(Icons.chat_bubble_outline,
size: 48, color: AppColors.textMuted.withOpacity(0.5)),
const SizedBox(height: 12),
const Text('输入文字或附件发送给 Agent',
style: TextStyle(color: AppColors.textSecondary, fontSize: 14)),
const SizedBox(height: 4),
const Text('Agent 将语音回复并显示在此',
style: TextStyle(color: AppColors.textMuted, fontSize: 12)),
],
),
)
: ListView.builder(
controller: _scrollController,
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
itemCount: _messages.length,
itemBuilder: (ctx, i) => _buildMessageBubble(_messages[i]),
),
if (_phase == _CallPhase.active) _buildWaveform(),
Spacer(flex: _isInputExpanded ? 1 : 3),
if (_isInputExpanded && _phase == _CallPhase.active)
_buildTextInputArea(),
_buildControls(),
SizedBox(height: _isInputExpanded ? 12 : 48),
],
),
),
if (_pendingAttachments.isNotEmpty)
Padding(
padding: const EdgeInsets.symmetric(horizontal: 16),
child: _buildAttachmentPreview(),
),
_buildTextInputArea(),
const SizedBox(height: 4),
],
);
}
/// Compact header for chat mode: green dot + title + duration + inline controls.
Widget _buildCompactCallHeader() {
return Container(
padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
child: Row(
children: [
Container(
width: 10,
height: 10,
margin: const EdgeInsets.only(left: 8),
decoration: BoxDecoration(
shape: BoxShape.circle,
color: _isReconnecting ? AppColors.warning : AppColors.success,
),
),
const SizedBox(width: 8),
const Text('iAgent',
style: TextStyle(fontWeight: FontWeight.w600, fontSize: 15)),
const SizedBox(width: 8),
Text(_durationLabel,
style: const TextStyle(color: AppColors.textSecondary, fontSize: 13)),
if (_isReconnecting) ...[
const SizedBox(width: 6),
const SizedBox(
width: 12, height: 12,
child: CircularProgressIndicator(strokeWidth: 1.5),
),
],
const Spacer(),
IconButton(
icon: Icon(_isMuted ? Icons.mic_off : Icons.mic, size: 20,
color: _isMuted ? AppColors.error : AppColors.textSecondary),
onPressed: _toggleMute,
visualDensity: VisualDensity.compact,
tooltip: _isMuted ? '取消静音' : '静音',
),
IconButton(
icon: const Icon(Icons.call_end, size: 20, color: AppColors.error),
onPressed: _endCall,
visualDensity: VisualDensity.compact,
tooltip: '挂断',
),
IconButton(
icon: Icon(_isSpeaker ? Icons.volume_up : Icons.hearing, size: 20,
color: AppColors.textSecondary),
onPressed: _toggleSpeaker,
visualDensity: VisualDensity.compact,
tooltip: _isSpeaker ? '听筒' : '扬声器',
),
IconButton(
icon: const Icon(Icons.keyboard_hide, size: 20,
color: AppColors.textSecondary),
onPressed: () => setState(() => _isInputExpanded = false),
visualDensity: VisualDensity.compact,
tooltip: '收起',
),
],
),
);
}
Widget _buildMessageBubble(_CallMessage msg) {
final isUser = msg.isUser;
return Padding(
padding: const EdgeInsets.only(bottom: 12),
child: Row(
mainAxisAlignment:
isUser ? MainAxisAlignment.end : MainAxisAlignment.start,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
if (!isUser) ...[
Container(
width: 28,
height: 28,
decoration: BoxDecoration(
shape: BoxShape.circle,
color: AppColors.primary.withOpacity(0.15),
),
child: const Icon(Icons.smart_toy, size: 16, color: AppColors.primary),
),
const SizedBox(width: 8),
],
Flexible(
child: Container(
padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 10),
decoration: BoxDecoration(
color: isUser
? AppColors.primary.withOpacity(0.15)
: AppColors.surfaceLight,
borderRadius: BorderRadius.only(
topLeft: const Radius.circular(16),
topRight: const Radius.circular(16),
bottomLeft: Radius.circular(isUser ? 16 : 4),
bottomRight: Radius.circular(isUser ? 4 : 16),
),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
if (msg.attachments != null && msg.attachments!.isNotEmpty) ...[
Wrap(
spacing: 4,
runSpacing: 4,
children: msg.attachments!.map((att) {
final isImage = att.mediaType.startsWith('image/');
if (isImage) {
return ClipRRect(
borderRadius: BorderRadius.circular(8),
child: Image.memory(
base64Decode(att.base64Data),
width: 100,
height: 100,
fit: BoxFit.cover,
cacheWidth: 200,
cacheHeight: 200,
),
);
}
return Container(
padding: const EdgeInsets.symmetric(
horizontal: 8, vertical: 4),
decoration: BoxDecoration(
color: AppColors.surface,
borderRadius: BorderRadius.circular(6),
),
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
const Icon(Icons.description,
size: 16, color: AppColors.textSecondary),
const SizedBox(width: 4),
Text(att.fileName ?? 'file',
style: const TextStyle(fontSize: 12)),
],
),
);
}).toList(),
),
if (msg.text.isNotEmpty) const SizedBox(height: 6),
],
if (msg.text.isNotEmpty)
Text(
msg.text,
style: const TextStyle(
color: AppColors.textPrimary,
fontSize: 14,
),
),
],
),
),
),
if (isUser) const SizedBox(width: 8),
],
),
);
}
@ -898,6 +1136,7 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
void dispose() {
_userEndedCall = true;
_textController.dispose();
_scrollController.dispose();
_durationTimer?.cancel();
_waveTimer?.cancel();
_waveController.dispose();
@ -909,6 +1148,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
}
}
// ---------------------------------------------------------------------------
// Simple message model for call-page chat display
// ---------------------------------------------------------------------------
class _CallMessage {
final String text;
final bool isUser;
final DateTime timestamp;
final List<ChatAttachment>? attachments;
_CallMessage({
required this.text,
required this.isUser,
required this.timestamp,
this.attachments,
});
}
// ---------------------------------------------------------------------------
// Small helper widget for mute / speaker buttons
// ---------------------------------------------------------------------------

14
packages/services/voice-agent/src/agent.py
View File

@ -320,6 +320,20 @@ async def entrypoint(ctx: JobContext) -> None:
if response:
logger.info("inject response: %s", response[:100])
session.say(response)
# Send response text back to Flutter for display
try:
reply_payload = json.dumps({
"type": "text_reply",
"text": response,
}).encode("utf-8")
await ctx.room.local_participant.publish_data(
reply_payload,
reliable=True,
topic="text_reply",
)
except Exception as pub_err:
logger.warning("Failed to publish text_reply: %s", pub_err)
else:
logger.warning("inject_text_message returned empty response")