fix: redesign voice call mixed-mode input with dual-layout architecture

Problem:
- Text input area caused BOTTOM OVERFLOWED BY 135 PIXELS when keyboard opened
- Input bar overlapped with call control buttons
- Sent messages were not displayed on screen (only SnackBar feedback)

Solution — split into two distinct layouts:

1. Call Mode (default):
   - Full-screen call UI: avatar, waveform, duration, large control buttons
   - Keyboard button in controls toggles to chat mode
   - No text input elements — clean voice-only interface

2. Chat Mode (tap keyboard button):
   - Compact call header: green status dot + "iAgent" + duration + inline
     mute/end/speaker/collapse controls
   - Scrollable message list (Expanded widget — properly handles keyboard)
   - User messages: right-aligned blue bubbles with attachment thumbnails
   - Agent responses: left-aligned gray bubbles with robot avatar
   - Input bar at bottom: attachment picker + text field + send button

Message display:
- User-sent text/attachments tracked in _messages list, shown as bubbles
- Agent responses sent back via LiveKit data channel (topic='text_reply')
  from voice-agent → Flutter, displayed as assistant bubbles
- Auto-scroll to latest message

Voice-agent change (agent.py):
- After session.say(response), publish response text back to Flutter via
  ctx.room.local_participant.publish_data() with topic='text_reply'
- Flutter listens for DataReceivedEvent to display agent responses

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-03-02 06:11:07 -08:00
parent ce63ece340
commit 63b986fced
2 changed files with 370 additions and 99 deletions

View File

@ -59,6 +59,8 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
bool _isInputExpanded = false; bool _isInputExpanded = false;
bool _isSending = false; bool _isSending = false;
final List<ChatAttachment> _pendingAttachments = []; final List<ChatAttachment> _pendingAttachments = [];
final List<_CallMessage> _messages = [];
final ScrollController _scrollController = ScrollController();
@override @override
void initState() { void initState() {
@ -138,6 +140,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
if (_phase != _CallPhase.ended && !_userEndedCall) { if (_phase != _CallPhase.ended && !_userEndedCall) {
_onCallEnded(); _onCallEnded();
} }
})
..on<DataReceivedEvent>((event) {
if (event.topic == 'text_reply') {
try {
final payload = jsonDecode(utf8.decode(event.data));
final text = payload['text'] as String? ?? '';
if (text.isNotEmpty && mounted) {
setState(() {
_messages.add(_CallMessage(
text: text,
isUser: false,
timestamp: DateTime.now(),
));
});
_scrollToBottom();
}
} catch (_) {}
}
}); });
// 4. Connect to LiveKit room (with timeout) // 4. Connect to LiveKit room (with timeout)
@ -258,6 +278,18 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
setState(() {}); setState(() {});
} }
void _scrollToBottom() {
WidgetsBinding.instance.addPostFrameCallback((_) {
if (_scrollController.hasClients) {
_scrollController.animateTo(
_scrollController.position.maxScrollExtent,
duration: const Duration(milliseconds: 200),
curve: Curves.easeOut,
);
}
});
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Text input + attachments (mixed-mode during voice call) // Text input + attachments (mixed-mode during voice call)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -297,17 +329,20 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
topic: 'text_inject', topic: 'text_inject',
); );
// Track message for display
setState(() {
_messages.add(_CallMessage(
text: text,
isUser: true,
timestamp: DateTime.now(),
attachments: _pendingAttachments.isNotEmpty
? List<ChatAttachment>.from(_pendingAttachments)
: null,
));
_pendingAttachments.clear();
});
_textController.clear(); _textController.clear();
setState(() => _pendingAttachments.clear()); _scrollToBottom();
if (mounted) {
ScaffoldMessenger.of(context).showSnackBar(
const SnackBar(
content: Text('已发送'),
duration: Duration(seconds: 1),
),
);
}
} catch (e) { } catch (e) {
if (mounted) { if (mounted) {
ScaffoldMessenger.of(context).showSnackBar( ScaffoldMessenger.of(context).showSnackBar(
@ -499,7 +534,16 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
return Scaffold( return Scaffold(
backgroundColor: AppColors.background, backgroundColor: AppColors.background,
body: SafeArea( body: SafeArea(
child: Center( child: _isInputExpanded && _phase == _CallPhase.active
? _buildChatModeLayout()
: _buildCallModeLayout(),
),
);
}
/// Full-screen call layout (avatar, waveform, big controls).
Widget _buildCallModeLayout() {
return Center(
child: Column( child: Column(
children: [ children: [
const Spacer(flex: 2), const Spacer(flex: 2),
@ -518,7 +562,6 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
style: const TextStyle( style: const TextStyle(
color: AppColors.textSecondary, fontSize: 15), color: AppColors.textSecondary, fontSize: 15),
), ),
// Inline error message
if (_errorMessage != null) ...[ if (_errorMessage != null) ...[
const SizedBox(height: 16), const SizedBox(height: 16),
Padding( Padding(
@ -588,15 +631,210 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
), ),
), ),
if (_phase == _CallPhase.active) _buildWaveform(), if (_phase == _CallPhase.active) _buildWaveform(),
Spacer(flex: _isInputExpanded ? 1 : 3), const Spacer(flex: 3),
if (_isInputExpanded && _phase == _CallPhase.active)
_buildTextInputArea(),
_buildControls(), _buildControls(),
SizedBox(height: _isInputExpanded ? 12 : 48), const SizedBox(height: 48),
],
),
);
}
/// Chat-mode layout: compact header + message list + input bar.
Widget _buildChatModeLayout() {
return Column(
children: [
_buildCompactCallHeader(),
const Divider(height: 1, color: AppColors.surfaceLight),
Expanded(
child: _messages.isEmpty
? Center(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Icon(Icons.chat_bubble_outline,
size: 48, color: AppColors.textMuted.withOpacity(0.5)),
const SizedBox(height: 12),
const Text('输入文字或附件发送给 Agent',
style: TextStyle(color: AppColors.textSecondary, fontSize: 14)),
const SizedBox(height: 4),
const Text('Agent 将语音回复并显示在此',
style: TextStyle(color: AppColors.textMuted, fontSize: 12)),
],
),
)
: ListView.builder(
controller: _scrollController,
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
itemCount: _messages.length,
itemBuilder: (ctx, i) => _buildMessageBubble(_messages[i]),
),
),
if (_pendingAttachments.isNotEmpty)
Padding(
padding: const EdgeInsets.symmetric(horizontal: 16),
child: _buildAttachmentPreview(),
),
_buildTextInputArea(),
const SizedBox(height: 4),
],
);
}
/// Compact header for chat mode: green dot + title + duration + inline controls.
Widget _buildCompactCallHeader() {
return Container(
padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
child: Row(
children: [
Container(
width: 10,
height: 10,
margin: const EdgeInsets.only(left: 8),
decoration: BoxDecoration(
shape: BoxShape.circle,
color: _isReconnecting ? AppColors.warning : AppColors.success,
),
),
const SizedBox(width: 8),
const Text('iAgent',
style: TextStyle(fontWeight: FontWeight.w600, fontSize: 15)),
const SizedBox(width: 8),
Text(_durationLabel,
style: const TextStyle(color: AppColors.textSecondary, fontSize: 13)),
if (_isReconnecting) ...[
const SizedBox(width: 6),
const SizedBox(
width: 12, height: 12,
child: CircularProgressIndicator(strokeWidth: 1.5),
),
],
const Spacer(),
IconButton(
icon: Icon(_isMuted ? Icons.mic_off : Icons.mic, size: 20,
color: _isMuted ? AppColors.error : AppColors.textSecondary),
onPressed: _toggleMute,
visualDensity: VisualDensity.compact,
tooltip: _isMuted ? '取消静音' : '静音',
),
IconButton(
icon: const Icon(Icons.call_end, size: 20, color: AppColors.error),
onPressed: _endCall,
visualDensity: VisualDensity.compact,
tooltip: '挂断',
),
IconButton(
icon: Icon(_isSpeaker ? Icons.volume_up : Icons.hearing, size: 20,
color: AppColors.textSecondary),
onPressed: _toggleSpeaker,
visualDensity: VisualDensity.compact,
tooltip: _isSpeaker ? '听筒' : '扬声器',
),
IconButton(
icon: const Icon(Icons.keyboard_hide, size: 20,
color: AppColors.textSecondary),
onPressed: () => setState(() => _isInputExpanded = false),
visualDensity: VisualDensity.compact,
tooltip: '收起',
),
],
),
);
}
Widget _buildMessageBubble(_CallMessage msg) {
final isUser = msg.isUser;
return Padding(
padding: const EdgeInsets.only(bottom: 12),
child: Row(
mainAxisAlignment:
isUser ? MainAxisAlignment.end : MainAxisAlignment.start,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
if (!isUser) ...[
Container(
width: 28,
height: 28,
decoration: BoxDecoration(
shape: BoxShape.circle,
color: AppColors.primary.withOpacity(0.15),
),
child: const Icon(Icons.smart_toy, size: 16, color: AppColors.primary),
),
const SizedBox(width: 8),
],
Flexible(
child: Container(
padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 10),
decoration: BoxDecoration(
color: isUser
? AppColors.primary.withOpacity(0.15)
: AppColors.surfaceLight,
borderRadius: BorderRadius.only(
topLeft: const Radius.circular(16),
topRight: const Radius.circular(16),
bottomLeft: Radius.circular(isUser ? 16 : 4),
bottomRight: Radius.circular(isUser ? 4 : 16),
),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
if (msg.attachments != null && msg.attachments!.isNotEmpty) ...[
Wrap(
spacing: 4,
runSpacing: 4,
children: msg.attachments!.map((att) {
final isImage = att.mediaType.startsWith('image/');
if (isImage) {
return ClipRRect(
borderRadius: BorderRadius.circular(8),
child: Image.memory(
base64Decode(att.base64Data),
width: 100,
height: 100,
fit: BoxFit.cover,
cacheWidth: 200,
cacheHeight: 200,
),
);
}
return Container(
padding: const EdgeInsets.symmetric(
horizontal: 8, vertical: 4),
decoration: BoxDecoration(
color: AppColors.surface,
borderRadius: BorderRadius.circular(6),
),
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
const Icon(Icons.description,
size: 16, color: AppColors.textSecondary),
const SizedBox(width: 4),
Text(att.fileName ?? 'file',
style: const TextStyle(fontSize: 12)),
],
),
);
}).toList(),
),
if (msg.text.isNotEmpty) const SizedBox(height: 6),
],
if (msg.text.isNotEmpty)
Text(
msg.text,
style: const TextStyle(
color: AppColors.textPrimary,
fontSize: 14,
),
),
], ],
), ),
), ),
), ),
if (isUser) const SizedBox(width: 8),
],
),
); );
} }
@ -898,6 +1136,7 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
void dispose() { void dispose() {
_userEndedCall = true; _userEndedCall = true;
_textController.dispose(); _textController.dispose();
_scrollController.dispose();
_durationTimer?.cancel(); _durationTimer?.cancel();
_waveTimer?.cancel(); _waveTimer?.cancel();
_waveController.dispose(); _waveController.dispose();
@ -909,6 +1148,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
} }
} }
// ---------------------------------------------------------------------------
// Simple message model for call-page chat display
// ---------------------------------------------------------------------------
class _CallMessage {
final String text;
final bool isUser;
final DateTime timestamp;
final List<ChatAttachment>? attachments;
_CallMessage({
required this.text,
required this.isUser,
required this.timestamp,
this.attachments,
});
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Small helper widget for mute / speaker buttons // Small helper widget for mute / speaker buttons
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -320,6 +320,20 @@ async def entrypoint(ctx: JobContext) -> None:
if response: if response:
logger.info("inject response: %s", response[:100]) logger.info("inject response: %s", response[:100])
session.say(response) session.say(response)
# Send response text back to Flutter for display
try:
reply_payload = json.dumps({
"type": "text_reply",
"text": response,
}).encode("utf-8")
await ctx.room.local_participant.publish_data(
reply_payload,
reliable=True,
topic="text_reply",
)
except Exception as pub_err:
logger.warning("Failed to publish text_reply: %s", pub_err)
else: else:
logger.warning("inject_text_message returned empty response") logger.warning("inject_text_message returned empty response")