fix: redesign voice call mixed-mode input with dual-layout architecture
Problem:
- Text input area caused BOTTOM OVERFLOWED BY 135 PIXELS when keyboard opened
- Input bar overlapped with call control buttons
- Sent messages were not displayed on screen (only SnackBar feedback)
Solution — split into two distinct layouts:
1. Call Mode (default):
- Full-screen call UI: avatar, waveform, duration, large control buttons
- Keyboard button in controls toggles to chat mode
- No text input elements — clean voice-only interface
2. Chat Mode (tap keyboard button):
- Compact call header: green status dot + "iAgent" + duration + inline
mute/end/speaker/collapse controls
- Scrollable message list (Expanded widget — properly handles keyboard)
- User messages: right-aligned blue bubbles with attachment thumbnails
- Agent responses: left-aligned gray bubbles with robot avatar
- Input bar at bottom: attachment picker + text field + send button
Message display:
- User-sent text/attachments tracked in _messages list, shown as bubbles
- Agent responses sent back via LiveKit data channel (topic='text_reply')
from voice-agent → Flutter, displayed as assistant bubbles
- Auto-scroll to latest message
Voice-agent change (agent.py):
- After session.say(response), publish response text back to Flutter via
ctx.room.local_participant.publish_data() with topic='text_reply'
- Flutter listens for DataReceivedEvent to display agent responses
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ce63ece340
commit
63b986fced
|
|
@ -59,6 +59,8 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
bool _isInputExpanded = false;
|
||||
bool _isSending = false;
|
||||
final List<ChatAttachment> _pendingAttachments = [];
|
||||
final List<_CallMessage> _messages = [];
|
||||
final ScrollController _scrollController = ScrollController();
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
|
|
@ -138,6 +140,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
if (_phase != _CallPhase.ended && !_userEndedCall) {
|
||||
_onCallEnded();
|
||||
}
|
||||
})
|
||||
..on<DataReceivedEvent>((event) {
|
||||
if (event.topic == 'text_reply') {
|
||||
try {
|
||||
final payload = jsonDecode(utf8.decode(event.data));
|
||||
final text = payload['text'] as String? ?? '';
|
||||
if (text.isNotEmpty && mounted) {
|
||||
setState(() {
|
||||
_messages.add(_CallMessage(
|
||||
text: text,
|
||||
isUser: false,
|
||||
timestamp: DateTime.now(),
|
||||
));
|
||||
});
|
||||
_scrollToBottom();
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
});
|
||||
|
||||
// 4. Connect to LiveKit room (with timeout)
|
||||
|
|
@ -258,6 +278,18 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
setState(() {});
|
||||
}
|
||||
|
||||
void _scrollToBottom() {
|
||||
WidgetsBinding.instance.addPostFrameCallback((_) {
|
||||
if (_scrollController.hasClients) {
|
||||
_scrollController.animateTo(
|
||||
_scrollController.position.maxScrollExtent,
|
||||
duration: const Duration(milliseconds: 200),
|
||||
curve: Curves.easeOut,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Text input + attachments (mixed-mode during voice call)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -297,17 +329,20 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
topic: 'text_inject',
|
||||
);
|
||||
|
||||
// Track message for display
|
||||
setState(() {
|
||||
_messages.add(_CallMessage(
|
||||
text: text,
|
||||
isUser: true,
|
||||
timestamp: DateTime.now(),
|
||||
attachments: _pendingAttachments.isNotEmpty
|
||||
? List<ChatAttachment>.from(_pendingAttachments)
|
||||
: null,
|
||||
));
|
||||
_pendingAttachments.clear();
|
||||
});
|
||||
_textController.clear();
|
||||
setState(() => _pendingAttachments.clear());
|
||||
|
||||
if (mounted) {
|
||||
ScaffoldMessenger.of(context).showSnackBar(
|
||||
const SnackBar(
|
||||
content: Text('已发送'),
|
||||
duration: Duration(seconds: 1),
|
||||
),
|
||||
);
|
||||
}
|
||||
_scrollToBottom();
|
||||
} catch (e) {
|
||||
if (mounted) {
|
||||
ScaffoldMessenger.of(context).showSnackBar(
|
||||
|
|
@ -499,7 +534,16 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
return Scaffold(
|
||||
backgroundColor: AppColors.background,
|
||||
body: SafeArea(
|
||||
child: Center(
|
||||
child: _isInputExpanded && _phase == _CallPhase.active
|
||||
? _buildChatModeLayout()
|
||||
: _buildCallModeLayout(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// Full-screen call layout (avatar, waveform, big controls).
|
||||
Widget _buildCallModeLayout() {
|
||||
return Center(
|
||||
child: Column(
|
||||
children: [
|
||||
const Spacer(flex: 2),
|
||||
|
|
@ -518,7 +562,6 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
style: const TextStyle(
|
||||
color: AppColors.textSecondary, fontSize: 15),
|
||||
),
|
||||
// Inline error message
|
||||
if (_errorMessage != null) ...[
|
||||
const SizedBox(height: 16),
|
||||
Padding(
|
||||
|
|
@ -588,15 +631,210 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
),
|
||||
),
|
||||
if (_phase == _CallPhase.active) _buildWaveform(),
|
||||
Spacer(flex: _isInputExpanded ? 1 : 3),
|
||||
if (_isInputExpanded && _phase == _CallPhase.active)
|
||||
_buildTextInputArea(),
|
||||
const Spacer(flex: 3),
|
||||
_buildControls(),
|
||||
SizedBox(height: _isInputExpanded ? 12 : 48),
|
||||
const SizedBox(height: 48),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// Chat-mode layout: compact header + message list + input bar.
|
||||
Widget _buildChatModeLayout() {
|
||||
return Column(
|
||||
children: [
|
||||
_buildCompactCallHeader(),
|
||||
const Divider(height: 1, color: AppColors.surfaceLight),
|
||||
Expanded(
|
||||
child: _messages.isEmpty
|
||||
? Center(
|
||||
child: Column(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
Icon(Icons.chat_bubble_outline,
|
||||
size: 48, color: AppColors.textMuted.withOpacity(0.5)),
|
||||
const SizedBox(height: 12),
|
||||
const Text('输入文字或附件发送给 Agent',
|
||||
style: TextStyle(color: AppColors.textSecondary, fontSize: 14)),
|
||||
const SizedBox(height: 4),
|
||||
const Text('Agent 将语音回复并显示在此',
|
||||
style: TextStyle(color: AppColors.textMuted, fontSize: 12)),
|
||||
],
|
||||
),
|
||||
)
|
||||
: ListView.builder(
|
||||
controller: _scrollController,
|
||||
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
|
||||
itemCount: _messages.length,
|
||||
itemBuilder: (ctx, i) => _buildMessageBubble(_messages[i]),
|
||||
),
|
||||
),
|
||||
if (_pendingAttachments.isNotEmpty)
|
||||
Padding(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 16),
|
||||
child: _buildAttachmentPreview(),
|
||||
),
|
||||
_buildTextInputArea(),
|
||||
const SizedBox(height: 4),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/// Compact header for chat mode: green dot + title + duration + inline controls.
|
||||
Widget _buildCompactCallHeader() {
|
||||
return Container(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
|
||||
child: Row(
|
||||
children: [
|
||||
Container(
|
||||
width: 10,
|
||||
height: 10,
|
||||
margin: const EdgeInsets.only(left: 8),
|
||||
decoration: BoxDecoration(
|
||||
shape: BoxShape.circle,
|
||||
color: _isReconnecting ? AppColors.warning : AppColors.success,
|
||||
),
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
const Text('iAgent',
|
||||
style: TextStyle(fontWeight: FontWeight.w600, fontSize: 15)),
|
||||
const SizedBox(width: 8),
|
||||
Text(_durationLabel,
|
||||
style: const TextStyle(color: AppColors.textSecondary, fontSize: 13)),
|
||||
if (_isReconnecting) ...[
|
||||
const SizedBox(width: 6),
|
||||
const SizedBox(
|
||||
width: 12, height: 12,
|
||||
child: CircularProgressIndicator(strokeWidth: 1.5),
|
||||
),
|
||||
],
|
||||
const Spacer(),
|
||||
IconButton(
|
||||
icon: Icon(_isMuted ? Icons.mic_off : Icons.mic, size: 20,
|
||||
color: _isMuted ? AppColors.error : AppColors.textSecondary),
|
||||
onPressed: _toggleMute,
|
||||
visualDensity: VisualDensity.compact,
|
||||
tooltip: _isMuted ? '取消静音' : '静音',
|
||||
),
|
||||
IconButton(
|
||||
icon: const Icon(Icons.call_end, size: 20, color: AppColors.error),
|
||||
onPressed: _endCall,
|
||||
visualDensity: VisualDensity.compact,
|
||||
tooltip: '挂断',
|
||||
),
|
||||
IconButton(
|
||||
icon: Icon(_isSpeaker ? Icons.volume_up : Icons.hearing, size: 20,
|
||||
color: AppColors.textSecondary),
|
||||
onPressed: _toggleSpeaker,
|
||||
visualDensity: VisualDensity.compact,
|
||||
tooltip: _isSpeaker ? '听筒' : '扬声器',
|
||||
),
|
||||
IconButton(
|
||||
icon: const Icon(Icons.keyboard_hide, size: 20,
|
||||
color: AppColors.textSecondary),
|
||||
onPressed: () => setState(() => _isInputExpanded = false),
|
||||
visualDensity: VisualDensity.compact,
|
||||
tooltip: '收起',
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Widget _buildMessageBubble(_CallMessage msg) {
|
||||
final isUser = msg.isUser;
|
||||
return Padding(
|
||||
padding: const EdgeInsets.only(bottom: 12),
|
||||
child: Row(
|
||||
mainAxisAlignment:
|
||||
isUser ? MainAxisAlignment.end : MainAxisAlignment.start,
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
if (!isUser) ...[
|
||||
Container(
|
||||
width: 28,
|
||||
height: 28,
|
||||
decoration: BoxDecoration(
|
||||
shape: BoxShape.circle,
|
||||
color: AppColors.primary.withOpacity(0.15),
|
||||
),
|
||||
child: const Icon(Icons.smart_toy, size: 16, color: AppColors.primary),
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
],
|
||||
Flexible(
|
||||
child: Container(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 10),
|
||||
decoration: BoxDecoration(
|
||||
color: isUser
|
||||
? AppColors.primary.withOpacity(0.15)
|
||||
: AppColors.surfaceLight,
|
||||
borderRadius: BorderRadius.only(
|
||||
topLeft: const Radius.circular(16),
|
||||
topRight: const Radius.circular(16),
|
||||
bottomLeft: Radius.circular(isUser ? 16 : 4),
|
||||
bottomRight: Radius.circular(isUser ? 4 : 16),
|
||||
),
|
||||
),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
if (msg.attachments != null && msg.attachments!.isNotEmpty) ...[
|
||||
Wrap(
|
||||
spacing: 4,
|
||||
runSpacing: 4,
|
||||
children: msg.attachments!.map((att) {
|
||||
final isImage = att.mediaType.startsWith('image/');
|
||||
if (isImage) {
|
||||
return ClipRRect(
|
||||
borderRadius: BorderRadius.circular(8),
|
||||
child: Image.memory(
|
||||
base64Decode(att.base64Data),
|
||||
width: 100,
|
||||
height: 100,
|
||||
fit: BoxFit.cover,
|
||||
cacheWidth: 200,
|
||||
cacheHeight: 200,
|
||||
),
|
||||
);
|
||||
}
|
||||
return Container(
|
||||
padding: const EdgeInsets.symmetric(
|
||||
horizontal: 8, vertical: 4),
|
||||
decoration: BoxDecoration(
|
||||
color: AppColors.surface,
|
||||
borderRadius: BorderRadius.circular(6),
|
||||
),
|
||||
child: Row(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
const Icon(Icons.description,
|
||||
size: 16, color: AppColors.textSecondary),
|
||||
const SizedBox(width: 4),
|
||||
Text(att.fileName ?? 'file',
|
||||
style: const TextStyle(fontSize: 12)),
|
||||
],
|
||||
),
|
||||
);
|
||||
}).toList(),
|
||||
),
|
||||
if (msg.text.isNotEmpty) const SizedBox(height: 6),
|
||||
],
|
||||
if (msg.text.isNotEmpty)
|
||||
Text(
|
||||
msg.text,
|
||||
style: const TextStyle(
|
||||
color: AppColors.textPrimary,
|
||||
fontSize: 14,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
if (isUser) const SizedBox(width: 8),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -898,6 +1136,7 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
void dispose() {
|
||||
_userEndedCall = true;
|
||||
_textController.dispose();
|
||||
_scrollController.dispose();
|
||||
_durationTimer?.cancel();
|
||||
_waveTimer?.cancel();
|
||||
_waveController.dispose();
|
||||
|
|
@ -909,6 +1148,24 @@ class _AgentCallPageState extends ConsumerState<AgentCallPage>
|
|||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Simple message model for call-page chat display
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class _CallMessage {
|
||||
final String text;
|
||||
final bool isUser;
|
||||
final DateTime timestamp;
|
||||
final List<ChatAttachment>? attachments;
|
||||
|
||||
_CallMessage({
|
||||
required this.text,
|
||||
required this.isUser,
|
||||
required this.timestamp,
|
||||
this.attachments,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Small helper widget for mute / speaker buttons
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -320,6 +320,20 @@ async def entrypoint(ctx: JobContext) -> None:
|
|||
if response:
|
||||
logger.info("inject response: %s", response[:100])
|
||||
session.say(response)
|
||||
|
||||
# Send response text back to Flutter for display
|
||||
try:
|
||||
reply_payload = json.dumps({
|
||||
"type": "text_reply",
|
||||
"text": response,
|
||||
}).encode("utf-8")
|
||||
await ctx.room.local_participant.publish_data(
|
||||
reply_payload,
|
||||
reliable=True,
|
||||
topic="text_reply",
|
||||
)
|
||||
except Exception as pub_err:
|
||||
logger.warning("Failed to publish text_reply: %s", pub_err)
|
||||
else:
|
||||
logger.warning("inject_text_message returned empty response")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue