1、TTS服务（出版：仅在后台调用音频设备播放，未传到前端）

byzs-module-ai/pom.xml

@@ -221,6 +221,18 @@
                 </exclusion>
             </exclusions>
         </dependency>
+
+        <!-- 阿里云 - 文本转语音 -->
+        <dependency>
+            <groupId>com.alibaba.nls</groupId>
+            <artifactId>nls-sdk-tts</artifactId>
+            <version>2.2.14</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.nls</groupId>
+            <artifactId>nls-sdk-common</artifactId>
+            <version>2.2.14</version>
+        </dependency>
     </dependencies>
 
 </project>

byzs-module-ai/src/main/java/cn/iocoder/byzs/module/ai/controller/admin/chat/vo/message/AiChatMessageSendRespVO.java

@@ -39,5 +39,22 @@ public class AiChatMessageSendRespVO {
         private LocalDateTime createTime;
 
     }
+    @Schema(description = "音频数据(Base64编码)", example = "UklGRigAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQQBAAD...")
+    private String audioData;
 
+    @Schema(description = "音频元数据")
+    private AudioMeta audioMeta; // 新增音频元数据字段
+
+    @Schema(description = "音频元数据")
+    @Data
+    public static class AudioMeta {
+        @Schema(description = "采样率", requiredMode = Schema.RequiredMode.REQUIRED, example = "24000")
+        private Integer sampleRate; // 采样率（必须与TTS设置一致）
+
+        @Schema(description = "位深", requiredMode = Schema.RequiredMode.REQUIRED, example = "16")
+        private Integer bitDepth; // 位深（通常为16）
+
+        @Schema(description = "声道数", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
+        private Integer channels; // 声道数（通常为1-单声道）
+    }
 }

byzs-module-ai/src/main/java/cn/iocoder/byzs/module/ai/controller/admin/tts/StreamTtsService.java

@@ -0,0 +1,267 @@
+package cn.iocoder.byzs.module.ai.controller.admin.tts;
+
+import com.alibaba.nls.client.AccessToken;
+import com.alibaba.nls.client.protocol.NlsClient;
+import com.alibaba.nls.client.protocol.OutputFormatEnum;
+import com.alibaba.nls.client.protocol.SampleRateEnum;
+import com.alibaba.nls.client.protocol.tts.StreamInputTts;
+import com.alibaba.nls.client.protocol.tts.StreamInputTtsListener;
+import com.alibaba.nls.client.protocol.tts.StreamInputTtsResponse;
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.PreDestroy;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.sound.sampled.*;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+@Service
+@Slf4j
+public class StreamTtsService {
+
+    @Value("${ai.tts.appKey:4SUOF4LfaU7FekyW}")
+    private String appKey;
+
+    @Value("${ai.tts.token:20aa8ae2577b414db6c3cbbfcdc287be}")
+    private String token;
+
+    @Value("${ai.tts.url:wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1}")
+    private String url;
+
+
+    @Value("${ai.tts.ALIYUN_AK_ID:LTAI5tQhMPLXtSgXiPiWbw6D}")
+    private String ALIYUN_AK_ID;
+    @Value("${ai.tts.ALIYUN_AK_SECRET:HCXpFYjl4swk0qwfIKa9s2bXx0AWcG}")
+    private String ALIYUN_AK_SECRET;
+
+    private NlsClient client;
+    private PlaybackRunnable playbackRunnable;
+    private Thread playbackThread;
+    private StreamInputTts synthesizer;
+
+    @PostConstruct
+    public void init() {
+        getToken();
+        // 初始化NlsClient
+        client = new NlsClient(url, token);
+    }
+
+    public String getToken(){
+        AccessToken accessToken = new AccessToken(ALIYUN_AK_ID, ALIYUN_AK_SECRET);
+        try {
+            accessToken.apply();
+            token = accessToken.getToken();
+            long expireTime = accessToken.getExpireTime();
+            log.info("token: {}, expireTime: {}", token, expireTime);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return token;
+    }
+
+    /**
+     * 开始TTS语音合成
+     */
+    public void startTts() {
+        // 初始化播放器
+        playbackRunnable = new PlaybackRunnable(24000);
+        try {
+            playbackRunnable.prepare();
+        } catch (LineUnavailableException e) {
+            log.error("初始化音频播放器失败", e);
+            throw new RuntimeException("初始化音频播放器失败", e);
+        }
+        playbackThread = new Thread(playbackRunnable);
+        playbackThread.start();
+
+        // 创建TTS实例
+        try {
+            synthesizer = new StreamInputTts(client, getSynthesizerListener(playbackRunnable));
+        } catch (Exception e) {
+            log.error("创建TTS实例", e);
+            throw new RuntimeException("创建TTS实例", e);
+        }
+        synthesizer.setAppKey(appKey);
+        synthesizer.setFormat(OutputFormatEnum.PCM);
+        synthesizer.setSampleRate(SampleRateEnum.SAMPLE_RATE_24K);
+        synthesizer.setVoice("longxiaochun");
+        synthesizer.setVolume(50);
+        synthesizer.setPitchRate(0);
+        synthesizer.setSpeechRate(0);
+
+        try {
+            synthesizer.startStreamInputTts();
+        } catch (Exception e) {
+            log.error("启动TTS失败", e);
+            throw new RuntimeException("启动TTS失败", e);
+        }
+    }
+
+    /**
+     * 发送文本到TTS服务进行语音合成
+     */
+    public void sendText(String text) {
+        if (synthesizer == null) {
+            throw new IllegalStateException("TTS服务尚未启动，请先调用startTts()");
+        }
+        try {
+            synthesizer.sendStreamInputTts(text);
+        } catch (Exception e) {
+            log.error("发送文本到TTS服务失败", e);
+            throw new RuntimeException("发送文本到TTS服务失败", e);
+        }
+    }
+
+    /**
+     * 结束TTS语音合成
+     */
+    public void stopTts() {
+        try {
+            if (synthesizer != null) {
+                synthesizer.stopStreamInputTts();
+                synthesizer.close();
+            }
+            if (playbackRunnable != null) {
+                playbackRunnable.stop();
+            }
+            if (playbackThread != null) {
+                playbackThread.join();
+            }
+        } catch (Exception e) {
+            log.error("停止TTS服务失败", e);
+        } finally {
+            synthesizer = null;
+            playbackRunnable = null;
+            playbackThread = null;
+        }
+    }
+
+    private StreamInputTtsListener getSynthesizerListener(final PlaybackRunnable audioPlayer) {
+        return new StreamInputTtsListener() {
+            private boolean firstRecvBinary = true;
+
+            @Override
+            public void onSynthesisStart(StreamInputTtsResponse response) {
+                log.info("TTS合成开始: name={}, status={}", response.getName(), response.getStatus());
+            }
+
+            @Override
+            public void onSentenceBegin(StreamInputTtsResponse response) {
+                log.info("句子开始合成: name={}, status={}", response.getName(), response.getStatus());
+            }
+
+            @Override
+            public void onSentenceEnd(StreamInputTtsResponse response) {
+                log.info("句子合成结束: name={}, status={}", response.getName(), response.getStatus());
+            }
+
+            @Override
+            public void onSynthesisComplete(StreamInputTtsResponse response) {
+                log.info("TTS合成完成: name={}, status={}", response.getName(), response.getStatus());
+                audioPlayer.stop();
+            }
+
+            @Override
+            public void onAudioData(ByteBuffer message) {
+                if (firstRecvBinary) {
+                    firstRecvBinary = false;
+                    log.info("收到第一包语音数据");
+                }
+                byte[] bytesArray = new byte[message.remaining()];
+                message.get(bytesArray, 0, bytesArray.length);
+                audioPlayer.put(ByteBuffer.wrap(bytesArray));
+            }
+
+            @Override
+            public void onSentenceSynthesis(StreamInputTtsResponse response) {
+                log.info("句子合成进度: name={}, status={}", response.getName(), response.getStatus());
+            }
+
+            @Override
+            public void onFail(StreamInputTtsResponse response) {
+                log.error("TTS合成失败: session_id={}, task_id={}, status={}, message={}",
+                        getStreamInputTts().getCurrentSessionId(),
+                        response.getTaskId(),
+                        response.getStatus(),
+                        response.getStatusText());
+                audioPlayer.stop();
+            }
+        };
+    }
+
+    class PlaybackRunnable implements Runnable {
+        private AudioFormat af;
+        private DataLine.Info info;
+        private SourceDataLine targetSource;
+        private AtomicBoolean runFlag;
+        private ConcurrentLinkedQueue<ByteBuffer> queue;
+
+        public PlaybackRunnable(int sample_rate) {
+            af = new AudioFormat(sample_rate, 16, 1, true, false);
+            info = new DataLine.Info(SourceDataLine.class, af);
+            targetSource = null;
+            runFlag = new AtomicBoolean(true);
+            queue = new ConcurrentLinkedQueue<>();
+        }
+
+        public void prepare() throws LineUnavailableException {
+            targetSource = (SourceDataLine) AudioSystem.getLine(info);
+            targetSource.open(af, 4096);
+            targetSource.start();
+        }
+
+        public void put(ByteBuffer buffer) {
+            queue.add(buffer);
+        }
+
+        public void stop() {
+            runFlag.set(false);
+        }
+
+        @Override
+        public void run() {
+            if (targetSource == null) {
+                return;
+            }
+            while (runFlag.get()) {
+                if (queue.isEmpty()) {
+                    try {
+                        Thread.sleep(10);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                        break;
+                    }
+                    continue;
+                }
+                ByteBuffer buffer = queue.poll();
+                if (buffer == null) {
+                    continue;
+                }
+                byte[] data = buffer.array();
+                targetSource.write(data, 0, data.length);
+            }
+            if (!queue.isEmpty()) {
+                ByteBuffer buffer;
+                while ((buffer = queue.poll()) != null) {
+                    byte[] data = buffer.array();
+                    targetSource.write(data, 0, data.length);
+                }
+            }
+            targetSource.drain();
+            targetSource.stop();
+            targetSource.close();
+        }
+    }
+
+    @PreDestroy
+    public void destroy() {
+        if (client != null) {
+            client.shutdown();
+        }
+        stopTts();
+    }
+}

byzs-module-ai/src/main/java/cn/iocoder/byzs/module/ai/service/chat/AiChatMessageServiceImpl.java

@@ -3,6 +3,7 @@ package cn.iocoder.byzs.module.ai.service.chat;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.util.ObjUtil;
 import cn.hutool.core.util.StrUtil;
+import cn.iocoder.byzs.module.ai.controller.admin.tts.StreamTtsService;
 import cn.iocoder.byzs.module.ai.enums.model.AiPlatformEnum;
 import cn.iocoder.byzs.module.ai.util.AiUtils;
 import cn.iocoder.byzs.framework.common.pojo.CommonResult;
@@ -86,6 +87,9 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
     private AiKnowledgeDocumentService knowledgeDocumentService;
     @Resource
     private AiToolService toolService;
+    @Resource
+    private StreamTtsService streamTtsService;  // 注入TTS服务
+
 
     @Transactional(rollbackFor = Exception.class)
     public AiChatMessageSendRespVO sendMessage(AiChatMessageSendReqVO sendReqVO, Long userId) {
@@ -136,7 +140,7 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
 
     @Override
     public Flux<CommonResult<AiChatMessageSendRespVO>> sendChatMessageStream(AiChatMessageSendReqVO sendReqVO,
-            Long userId) {
+                                                                             Long userId) {
         // 1.1 校验对话存在
         AiChatConversationDO conversation = chatConversationService
                 .validateChatConversationExists(sendReqVO.getConversationId());
@@ -166,7 +170,10 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
         Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, model, sendReqVO);
         Flux<ChatResponse> streamResponse = chatModel.stream(prompt);
 
-        // 4.3 流式返回
+        // 4.3 初始化TTS服务
+        streamTtsService.startTts();
+
+        // 4.4 流式返回并处理TTS
         StringBuffer contentBuffer = new StringBuffer();
         return streamResponse.map(chunk -> {
             // 处理知识库的返回，只有首次才有
@@ -182,8 +189,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
             }
             // 响应结果
             String newContent = chunk.getResult() != null ? chunk.getResult().getOutput().getText() : null;
-            newContent = StrUtil.nullToDefault(newContent, ""); // 避免 null 的 情况
+            newContent = StrUtil.nullToDefault(newContent, ""); // 避免 null 的情况
+
             contentBuffer.append(newContent);
+
+            // 发送新内容到TTS服务进行语音合成
+            if (StrUtil.isNotBlank(newContent)) {
+                streamTtsService.sendText(newContent);
+            }
             return success(new AiChatMessageSendRespVO()
                     .setSend(BeanUtils.toBean(userMessage, AiChatMessageSendRespVO.Message.class))
                     .setReceive(BeanUtils.toBean(assistantMessage, AiChatMessageSendRespVO.Message.class)
@@ -192,16 +205,20 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
             // 忽略租户，因为 Flux 异步无法透传租户
             TenantUtils.executeIgnore(() -> chatMessageMapper.updateById(
                     new AiChatMessageDO().setId(assistantMessage.getId()).setContent(contentBuffer.toString())));
+            // 通知TTS服务文本发送完成
+            streamTtsService.stopTts();
         }).doOnError(throwable -> {
             log.error("[sendChatMessageStream][userId({}) sendReqVO({}) 发生异常]", userId, sendReqVO, throwable);
             // 忽略租户，因为 Flux 异步无法透传租户
             TenantUtils.executeIgnore(() -> chatMessageMapper.updateById(
                     new AiChatMessageDO().setId(assistantMessage.getId()).setContent(throwable.getMessage())));
+            // 发生错误时停止TTS服务
+            streamTtsService.stopTts();
         }).onErrorResume(error -> Flux.just(error(ErrorCodeConstants.CHAT_STREAM_ERROR)));
     }
 
     private List<AiKnowledgeSegmentSearchRespBO> recallKnowledgeSegment(String content,
-            AiChatConversationDO conversation) {
+                                                                        AiChatConversationDO conversation) {
         // 1. 查询聊天角色
         if (conversation == null || conversation.getRoleId() == null) {
             return Collections.emptyList();
@@ -273,8 +290,8 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
      * @return 消息上下文
      */
     private List<AiChatMessageDO> filterContextMessages(List<AiChatMessageDO> messages,
-            AiChatConversationDO conversation,
-            AiChatMessageSendReqVO sendReqVO) {
+                                                        AiChatConversationDO conversation,
+                                                        AiChatMessageSendReqVO sendReqVO) {
         if (conversation.getMaxContexts() == null || ObjUtil.notEqual(sendReqVO.getUseContext(), Boolean.TRUE)) {
             return Collections.emptyList();
         }
@@ -303,9 +320,9 @@ public class AiChatMessageServiceImpl implements AiChatMessageService {
     }
 
     private AiChatMessageDO createChatMessage(Long conversationId, Long replyId,
-            AiModelDO model, Long userId, Long roleId,
-            MessageType messageType, String content, Boolean useContext,
-            List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments) {
+                                              AiModelDO model, Long userId, Long roleId,
+                                              MessageType messageType, String content, Boolean useContext,
+                                              List<AiKnowledgeSegmentSearchRespBO> knowledgeSegments) {
         AiChatMessageDO message = new AiChatMessageDO().setConversationId(conversationId).setReplyId(replyId)
                 .setModel(model.getModel()).setModelId(model.getId()).setUserId(userId).setRoleId(roleId)
                 .setType(messageType.getValue()).setContent(content).setUseContext(useContext)