Merge branch 'master' of https://g.civnet.cn:8443/ReactWeb5/CivWeb

src/components/SoundAi/Ui/Dialog/index.jsx

@@ -4,7 +4,7 @@ import { Tooltip, Typography } from 'antd'
 import { Fragment, useCallback, useContext, useEffect, useRef, useState } from 'react'
 import { event } from '@wisdom-utils/utils'
 import moment from 'moment'
-import { Directive, AllDirectiveTypes, CloseSessionDirective } from '../../core'
+import { Directive, AllDirectiveTypes, CloseSessionDirective , PandaSpeechSynthesisUtterance} from '../../core'
 import context from '../../config'
 import SparkDirective from '../../core/directive/SparkDirective'
 
@@ -19,8 +19,10 @@ const Dialog = props => {
   const timeRef = useRef(null)
   const isDebug = globalConfig.get("isDebug")
   const currentDirectiveRef = useRef(null)
+  const pandaSpeechSynthesisUtteranceRef = useRef(null)
   //是否开启系统语音
   const isOpenSystemSound = globalConfig.getIn(['key', 'isOpenSystemSound'])
+  const defaultEmptyMsg = "亲,我暂时还未理解你的意思，请换一种方式表达或者描述更加具体"
 
   const [data, setData] = useState(() => [{
     role: "system",
@@ -47,6 +49,7 @@ const Dialog = props => {
     closeDirectiveRef.current = new Directive({
       directiveTypes: [CloseSessionDirective]
     })
+    pandaSpeechSynthesisUtteranceRef.current = new PandaSpeechSynthesisUtterance()
     if (globalConfig.get('key').isOpenSparkModel) {
       directiveRef.current.directiveTypes.push(SparkDirective)
       directiveRef.current._directiveTypes.push(SparkDirective)
@@ -56,14 +59,24 @@ const Dialog = props => {
     return () => {
       event.off("aiSound:finish", finish)
       event.off("aiSound:frame", frame)
+      if(pandaSpeechSynthesisUtteranceRef.current) {
+        pandaSpeechSynthesisUtteranceRef.current.clear()
+        pandaSpeechSynthesisUtteranceRef.current = null
+      }
       destroyTime()
     }
   }, [])
 
   const finish = useCallback(async ({ resultText }) => {
     if (!resultText) return
+    pandaSpeechSynthesisUtteranceRef.current?.clear()
+    if(pandaRecordWebSocketRef.current) {
       if (isOpenSystemSound) {
-      pandaRecordWebSocketRef.current && pandaRecordWebSocketRef.current.setStatus("pause")
+        pandaRecordWebSocketRef.current.stopRecorderManager()
+      } 
+      // else {
+      //   pandaRecordWebSocketRef.current.setStatus("pause")
+      // }
     }
     const directive = directiveRef.current.parse({
       text: resultText
@@ -77,24 +90,45 @@ const Dialog = props => {
       let resultMsg = ""
       await directive.excute({
         sendMsg: msg => {
+          isOpenSystemSound && pandaSpeechSynthesisUtteranceRef.current.push(msg)
           resultMsg += msg
           updateLastData({
             content: resultMsg
           })
         }
       })
-      if (isOpenSystemSound) {
-        pandaRecordWebSocketRef.current && pandaRecordWebSocketRef.current.setStatus("playing")
+      if(isOpenSystemSound) {
+        destroyTime()
+        await pandaSpeechSynthesisUtteranceRef.current.speakEnd()
+        startTime()
+      }
+      if(pandaRecordWebSocketRef.current) {
+        if(isOpenSystemSound) {
+          pandaRecordWebSocketRef.current.resumeRecorderManager()
+        } 
+        // else {
+        //   pandaRecordWebSocketRef.current.setStatus("playing")
+        // }
       } 
       currentDirectiveRef.current = null
       return
     }
+
     sendContent({
       role: "system",
-      content: "亲,我暂时还未理解你的意思，请换一种方式表达或者描述更加具体"
+      content: defaultEmptyMsg
     })
-    if (isOpenSystemSound) {
-      pandaRecordWebSocketRef.current && pandaRecordWebSocketRef.current.setStatus("playing")
+    if(isOpenSystemSound) {
+      pandaSpeechSynthesisUtteranceRef.current.push(defaultEmptyMsg)
+      await pandaSpeechSynthesisUtteranceRef.current.speakEnd()
+    }
+    if(pandaRecordWebSocketRef.current) {
+      if(isOpenSystemSound) {
+        pandaRecordWebSocketRef.current.resumeRecorderManager()
+      } 
+      // else {
+      //   pandaRecordWebSocketRef.current.setStatus("playing")
+      // }
     }
   }, [])
 
@@ -116,14 +150,10 @@ const Dialog = props => {
   const closeSession = ({ resultText }) => {
     if (currentDirectiveRef.current) {
       currentDirectiveRef.current.closeResponse()
-      pandaRecordWebSocketRef.current.wsClose()
-      pandaRecordWebSocketRef.current.setStatus("pause")
-      setTimeout(() => {
-        pandaRecordWebSocketRef.current.setStatus("playing")
-      }, 500)
+      delayStartSocket()
     } else {
       if (!isDebug) {
-        pandaRecordWebSocketRef.current.wsClose()
+        delayStartSocket()
         event.emit("aiSound:finish", {
           resultText
         })
@@ -131,6 +161,14 @@ const Dialog = props => {
     }
   }
 
+  const delayStartSocket = () => {
+    pandaRecordWebSocketRef.current.wsClose()
+    pandaRecordWebSocketRef.current.setStatus("pause")
+    setTimeout(() => {
+      pandaRecordWebSocketRef.current.setStatus("playing")
+    }, 500)
+  }
+
   const updateUserData = ({ resultTextTemp }) => {
     setData(data => {
       const length = data.length

src/components/SoundAi/core/core/SpeechSynthesisUtterance/index.js

+class PandaSpeechSynthesisUtterance {
+    constructor (options = {}) {
+        this.voiceParams = this.getVoiceParams(options.voiceParams ?? {})
+        this.data = []
+        window.speechSynthesis.cancel()
+        this.resolve = null
+        this.reject = null
+        this.onEnd = this.onEnd.bind(this)
+    }
+
+    getVoiceParams (voiceParams) {
+        return {
+            volume : 1,
+            rate : 1,
+            pitch : 1,
+            ...voiceParams
+        }
+    }
+
+    push(text) {
+        if(!text) return
+        this.data.push(this.getSpeechSynthesisUtterance(text))
+        window.speechSynthesis.speak(this.data.at(-1))
+    }
+
+    getSpeechSynthesisUtterance(text) {
+        const speak = new SpeechSynthesisUtterance(text)
+        speak.addEventListener("end", this.onEnd)
+        for(let key in this.voiceParams) {
+            speak[key] = this.voiceParams[key]
+        }
+        return speak
+    }
+
+    onEnd () {
+        this.data.shift()
+        if(this.data.length == 0) {
+            this.resolve && this.resolve()
+        }
+     }
+
+    speakEnd () {
+        return new Promise((resolve , reject) => {
+            this.resolve = resolve
+            this.reject = reject
+        })
+    }
+
+    clear() {
+        this.data = []
+        window.speechSynthesis.cancel()
+    }
+}
+
+export default PandaSpeechSynthesisUtterance
\ No newline at end of file

src/components/SoundAi/core/core/sockets/PandaRecordWebSocket.js

@@ -121,6 +121,20 @@ class PandaRecordWebSocket extends PandaWebSocket {
     this.recorderManager.onFrameRecorded = this.onRecorderFrameRecorded;
   }
 
+  pauseRecorederManager() {
+    if(this.recorderManager) {
+      this.recorderManager.stop()
+    }
+  }
+
+  resumeRecorderManager() {
+    this.createRecorder()
+    this.recorderManager.start({
+      sampleRate: 16000,
+      frameSize: 1280,
+    })
+  }
+
   onRecorderStart(e) {
     this.recorderManager.status = 'open';
     this.events.recorderStart();

src/components/SoundAi/core/core/sockets/PandaWebSocket.js

@@ -66,7 +66,7 @@ class PandaWebSocket {
 
   wsClose() {
     if (this.ws) {
-      this.ws.close();
+      this.ws.close(1000);
       this.ws = null;
     }
   }

src/components/SoundAi/core/directive/CloseMenuDirective/index.js

@@ -9,15 +9,17 @@ class CloseMenuDirective extends BaseDirective {
   match() {
     const { text } = this;
     let result = null;
-    result = new RegExp(`关闭(所有).*(?=(?:[${config.punctuationMark}]?))`)
+    if(!result) {
+      result = text.match(new RegExp(`关闭(所有).*(?=(?:[${config.punctuationMark}]?))`))
+    }
     if (!result) {
-      result = new RegExp(`关闭(当前).*(?=(?:[${config.punctuationMark}]?))`)
+      result = text.match(new RegExp(`关闭(当前).*(?=(?:[${config.punctuationMark}]?))`))
     }
     if (!result) {
       result = text.match(/关闭(.*)(?=(?:菜单|功能|路径))/);
     }
     if (!result) {
-      result = new RegExp(`关闭(.*)(?=(?:[${config.punctuationMark}]?))`)
+      result = text.match(new RegExp(`关闭(.*)(?=(?:[${config.punctuationMark}]?))`))
     }
     if (result && result.length >= 2 && !this.matchResult) {
       this.matchResult = result[1];

src/components/SoundAi/core/directive/OpenMenuDirective/index.js

@@ -15,7 +15,7 @@ class OpenMenuDirective extends BaseDirective {
             result = text.match(/打开(.*)(?=(?:菜单|功能|路径))/)
         }
         if (!result) {
-            result = new RegExp(`打开(.*)(?=(?:[${config.punctuationMark}]?))`)
+            result = text.match(new RegExp(`打开(.*)(?=(?:[${config.punctuationMark}]?))`))
         }
         if (result && result.length >= 2 && !this.matchResult) {
             this.matchResult = result[1]

src/components/SoundAi/core/directive/TestDirective.js

@@ -27,8 +27,14 @@ class TestDirective extends BaseDirective {
     }
 
     testCloseSession({sendMsg}) {
+        let count = 0;
+        let random = parseInt(Math.random() * 50 + 1)
         this.timer = setInterval(() => {
-            sendMsg(parseInt(Math.random()* 10))
+            count++
+            if(count % random == 0) {
+                random =  parseInt(Math.random() * 50 + 1)
+                sendMsg(count)
+            } 
         }, 100)
     }
 }

src/components/SoundAi/core/index.js

@@ -7,6 +7,7 @@ import Directive , {DirectiveTypes, AllDirectiveTypes} from './directive';
 import AwakenDirective from './directive/AwakenDirective';
 import ByeDirective from './directive/ByeDirective';
 import CloseSessionDirective from './directive/CloseSessionDirective';
+import PandaSpeechSynthesisUtterance from './core/SpeechSynthesisUtterance';
 
 export {
     RecorderManager,
@@ -19,5 +20,6 @@ export {
     ByeDirective,
     DirectiveTypes,
     AllDirectiveTypes,
-    CloseSessionDirective
+    CloseSessionDirective,
+    PandaSpeechSynthesisUtterance
 };

src/components/SoundAi/testCommand.js

@@ -69,7 +69,11 @@ const testCommand = async () => {
   // });
   // await testFrameText("熊猫智慧水务解决方案是什么")
   // await Utils.delayTime(6000)
-  // await testFrameText("打开场景搭建功能菜单")
+  // await testFrameText(" 关闭当前他的。")
+  await testFrameText("今天武汉天气怎么样")
+  // await  testFrameText("熊猫智慧水务解决方案是什么")
+  // await Utils.delayTime(6000)
+  // await testFrameText("我需要测试熊猫测试指令")
 }
 
 export default testCommand
\ No newline at end of file