feat: 完成UI界面

src/components/SoundAi/PandaUi.js

@@ -64,9 +64,11 @@ const PandaUi = props => {
             events : {
                 frame (data)  { 
                     event.emit("aiSound:frame", data)
+                    console.log(data.resultTextTemp)
                 }, 
                 finish (data) {
                     event.emit("aiSound:finish", data)
+                    console.log(data.resultText)
                 },
                 fail ({data}) {
                     message.info(data.msg)
@@ -75,6 +77,7 @@ const PandaUi = props => {
                         loading: false,
                         checked: false
                     })
+                    setIsStartRecorder(false)
                 }
             }
         })

src/components/SoundAi/Ui/Dialog/index.jsx

 import style from './style.less'
-import { QuestionCircleOutlined } from '@ant-design/icons'
 import { withRouter } from '@wisdom-utils/runtime'
 import { Tooltip, Typography } from 'antd'
 import { Fragment, useCallback, useEffect, useRef, useState } from 'react'
 import { event } from '@wisdom-utils/utils'
 import moment from 'moment'
-import { Directive } from '../../core'
+import { Directive , AllDirectiveTypes} from '../../core'
+import config from '../../config'
 
 const Dialog = props => {
 
-    const { pandaRecordWebSocketRef } = props
+    const { pandaRecordWebSocketRef , close} = props
     const ulRef = useRef(null)
     const directiveRef = useRef(null)
+    const toolTipRef = useRef(null)
+    const timeRef = useRef(null)
     const [data, setData] = useState([])
+    
+
+    useEffect(() => {
+        if (!ulRef.current.lastChild) return
+        ulRef.current.lastChild.scrollIntoView({
+            block: 'end',
+            behavior: 'smooth'
+        })
+        startTime()
+    }, [data])
+
+    useEffect(() => {
+        directiveRef.current = new Directive({
+            params: {
+            }
+        })
+        event.on("aiSound:finish", finish)
+        event.on("aiSound:frame", frame)
+        startTime()
+        return () => {
+            event.off("aiSound:finish", finish)
+            event.off("aiSound:frame", frame)
+            destroyTime()
+        }
+    }, [])
 
     const finish = useCallback(async ({ resultText }) => {
         pandaRecordWebSocketRef.current.setStatus("pause")
@@ -38,14 +65,6 @@ const Dialog = props => {
         pandaRecordWebSocketRef.current.setStatus("playing")
     }, [])
 
-    useEffect(() => {
-        if (!ulRef.current.lastChild) return
-        ulRef.current.lastChild.scrollIntoView({
-            block: 'end',
-            behavior: 'smooth'
-        })
-    }, [data])
-
     const frame = useCallback((data) => {
         const { resultText, resultTextTemp } = data
         console.log(resultTextTemp)
@@ -76,22 +95,6 @@ const Dialog = props => {
         })
     }, [])
 
-    useEffect(() => {
-        directiveRef.current = new Directive({
-            params: {
-            }
-        })
-    }, [])
-
-    useEffect(() => {
-        event.on("aiSound:finish", finish)
-        event.on("aiSound:frame", frame)
-        return () => {
-            event.off("aiSound:finish", finish)
-            event.off("aiSound:frame", frame)
-        }
-    }, [])
-
     const sendContent = ({ role = "system", content, ...params }) => {
         setData(i => i.concat([{
             role,
@@ -117,6 +120,19 @@ const Dialog = props => {
         }))
     }
 
+    const startTime = () => {
+        destroyTime()
+        timeRef.current = setTimeout(() => {
+            close && close()
+        }, config.closePandaAiTime)
+    }
+
+    const destroyTime = () => {
+        if(timeRef.current) {
+            clearTimeout(timeRef.current)
+            timeRef.current = null
+        }
+    }
 
     return (<div className={style.container}>
         <div className={style.pandaIcon}></div>
@@ -126,19 +142,23 @@ const Dialog = props => {
                 <div className={style.help}>
                     <Tooltip
                         placement="bottomRight"
-                        title = {<span>邹绪超</span>}
-                        visible = {true}
+                        title = {
+                            <div className={style.toolTipContent}>
+                                 <p className={style.toolTipTitle}>支持以下语音指令集</p>
+                                 <div className={style.toolTipWrapper}>
+                                    <ul>
+                                        {
+                                            AllDirectiveTypes.map((item , index) => <li key={index}>{index + 1}.{item.name}</li>)
+                                        }
+                                    </ul>
+                                 </div>
+                            </div>
+                        }
+                        getPopupContainer = {() => toolTipRef.current}
+                        arrowPointAtCenter = {true}
                     >
-                        <span className={style.helpIcon}></span>
+                        <span className={style.helpIcon} ref = {toolTipRef}></span>
                     </Tooltip>
-                    {/* <div className={style.toolTipContent}>
-                        <p>支持以下语音指令集</p>
-                        <ul className={style.toolTip}>
-                            {
-                                [].map((name, index) => <li key={index}>{index + 1}.{name}</li>)
-                            }
-                        </ul>
-                    </div> */}
                 </div>
             </div>
             <div className={style.content}>

src/components/SoundAi/Ui/Dialog/style.less

@@ -73,34 +73,27 @@
                 }
 
                 .toolTipContent {
-                    display: none;
-                    position: absolute;
-                    white-space: nowrap;
-                    right: 0;
-                    font-weight: normal;
-                    font-size: 14px;
-                    // background: #f5f5f5;
-                    padding: 0 10px;
-                    border-radius: 5px;
-                    z-index: 10;
-
-                    >p {
-                        margin-bottom: 0;
-                        font-weight: bold;
+                    display: flex;
+                    flex-direction: column;
+                    .toolTipTitle{
+                        margin-bottom: 5px;
+                        white-space: nowrap;
                     }
-
-                    .toolTip {
-                        display: flex;
-                        flex-direction: column;
-                        margin-bottom: 0;
-
-                        >li {
-                            white-space: nowrap;
+                    .toolTipWrapper{
+                        max-height: 200px;
+                        overflow-y: auto;
+                        >ul{
+                            list-style: none;
+                            flex-direction: column;
                             display: flex;
-                            align-items: center;
-                            height: 30px;
+                            max-width: 207px;
+                            >li{
+                                margin-bottom: 5px;
+                                width: 100%;
+                            }
                         }
                     }
+         
                 }
 
             }

src/components/SoundAi/Ui/PandaTip/index.jsx

@@ -4,12 +4,16 @@ import image from '@/assets/images/soundAi/眨眼.gif'
 import { event } from '@wisdom-utils/utils'
 import { AwakenDirective, ByeDirective, Directive } from '../../core'
 import Dialog from '../Dialog'
+import config from '../../config'
 
 const PandaTip = props => {
+    const { pandaRecordWebSocketRef } = props
 
     const [status, setStatus] = useState("close")
     const directiveRef = useRef(null)
-    const { pandaRecordWebSocketRef } = props
+    const [initOpen, setInitOpen] = useState(false)
+    const [showTip, setShowTip] = useState(true)
+    const timeRef = useRef(null)
     const finish = useCallback(({ resultText }) => {
         const directive = directiveRef.current.parse({
             text: resultText
@@ -30,17 +34,54 @@ const PandaTip = props => {
 
     useEffect(() => {
         event.on("aiSound:finish", finish)
+        if (status == "open") {
+            destroyTime()
+        } else if (status == "close") {
+            if (initOpen == true) {
+                startTime()
+                setShowTip(false)
+            } else {
+                setShowTip(true)
+            }
+        }
         return () => {
             event.off("aiSound:finish", finish)
         }
-    }, [status])
+    }, [status , initOpen])
+
+    const startTime = () => {
+        destroyTime()
+        timeRef.current = setTimeout(() => {
+            setShowTip(true)
+        }, config.showOpenAiTipTime)
+    }
+
+    const destroyTime = () => {
+        if (timeRef.current) {
+            clearTimeout(timeRef.current)
+            timeRef.current = null
+        }
+    }
+
+
+    const close = () => {
+        setStatus("close")
+        setInitOpen(true)
+        // let count = 0
+        // setInterval(() => {
+        //     ++count;
+        //     console.log(count)
+        // }, 1000)
+    }
 
     return (<div className={style.container}>
         {
-            status == "open" ? <Dialog pandaRecordWebSocketRef={pandaRecordWebSocketRef} /> : <div className={style.content}>
-                <div className={style.tipImage}>
-                    <span>呼叫“熊猫熊猫”唤醒语音小助手。</span>
-                </div>
+            status == "open" ? <Dialog close={close} pandaRecordWebSocketRef={pandaRecordWebSocketRef} /> : <div className={style.content}>
+                {
+                    showTip ? <div className={style.tipImage}>
+                        <span>呼叫“熊猫熊猫”唤醒语音小助手。</span>
+                    </div> : null
+                }
                 <img src={image} className={style.image} />
             </div>
         }

src/components/SoundAi/config.js

+const config = {
+    closePandaAiTime : 1000 * 60,
+    showOpenAiTipTime :  1000 * 60
+}
+
+export default config
\ No newline at end of file

src/components/SoundAi/core/core/sockets/PandaRecordWebSocket.js

@@ -23,6 +23,7 @@ class PandaRecordWebSocket extends PandaWebSocket {
     this.count = 0;
     this.initFrameBuffer = null;
     this.status = "playing";
+    this.startTime = new Date()
   }
 
   async start() {
@@ -132,13 +133,14 @@ class PandaRecordWebSocket extends PandaWebSocket {
     return isEmpty;
   }
 
-  onRecorderFrameRecorded({ isLastFrame, frameBuffer }) {
+  onRecorderFrameRecorded({ isLastFrame }) {
+    const frameBuffer = new ArrayBuffer(1280)
     const { encoding, format } = this.config;
     if (!this.ws) {
       const isEmptyData = this.getIsEmptyData(frameBuffer, 200, 20);
       if (isEmptyData) {
         this.initFrameBuffer = null;
-        console.log('系统檢測到輸入');
+        console.log('系统未检测到输入');
         return;
       }
       if(this.status == "paused") return
@@ -148,6 +150,32 @@ class PandaRecordWebSocket extends PandaWebSocket {
       this.createWs();
     }
     if (this.ws.readyState === this.ws.OPEN) {
+      // if((new Date().getTime() - this.startTime.getTime())  >= (this.config.vad_eos )) {
+      //   console.log("主动断开")
+      //   this.ws.send(
+      //     JSON.stringify({
+      //       data: {
+      //         format,
+      //         encoding,
+      //         status: 2,
+      //       },
+      //     }),
+      //   );
+      //   this.wsClose()
+      // } else {
+      //   console.log("2222")
+      //   this.ws.send(
+      //     JSON.stringify({
+      //       data: {
+      //         format,
+      //         encoding,
+      //         status: isLastFrame ? 2 : 1,
+      //         audio: Utils.toBase64(frameBuffer),
+      //       },
+      //     }),
+      //   );
+      // }
+
       this.ws.send(
         JSON.stringify({
           data: {
@@ -162,10 +190,24 @@ class PandaRecordWebSocket extends PandaWebSocket {
         this.destroy();
       }
     }
+    
+ 
   }
 
   reconnect() {
+    const { encoding, format } = this.config;
     this.count += 1;
+    if (this.ws && this.ws.readyState === this.ws.OPEN) {
+      this.ws.send(
+        JSON.stringify({
+          data: {
+            format,
+            encoding,
+            status: 2,
+          },
+        }),
+      );
+    }
     this.wsClose();
   }
 
@@ -202,9 +244,11 @@ class PandaRecordWebSocket extends PandaWebSocket {
       data,
     };
     this.ws.send(JSON.stringify(params));
+    this.startTime = new Date()
   }
 
   wsOnmessage(e) {
+    this.startTime = new Date()
     super.wsOnmessage(e);
     const jsonData = JSON.parse(e.data);
     if (jsonData.data && jsonData.data.result) {
@@ -216,7 +260,7 @@ class PandaRecordWebSocket extends PandaWebSocket {
       }
       // 开启wpgs会有此字段(前提：在控制台开通动态修正功能)
       // 取值为 "apd"时表示该片结果是追加到前面的最终结果；取值为"rpl" 时表示替换前面的部分结果，替换范围为rg字段
- 
+
       if (data.pgs) {
         if (data.pgs === 'apd') {
           // 将resultTextTemp同步给resultText
@@ -227,15 +271,14 @@ class PandaRecordWebSocket extends PandaWebSocket {
       } else {
         this.resultText = this.resultText + str;
       }
-      if(!this.resultTextTemp) {
-        return
+      if(this.resultTextTemp) {
+        this.events.frame({
+          resultTextTemp: this.resultTextTemp,
+          resultText: this.resultText,
+          pgs: data.pgs,
+          count: this.count,
+        });
       }
-      this.events.frame({
-        resultTextTemp: this.resultTextTemp,
-        resultText: this.resultText,
-        pgs: data.pgs,
-        count: this.count,
-      });
     }
     if (jsonData.code == 0 && jsonData.data.status == 2) {
       const resultText = this.resultTextTemp ? this.resultTextTemp : this.resultText

src/components/SoundAi/core/directive/AwakenDirective.js

 import BaseDirective from "./BaseDirective"
 
 class AwakenDirective extends BaseDirective{
+    static name = "熊猫熊猫唤醒"
     match () {
         const {text} = this
         if(/熊猫熊猫/.test(text)) {

src/components/SoundAi/core/directive/ByeDirective.js

 import BaseDirective from "./BaseDirective";
 
 class ByeDirective extends BaseDirective {
+    static name = "再见或者拜拜关闭熊猫"
     match () {
         const {text} = this
         if(/再见|拜拜|关闭熊猫/.test(text)) {

src/components/SoundAi/core/directive/CloseMenuDirective.js

@@ -3,6 +3,7 @@ import {store , event} from '@wisdom-utils/utils'
 import Utils from "../utils/Utils";
 
 class CloseMenuDirective extends BaseDirective {
+    static name="关闭菜单或者模糊匹配关闭菜单"
     match () {
         const { text } = this
         let result = null

src/components/SoundAi/core/directive/HomePageDirective.js

@@ -2,6 +2,7 @@ import Utils from '../utils/Utils'
 import BaseDirective from './BaseDirective'
 
 class HomePageDirective extends BaseDirective {
+    static name = "返回首页"
     match () {
         const {text} = this
         if(/首页/.test(text)) {

src/components/SoundAi/core/directive/OpenMenuDirective.js

@@ -4,6 +4,7 @@ import Utils from "../utils/Utils";
 import { uniq } from "lodash"; 
 
 class OpenMenuDirective extends BaseDirective {
+    static name="打开菜单或者模糊匹配打开菜单"
     match() {
         const { text } = this
         let result = null

src/components/SoundAi/core/directive/index.js

@@ -2,6 +2,7 @@ import AwakenDirective from "./AwakenDirective"
 import OpenMenuDirective from "./OpenMenuDirective"
 import HomePageDirective from "./HomePageDirective"
 import CloseMenuDirective from "./CloseMenuDirective"
+import ByeDirective from "./ByeDirective"
 
 const DirectiveTypes = [
     AwakenDirective,
@@ -9,6 +10,13 @@ const DirectiveTypes = [
     OpenMenuDirective,
     CloseMenuDirective
 ]
+const AllDirectiveTypes = [
+    AwakenDirective,
+    HomePageDirective,
+    OpenMenuDirective,
+    CloseMenuDirective,
+    ByeDirective
+]
 class Directive {
     constructor (options = {}) {
         this.directiveTypes = options.directiveTypes || DirectiveTypes
@@ -40,4 +48,8 @@ class Directive {
     }
 }
 
-export default Directive
\ No newline at end of file
+export default Directive
+export {
+    DirectiveTypes,
+    AllDirectiveTypes
+}
\ No newline at end of file

src/components/SoundAi/core/index.js

@@ -3,7 +3,7 @@ import PandaWebSocket from './core/sockets/PandaWebSocket';
 import PandaRecordWebSocket from './core/sockets/PandaRecordWebSocket';
 import Utils from './utils/Utils';
 import PandaSparkWebSocket from './core/sockets/PandaSparkWebSocket';
-import Directive from './directive';
+import Directive , {DirectiveTypes, AllDirectiveTypes} from './directive';
 import AwakenDirective from './directive/AwakenDirective';
 import ByeDirective from './directive/ByeDirective';
 
@@ -15,5 +15,7 @@ export {
     PandaSparkWebSocket,
     Directive,
     AwakenDirective,
-    ByeDirective
+    ByeDirective,
+    DirectiveTypes,
+    AllDirectiveTypes
 };