Skip to content

C
CivWeb
Commit 9a13c9d2 authored by 邹绪超's avatar 邹绪超
Browse files
Options

feat: 完成关闭指令

parent f82a7aff
Pipeline #85818 passed with stages
          Hide whitespace changes
          Inline Side-by-side
          Showing with 292 additions and 248 deletions
          package.json
          View file @ 9a13c9d2
          ......@@ -115,7 +115,7 @@
          "@wisdom-map/arcgismap": "1.4.0-261",
          "@wisdom-map/basemap": "1.1.0-41",
          "@wisdom-map/util": "^1.0.28-0",
          "@wisdom-utils/components": "0.1.351",
          "@wisdom-utils/components": "0.1.352",
          "@wisdom-utils/runtime": "0.0.48",
          "@wisdom-utils/utils": "0.1.388",
          "@congcongcai/jieba.js" : "0.0.3",
          ......
          src/components/SoundAi/PandaUi.js
          View file @ 9a13c9d2
          import { request, event } from '@wisdom-utils/utils'
          import { useCallback, useEffect, useRef, useState } from 'react'
          import { PandaRecordWebSocket } from './core'
          import { message } from 'antd'
          import PandaTip from './Ui/PandaTip'
          import { init, cut, extract, ready } from "@congcongcai/jieba.js";
          import { request, event } from '@wisdom-utils/utils';
          import { useCallback, useEffect, useRef, useState } from 'react';
          import { message } from 'antd';
          import { init, cut, extract, ready } from '@congcongcai/jieba.js';
          import { PandaRecordWebSocket } from './core';
          import PandaTip from './Ui/PandaTip';
          const PandaUi = props => {
          const pandaRecordWebSocketRef = useRef(null);
          const [isStartRecorder, setIsStartRecorder] = useState(false);
          const { keyRef } = props;
          const pandaRecordWebSocketRef = useRef(null)
          const [isStartRecorder, setIsStartRecorder] = useState(false)
          const {keyRef} = props
          useEffect(async () => {
          try {
          await initJieBa()
          event.emit("record:changeVisible", {
          visible: true
          })
          event.on("record:changedStaus", changeSwitch)
          } catch (error) {
          console.warn(error)
          }
          return () => {
          event.off("record:changedStaus", changeSwitch)
          }
          }, [])
          useEffect(async () => {
          try {
          await initJieBa();
          event.emit('record:changeVisible', {
          visible: true,
          });
          event.on('record:changedStaus', changeSwitch);
          } catch (error) {
          console.warn(error);
          }
          return () => {
          event.off('record:changedStaus', changeSwitch);
          };
          }, [changeSwitch]);
          const changeSwitch = useCallback(async ({ recordData, e: checked }) => {
          if (checked) {
          emitSwitch({
          loading: true
          })
          createPandaRecordWebSocket()
          const data = await pandaRecordWebSocketRef.current.start()
          emitSwitch({
          loading: false
          })
          if (!data.isSuccess) {
          return message.info(data.msg)
          }
          emitSwitch({
          checked: true
          })
          setIsStartRecorder(true)
          } else {
          emitSwitch({
          loading: true
          })
          await destroyPandaRecorderWebSocket()
          emitSwitch({
          loading: false,
          checked: false
          })
          setIsStartRecorder(false)
          const changeSwitch = useCallback(
          async ({ recordData, e: checked }) => {
          if (checked) {
          emitSwitch({
          loading: true,
          });
          createPandaRecordWebSocket();
          const data = await pandaRecordWebSocketRef.current.start();
          emitSwitch({
          loading: false,
          });
          if (!data.isSuccess) {
          return message.info(data.msg);
          }
          }, [])
          emitSwitch({
          checked: true,
          });
          setIsStartRecorder(true);
          } else {
          emitSwitch({
          loading: true,
          });
          await destroyPandaRecorderWebSocket();
          emitSwitch({
          loading: false,
          checked: false,
          });
          setIsStartRecorder(false);
          }
          },
          [createPandaRecordWebSocket],
          );
          const createPandaRecordWebSocket = async () => {
          destroyPandaRecorderWebSocket()
          pandaRecordWebSocketRef.current = new PandaRecordWebSocket({
          config : keyRef.current.shortSound,
          events : {
          frame (data) {
          event.emit("aiSound:frame", data)
          console.log(data.resultTextTemp)
          },
          finish (data) {
          event.emit("aiSound:finish", data)
          console.log(data.resultText)
          },
          fail ({data}) {
          message.info(data.msg)
          console.log(data)
          emitSwitch({
          loading: false,
          checked: false
          })
          setIsStartRecorder(false)
          }
          }
          })
          // await testText("请打开泵站。")
          // await testText("第2个")
          // await testText("请打开泵站。")
          // await testText("第3个")
          // await testText("关闭泵站。");
          // await testText("第二个");
          // await testText("我需要打开浩源孪生平台菜单")
          // await testText("我需要返回首页")
          // await testText("我需要打开礼泉孪生平台菜单")
          // await testText("我需要打开浩源孪生平台菜单")
          // await testText("我需要关闭平台菜单")
          // await testText("我需要打开孪生平台菜单")
          // await testText("我需要打开第二个菜单")
          // await testText("我需要打开浩源孪生平台菜单")
          }
          const createPandaRecordWebSocket = async () => {
          destroyPandaRecorderWebSocket();
          pandaRecordWebSocketRef.current = new PandaRecordWebSocket({
          config: keyRef.current.shortSound,
          events: {
          frame(data) {
          event.emit('aiSound:frame', data);
          console.log(data.resultTextTemp);
          },
          finish(data) {
          event.emit('aiSound:finish', data);
          console.log(data.resultText);
          },
          fail({ data }) {
          message.info(data.msg);
          console.log(data);
          emitSwitch({
          loading: false,
          checked: false,
          });
          setIsStartRecorder(false);
          },
          },
          });
          // await testText("请打开海港泵站。")
          // await testText("打开第1个");
          // await testText("打开熊猫孪生水厂。");
          // await testText("关闭当前菜单")
          // await testText("关闭所有菜单。");
          // await testText("请打开泵站。")
          // await testText("第2个")
          // await testText("请打开泵站。")
          // await testText("第3个")
          // await testText("关闭泵站。");
          // await testText("第二个");
          // await testText("我需要打开浩源孪生平台菜单")
          // await testText("我需要返回首页")
          // await testText("我需要打开礼泉孪生平台菜单")
          // await testText("我需要打开浩源孪生平台菜单")
          // await testText("我需要关闭平台菜单")
          // await testText("我需要打开孪生平台菜单")
          // await testText("我需要打开第二个菜单")
          // await testText("我需要打开浩源孪生平台菜单")
          };
          const testText = text => new Promise((resolve , reject) => {
          const testText = text =>
          new Promise((resolve, reject) => {
          setTimeout(() => {
          event.emit('aiSound:frame', {
          resultTextTemp: text,
          });
          setTimeout(() => {
          event.emit("aiSound:frame", {
          resultTextTemp: text
          })
          setTimeout(() => {
          event.emit("aiSound:finish", {
          resultText: text
          })
          setTimeout(() => {
          resolve()
          }, 500)
          }, 500)
          }, 500)
          })
          event.emit('aiSound:finish', {
          resultText: text,
          });
          setTimeout(() => {
          resolve();
          }, 500);
          }, 500);
          }, 500);
          });
          const destroyPandaRecorderWebSocket = async () => {
          if (pandaRecordWebSocketRef.current) {
          await pandaRecordWebSocketRef.current.destroy()
          pandaRecordWebSocketRef.current = null
          }
          const destroyPandaRecorderWebSocket = async () => {
          if (pandaRecordWebSocketRef.current) {
          await pandaRecordWebSocketRef.current.destroy();
          pandaRecordWebSocketRef.current = null;
          }
          };
          const emitSwitch = data => {
          event.emit("record:changeSwicth", data)
          }
          const emitSwitch = data => {
          event.emit('record:changeSwicth', data);
          };
          const initJieBa = () => new Promise((resolve, reject) => {
          ready().then(() => {
          init()
          resolve()
          }).catch(() => {
          reject()
          const initJieBa = () =>
          new Promise((resolve, reject) => {
          ready()
          .then(() => {
          init();
          resolve();
          })
          })
          return (<>
          {
          isStartRecorder ? <PandaTip pandaRecordWebSocketRef={pandaRecordWebSocketRef} /> : null
          }
          </>)
          }
          .catch(() => {
          reject();
          });
          });
          return <>{isStartRecorder ? <PandaTip pandaRecordWebSocketRef={pandaRecordWebSocketRef} /> : null}</>;
          };
          export default PandaUi
          \ No newline at end of file
          export default PandaUi;
          src/components/SoundAi/Ui/PandaTip/index.jsx
          View file @ 9a13c9d2
          ......@@ -22,7 +22,8 @@ const PandaTip = props => {
          if (directive instanceof AwakenDirective && status == "close") {
          setStatus("open")
          } else if (directive instanceof ByeDirective && status == "open") {
          setStatus("close")
          // setStatus("close")
          close()
          }
          }, [status])
          ......
          src/components/SoundAi/core/core/sockets/PandaRecordWebSocket.js
          View file @ 9a13c9d2
          import PandaWebSocket from './PandaWebSocket';
          import CryptoJS from 'crypto-js';
          import PandaWebSocket from './PandaWebSocket';
          import RecorderManager from '../../libs/aiSounds/index.umd1.js';
          import Utils from '../../utils/Utils.js';
          import { FAIL_TYPE_DATA_ERRRO_MSG, FAIL_TYPE_DATA_ERRRO } from '../../constants/index.js';
          class PandaRecordWebSocket extends PandaWebSocket {
          static url = 'wss://iat-api.xfyun.cn/v2/iat';
          static host = 'iat-api.xfyun.cn';
          static algorithm = 'hmac-sha256';
          static headers = 'host date request-line';
          constructor(options) {
          ......@@ -22,8 +25,8 @@ class PandaRecordWebSocket extends PandaWebSocket {
          this.resultTextTemp = '';
          this.count = 0;
          this.initFrameBuffer = null;
          this.status = "playing";
          this.startTime = new Date()
          this.status = 'playing';
          this.startTime = new Date();
          }
          async start() {
          ......@@ -44,31 +47,31 @@ class PandaRecordWebSocket extends PandaWebSocket {
          }
          initConfig() {
          //一句话间隔5s
          // 一句话间隔5s
          if (this.config.vad_eos == undefined) {
          this.config.vad_eos = 2000;
          }
          //动态修正
          // 动态修正
          if (this.config.dwa == undefined) {
          this.config.dwa = 'wpgs';
          }
          //标点符号
          // 标点符号
          if (this.config.ptt == undefined) {
          this.config.ptt = 1;
          }
          //采样
          // 采样
          if (this.config.format == undefined) {
          this.config.format = 'audio/L16;rate=16000';
          }
          //音频格式
          // 音频格式
          if (this.config.encoding == undefined) {
          this.config.encoding = 'raw';
          }
          //验证延迟时间
          // 验证延迟时间
          if (this.config.checkDelayDeciveTime == undefined) {
          this.config.checkDelayDeciveTime = 1500;
          }
          //worker
          // worker
          if (this.config.worker == undefined) {
          this.config.worker = 'https://civgis.panda-water.cn/system/pandaAiSoundWorker';
          }
          ......@@ -76,15 +79,15 @@ class PandaRecordWebSocket extends PandaWebSocket {
          initEvents() {
          if (!this.events.recorderStart) {
          this.events.recorderStart = function () {};
          this.events.recorderStart = function() {};
          }
          if (!this.events.recorderStop) {
          this.events.recorderStop = function () {};
          this.events.recorderStop = function() {};
          }
          }
          setStatus(status) {
          this.status = status
          this.status = status;
          }
          getWebSocketUrl() {
          ......@@ -96,9 +99,13 @@ class PandaRecordWebSocket extends PandaWebSocket {
          const signatureOrigin = `host: ${PandaRecordWebSocket.host}\ndate: ${date}\nGET /v2/iat HTTP/1.1`;
          const signatureSha = CryptoJS.HmacSHA256(signatureOrigin, apiSecret);
          const signature = CryptoJS.enc.Base64.stringify(signatureSha);
          const authorizationOrigin = `api_key="${apiKey}", algorithm="${PandaRecordWebSocket.algorithm}", headers="${PandaRecordWebSocket.headers}", signature="${signature}"`;
          const authorizationOrigin = `api_key="${apiKey}", algorithm="${PandaRecordWebSocket.algorithm}", headers="${
          PandaRecordWebSocket.headers
          }", signature="${signature}"`;
          const authorization = btoa(authorizationOrigin);
          const url = `${PandaRecordWebSocket.url}?authorization=${authorization}&date=${date}&host=${PandaRecordWebSocket.host}`;
          const url = `${PandaRecordWebSocket.url}?authorization=${authorization}&date=${date}&host=${
          PandaRecordWebSocket.host
          }`;
          return url;
          }
          ......@@ -118,8 +125,8 @@ class PandaRecordWebSocket extends PandaWebSocket {
          }
          getIsEmptyData(buffer, length, count) {
          let _count = 0,
          isEmpty = true;
          let _count = 0;
          let isEmpty = true;
          const array = new Uint8Array(buffer);
          for (let i = 0; i < length; ++i) {
          if (array[i] != 0) {
          ......@@ -133,8 +140,7 @@ class PandaRecordWebSocket extends PandaWebSocket {
          return isEmpty;
          }
          onRecorderFrameRecorded({ isLastFrame }) {
          const frameBuffer = new ArrayBuffer(1280)
          onRecorderFrameRecorded({ isLastFrame, frameBuffer }) {
          const { encoding, format } = this.config;
          if (!this.ws) {
          const isEmptyData = this.getIsEmptyData(frameBuffer, 200, 20);
          ......@@ -143,7 +149,7 @@ class PandaRecordWebSocket extends PandaWebSocket {
          console.log('系统未检测到输入');
          return;
          }
          if(this.status == "paused") return
          if (this.status == 'paused') return;
          this.initFrameBuffer = frameBuffer;
          this.resultText = '';
          this.resultTextTemp = '';
          ......@@ -190,8 +196,6 @@ class PandaRecordWebSocket extends PandaWebSocket {
          this.destroy();
          }
          }
          }
          reconnect() {
          ......@@ -239,24 +243,24 @@ class PandaRecordWebSocket extends PandaWebSocket {
          language: 'zh_cn',
          domain: 'iat',
          accent: 'mandarin',
          vad_eos: vad_eos,
          vad_eos,
          },
          data,
          };
          this.ws.send(JSON.stringify(params));
          this.startTime = new Date()
          this.startTime = new Date();
          }
          wsOnmessage(e) {
          this.startTime = new Date()
          this.startTime = new Date();
          super.wsOnmessage(e);
          const jsonData = JSON.parse(e.data);
          if (jsonData.data && jsonData.data.result) {
          let data = jsonData.data.result;
          const data = jsonData.data.result;
          let str = '';
          let ws = data.ws;
          const { ws } = data;
          for (let i = 0; i < ws.length; i++) {
          str = str + ws[i].cw[0].w;
          str += ws[i].cw[0].w;
          }
          // 开启wpgs会有此字段(前提：在控制台开通动态修正功能)
          // 取值为 "apd"时表示该片结果是追加到前面的最终结果；取值为"rpl" 时表示替换前面的部分结果，替换范围为rg字段
          ......@@ -271,7 +275,7 @@ class PandaRecordWebSocket extends PandaWebSocket {
          } else {
          this.resultText = this.resultText + str;
          }
          if(this.resultTextTemp) {
          if (this.resultTextTemp) {
          this.events.frame({
          resultTextTemp: this.resultTextTemp,
          resultText: this.resultText,
          ......@@ -281,8 +285,8 @@ class PandaRecordWebSocket extends PandaWebSocket {
          }
          }
          if (jsonData.code == 0 && jsonData.data.status == 2) {
          const resultText = this.resultTextTemp ? this.resultTextTemp : this.resultText
          if(!resultText) return
          const resultText = this.resultTextTemp ? this.resultTextTemp : this.resultText;
          if (!resultText) return;
          this.events.finish({
          resultText,
          });
          ......@@ -294,10 +298,10 @@ class PandaRecordWebSocket extends PandaWebSocket {
          msg: FAIL_TYPE_DATA_ERRRO_MSG,
          data: {
          code: jsonData.code,
          msg : jsonData.message
          msg: jsonData.message,
          },
          });
          console.log(jsonData)
          console.log(jsonData);
          this.destroy();
          }
          }
          ......
          src/components/SoundAi/core/directive/CloseMenuDirective.js
          View file @ 9a13c9d2
          import BaseDirective from "./BaseDirective";
          import {store , event} from '@wisdom-utils/utils'
          import Utils from "../utils/Utils";
          import { store, event } from '@wisdom-utils/utils';
          import BaseDirective from './BaseDirective';
          import Utils from '../utils/Utils';
          class CloseMenuDirective extends BaseDirective {
          static name="关闭菜单或者模糊匹配关闭菜单"
          match () {
          const { text } = this
          let result = null
          static name = '关闭菜单或者模糊匹配关闭菜单';
          if(this.getBeforeLastGroupDirective()) {
          result = text.match(/第([一|二|三|四|五|六|七|八|九|十]+)个/)
          if(result && result.length >= 2) {
          this.matchResult = Utils.chineseToNumber(result[1])
          } else if(!result) {
          result = text.match(/第[\u4e00-\u9fa5]?(\d+)个/)
          if(result && result.length >= 2) {
          this.matchResult = Number(result[1])
          }
          }
          }
          match() {
          const { text } = this;
          let result = null;
          result = text.match(/关闭(.*)(?=(?:菜单|功能|路径))/)
          if (!result) {
          result = text.match(/关闭(.*)(?=(?:,|，|。))/)
          }
          if (result && result.length >= 2 && !this.matchResult) {
          this.matchResult = result[1]
          if (this.getBeforeLastGroupDirective()) {
          result = text.match(/第([一|二|三|四|五|六|七|八|九|十]+)个/);
          if (result && result.length >= 2) {
          this.matchResult = Utils.chineseToNumber(result[1]);
          } else if (!result) {
          result = text.match(/第[\u4e00-\u9fa5]?(\d+)个/);
          if (result && result.length >= 2) {
          this.matchResult = Number(result[1]);
          }
          return !!this.matchResult
          }
          }
          getBeforeLastGroupDirective() {
          const lastMenu = this.directives.at(-1)
          if(!lastMenu) return false
          if(!(lastMenu instanceof CloseMenuDirective) ) return false
          if(!lastMenu.data || lastMenu.data.type != "widgets" || lastMenu.data.widgets.length == 0) return false
          return true
          result = text.match(/关闭(所有).*(?=(?:,|，|。))/);
          if (!result) {
          result = text.match(/关闭(当前).*(?=(?:,|，|。))/);
          }
          async excute () {
          let widgets = null
          if(typeof this.matchResult == "number") {
          const widgets = this.getOpenedMenus()
          if(this.matchResult == 0 || this.matchResult > widgets.length) {
          this.data = {}
          this.data.type = "widgets"
          this.data.widgets = widgets
          return `索引超出了界限，请重新语音选择关闭第几个菜单`
          } else {
          const menu = widgets[this.matchResult]
          event.emit("event:closeMenuByName", menu.name)
          return `关闭${menu.name}菜单成功`
          }
          }
          else {
          widgets = this.getIsOneWdiget()
          if(!widgets) {
          widgets = this.getCompleteMenus()
          }
          if(!widgets) {
          return "已打开菜单中没有查找到此菜单,请检查语音是否正确"
          }
          if(widgets.length == 1) {
          event.emit("event:closeMenuByName", widgets[0].name)
          return `关闭${widgets[0].name}菜单成功`
          } else {
          this.data = {}
          this.data.type = "widgets"
          this.data.widgets = widgets
          return this.widgetsToMsg(widgets)
          }
          }
          if (!result) {
          result = text.match(/关闭(.*)(?=(?:菜单|功能|路径))/);
          }
          widgetsToMsg (widgets) {
          return `有以下${widgets.length}个菜单打开\n${widgets.map((widget, index) => `${index + 1}.${widget.name}`).join("\n")},\n请语音确认一下关闭第几个菜单`
          if (!result) {
          result = text.match(/关闭(.*)(?=(?:,|，|。))/);
          }
          getCompleteMenus () {
          const widgets = this.getOpenedMenus()
          const reg = new RegExp(this.matchResult)
          const results = widgets.filter(i => reg.test(i.name))
          return results.length > 0 ? results : null
          if (result && result.length >= 2 && !this.matchResult) {
          this.matchResult = result[1];
          }
          return !!this.matchResult;
          }
          getBeforeLastGroupDirective() {
          const lastMenu = this.directives.at(-1);
          if (!lastMenu) return false;
          if (!(lastMenu instanceof CloseMenuDirective)) return false;
          if (!lastMenu.data || lastMenu.data.type != 'widgets' || lastMenu.data.widgets.length == 0) return false;
          return true;
          }
          getIsOneWdiget () {
          const widgets = this.getOpenedMenus()
          const widget = widgets.find(i => i.name == this.matchResult)
          return widget ? [widget] : null
          async excute() {
          let widgets = null;
          if (typeof this.matchResult === 'number') {
          const widgets = this.getOpenedMenus();
          if (this.matchResult == 0 || this.matchResult > widgets.length) {
          this.data = {};
          this.data.type = 'widgets';
          this.data.widgets = widgets;
          return `索引超出了界限，请重新语音选择关闭第几个菜单`;
          }
          const menu = widgets[this.matchResult];
          event.emit('event:closeMenuByName', menu.name);
          return `关闭${menu.name}菜单成功`;
          }
          if (this.matchResult == '所有') {
          return this.closeAllMenu();
          }
          if (this.matchResult == '当前') {
          return this.closeCurrentMenu();
          }
          widgets = this.getIsOneWdiget();
          if (!widgets) {
          widgets = this.getCompleteMenus();
          }
          if (!widgets) {
          return '已打开菜单中没有查找到此菜单,请检查语音是否正确';
          }
          if (widgets.length == 1) {
          event.emit('event:closeMenuByName', widgets[0].name);
          return `关闭${widgets[0].name}菜单成功`;
          }
          this.data = {};
          this.data.type = 'widgets';
          this.data.widgets = widgets;
          return this.widgetsToMsg(widgets);
          }
          widgetsToMsg(widgets) {
          return `有以下${widgets.length}个菜单打开\n${widgets
          .map((widget, index) => `${index + 1}.${widget.name}`)
          .join('\n')},\n请语音确认一下关闭第几个菜单`;
          }
          closeAllMenu() {
          event.emit('event:closeAllMenu', true);
          return `成功关闭所有菜单`;
          }
          getOpenedMenus() {
          const menus = store.get("event:openedMenus")
          return Array.from(menus.current.values())
          closeCurrentMenu() {
          console.log(store);
          console.log(window.location);
          const menus = this.getOpenedMenus();
          const menu = location.pathname.match(/\/civbase(.*)/);
          if (!menu || menu.length <= 1) {
          return '系统发生匹配错误';
          }
          const currentMenu = menus.find(item => item.location.pathname == decodeURI(menu[1]));
          if (!currentMenu) {
          return '系统发生异常';
          }
          const isHome = currentMenu.name == '首页';
          if (isHome) {
          return `当前菜单为首页，不能被关闭`;
          }
          event.emit('event:closeMenuByName', currentMenu.name);
          return `关闭${currentMenu.name}菜单成功`;
          }
          getCompleteMenus() {
          const widgets = this.getOpenedMenus();
          const reg = new RegExp(this.matchResult);
          const results = widgets.filter(i => reg.test(i.name));
          return results.length > 0 ? results : null;
          }
          getIsOneWdiget() {
          const widgets = this.getOpenedMenus();
          const widget = widgets.find(i => i.name == this.matchResult);
          return widget ? [widget] : null;
          }
          getOpenedMenus() {
          const menus = store.get('event:openedMenus');
          return Array.from(menus.current.values());
          }
          }
          export default CloseMenuDirective
          \ No newline at end of file
          export default CloseMenuDirective;
          Markdown is supported
          0% or
          You are about to add 0 people to the discussion. Proceed with caution.
          Finish editing this message first!
          Please register or to comment