JS实时麦克风录音并通过WebSocket实时传递到后台

短命女 2022-08-30 11:55 742阅读 0赞

# JS实时麦克风录音并通过WebSocket将pcm传到后台并处理 #

### 文章目录 ###

*  JS实时麦克风录音并通过WebSocket将pcm传到后台并处理
 *   *  前端
     *  后端websoket接收
     *  主要思路
     *  附录

## 前端 ##

<html>
    
    <head>
        <meta charset="UTF-8">
        <title>Simple Recorder.js demo with record, stop and pause</title>
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        
        <style type="text/css">
            .comments {
        
                width: 100%; /*自动适应父布局宽度*/
                overflow: auto;
                word-break: break-all;
                /*在ie中解决断行问题(防止自动变为在一行显示，主要解决ie兼容问题，ie8中当设宽度为100%时，文本域类容超过一行时，
                当我们双击文本内容就会自动变为一行显示，所以只能用ie的专有断行属性“word-break或word-wrap”控制其断行)*/
            }
        </style>
    </head>
    
    <body>
    <div id="controls">
        <button id="recordButton">Record</button>
        <button id="stopButton">Stop</button>
    </div>
    
    <textarea id="textResult" class="comments" rows="10" cols="10"></textarea>
    
    </body>
    <script type="text/javascript" src="./js/recorder3.js"></script>
    <script>
    
        var ws = null; //实现WebSocket
    
        var interval; // 定时器
    
        let recorder = new Recorder({
        
            sampleBits: 16,                 // 采样位数，支持 8 或 16，默认是16
            sampleRate: 16000,              // 采样率，支持 11025、16000、22050、24000、44100、48000，根据浏览器默认值，
            numChannels: 1,                 // 声道，支持 1 或 2， 默认是1
            // compiling: false,(0.x版本中生效,1.x增加中)  // 是否边录边转换，默认是false
            compiling: true
        });
    
        var recordButton = document.getElementById("recordButton");
        var stopButton = document.getElementById("stopButton");
        var textResult = document.getElementById("textResult");
    
        recordButton.addEventListener("click", startRecording);
        stopButton.addEventListener("click", stopRecording);
    
        // 录音
        function startRecording() {
        
            console.log("recordButton clicked");
            recorder.start().then(() => {
        
                // 开始录音
                useWebSocket();
            }, (error) => {
        
                // 出错了
                console.log(`出错了`);
            });
    
        }
    
        // 停止录音
        function stopRecording() {
        
            console.log("stopButton clicked", recorder.getPCMBlob());
    
            recorder.stop();
    
            if (ws) {
        
                ws.close();
            }
    
            clearInterval(interval);
    
            textResult.innerText = '';
    
            // recorder.getPCMBlob();
            // recorder.downloadPCM('aaa');
    
        }
    
        /*
        * WebSocket
        */
        function useWebSocket() {
        
            // console.log(recorder.getNextData())
            ws = new WebSocket("ws://" + window.location.host + "/websocket/chat/audio");
    
            ws.binaryType = 'arraybuffer'; //传输的是 ArrayBuffer 类型的数据
            ws.onopen = function () {
        
                console.log('握手成功');
                if (ws.readyState === 1) {
         //ws进入连接状态，则每隔500毫秒发送一包数据
                    interval = setInterval(() => {
        
                        // recorder.getNextData();
                        // recorder.getWholeData();
                        // console.log(recorder.getNextData()); 可以将pcm 打印出来，查看pcm 的正确性
                        ws.send(recorder.getNextData());
                    }, 1000)
    
                }
    
            };
    
            ws.onmessage = function (msg) {
        
                var jsonStr = msg.data;
                var json = JSON.parse(jsonStr);
                textResult.innerText = json.msg;
                autoTextarea(document.getElementById("textResult"));
            };
    
            ws.onerror = function (err) {
        
                console.error(err);
                textResult.innerText = '';
            };
    
            ws.onclose = function (msg) {
        
                console.info(msg);
                textResult.innerText = '';
            };
    
        }
    
        /**
         * 文本框根据输入内容自适应高度
         * @param                {HTMLElement}        输入框元素
         * @param                {Number}                设置光标与输入框保持的距离(默认0)
         * @param                {Number}                设置最大高度(可选)
         */
        var autoTextarea = function (elem, extra, maxHeight) {
        
            //判断elem是否为数组
            if (elem.length > 0) {
        
                for (var i = 0; i < elem.length; i++) {
        
                    e(elem[i]);
                }
            } else {
        
                e(elem);
            }
    
            function e(elem) {
        
                extra = extra || 0;
                var isFirefox = !!document.getBoxObjectFor || 'mozInnerScreenX' in window,
                    isOpera = !!window.opera && !!window.opera.toString().indexOf('Opera'),
                    addEvent = function (type, callback) {
        
                        elem.addEventListener ?
                            elem.addEventListener(type, callback, false) :
                            elem.attachEvent('on' + type, callback);
                    },
                    getStyle = elem.currentStyle ? function (name) {
        
                        var val = elem.currentStyle[name];
    
                        if (name === 'height' && val.search(/px/i) !== 1) {
        
                            var rect = elem.getBoundingClientRect();
                            return rect.bottom - rect.top -
                                parseFloat(getStyle('paddingTop')) -
                                parseFloat(getStyle('paddingBottom')) + 'px';
                        }
                        ;
    
                        return val;
                    } : function (name) {
        
                        return getComputedStyle(elem, null)[name];
                    },
                    minHeight = parseFloat(getStyle('height'));
    
                elem.style.resize = 'none';
    
                var change = function () {
        
                    var scrollTop, height,
                        padding = 0,
                        style = elem.style;
    
                    if (elem._length === elem.value.length) return;
                    elem._length = elem.value.length;
    
                    if (!isFirefox && !isOpera) {
        
                        padding = parseInt(getStyle('paddingTop')) + parseInt(getStyle('paddingBottom'));
                    }
                    ;
                    scrollTop = document.body.scrollTop || document.documentElement.scrollTop;
    
                    elem.style.height = minHeight + 'px';
                    if (elem.scrollHeight > minHeight) {
        
                        if (maxHeight && elem.scrollHeight > maxHeight) {
        
                            height = maxHeight - padding;
                            style.overflowY = 'auto';
                        } else {
        
                            height = elem.scrollHeight - padding;
                            style.overflowY = 'hidden';
                        }
                        ;
                        style.height = height + extra + 'px';
                        scrollTop += parseInt(style.height) - elem.currHeight;
                        document.body.scrollTop = scrollTop;
                        document.documentElement.scrollTop = scrollTop;
                        elem.currHeight = parseInt(style.height);
                    }
                    ;
                };
    
                addEvent('propertychange', change);
                addEvent('input', change);
                addEvent('focus', change);
                change();
            }
        };
    
    </script>
    </html>

## 后端websoket接收 ##

@Override
        @OnMessage(maxMessageSize = 10000000)
        public void onMessage(ByteBuffer message) {
        
    
            // 为空时 不处理
            if (ObjectUtil.isEmpty(message)) {
        
                return;
            }
    
            CompletableFuture.runAsync(() -> {
        
    
                String userName = this.getUserName();
    
                // 整体录音存储
                appendBuffer(message);
    
                // 休眠  防止过快
                try {
        
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
        
                    e.printStackTrace();
                }
    
                // 临时 存储
                ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
                byte[] bytes = message.array();
                try {
        
                    byteArrayOutputStream.write(bytes);
                } catch (IOException e) {
        
                    try {
        
                        byteArrayOutputStream.close();
                    } catch (IOException ex) {
        
                        ex.printStackTrace();
                    }
                    e.printStackTrace();
                    return;
                }
    
                // 内存处理 pcm转wav
                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                PcmCovWavUtil.convertWaveFile(byteArrayOutputStream, outputStream);
    
                wav waveFile = new wav();
                waveFile.GetFromBytes(outputStream.toByteArray());
    
                try {
        
                    // 整体音频存储
                    ByteArrayOutputStream stream = byteArrayOutputStreamConcurrentHashMap.get(userName);
                    if (ObjectUtil.isNotEmpty(stream)) {
        
                        String fileName = "d://temp_" + userName + ".wav";
                        FileOutputStream fileOutputStream = new FileOutputStream(new File(fileName));
                        IoUtil.copy(new ByteArrayInputStream(stream.toByteArray()), fileOutputStream);
                        fileOutputStream.flush();
                        fileOutputStream.close();
                    }
                } catch (Exception e) {
        
                    e.printStackTrace();
                }
    
                String result = ApiUtils.sendToServer("qwertasd", waveFile.fs, waveFile.samples);
                StringBuffer stringBuffer = appendResult(result);
    
                // 转写结果 发送
                Message msg = new Message(getUserName(), Message.MsgConstant.MSG_TO_ALL, stringBuffer.toString());
                super.onMessage(msg.toString());
    
            }, threadPoolExecutor);
        }

## 主要思路 ##

*  前端链接后端websoket服务
 *  前端实时将pcm音频流传输到后端
 *  后端定义静态变量，合并接收pcm片段
 *  将合并的pcm流任意转换为wav、mp3等

完整项目参见：[https://gitee.com/yzdyzdyzd/speechToText][https_gitee.com_yzdyzdyzd_speechToText]

## 附录 ##

*  正确pcm 格式  
    ![在这里插入图片描述][c48944ccab460dd38114e8e74f3ea33d.png_pic_center]

[https_gitee.com_yzdyzdyzd_speechToText]: https://gitee.com/yzdyzdyzd/speechToText
[c48944ccab460dd38114e8e74f3ea33d.png_pic_center]: /images/20220829/1577fb208a374e76b5064a69e682e1fc.png