数字人系列四：Motionverse 接入chatgpt、文心一言等国内外大语言模型

1. 下载插件：motionverse官网地址：概述 · Motionverse 接口文档 (deepscience.cn)

2. 按照官方文档新建Unity工程：对接说明 · Motionverse 接口文档 (deepscience.cn)

3. 通过我们自己的ASR，将语音转换为文本，文本通过大语言模型（chatgpt、文心一言以及其他大语言模型）生成的结果，直接通过生成式AI技术，把文本转化为AI智能体的声音、动作和表情，和大语言模型完美连接，只需要在获取到文本的时候调用以下代码即可：
代码示例：

DriveTask task = new DriveTask();
task.player = player;
task.text = question;
NLPDrive.GetDrive(task);

其中，player即为挂载motionverse插件中Player脚本的对象，question即为大语言模型获取到的答案。

还可以自己生成语音，通过语音链接调用以下代码：

DriveTask task = new DriveTask();

task.player = player;

task.text = audioUrl;

AudioUrlDrive.GetDrive(task);

其中，player即为挂载motionverse插件中Player脚本的对象，audioUrl即为语音链接。

4. 新建脚本AskManager，并挂载到场景中（可新建空物体），脚本代码如下：

using LitJson;
using Motionverse;
using MotionverseSDK;
using System;
using System.Collections;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;
using UnityEngine.UI;

public class AskManager : MonoBehaviour
{
    public Player player;
    public Dropdown dropdown;
    public InputField inputField;
    public Button btnSend;

    public string chatGptKey = "";
    private string chatUrl = "https://chatgpt.kazava.io/v1/chat/completions";

    public string wenXinAPI = "";
    public string wenXinSECRET = "";
    private string wenXinToken = "";
    private int curSelectNLP = 0;

    // Start is called before the first frame update
    private void Awake()
    {
        for (int i = 0; i < Display.displays.Length; i++)
        {
            Display.displays[i].Activate();
        }

    }
    void Start()
    {
        dropdown.onValueChanged.AddListener((value) =>
        {
            curSelectNLP = value;
        });
        StartCoroutine(GetWenxinToken());
        btnSend.onClick.AddListener(() =>
        {
            if (string.IsNullOrEmpty(inputField.text))
            {
                Debug.Log("请输入内容！");
            }
            else
            {
                GetAnswer(inputField.text);
            }
        });
    }

    public void GetAnswer(string q)
    {
        StartCoroutine(RealAnswer(q));
    }



    public IEnumerator RealAnswer(string question)
    {
        switch (curSelectNLP)
        {
            case 0:
                DriveTask task = new DriveTask();
                task.player = player;
                task.text = question;
                NLPDrive.GetDrive(task);
                break;
            case 1:
                StartCoroutine(RequestChat(question));
                break;
            case 2:
                StartCoroutine(ChatCompletions(question));
                break;
            default:
                break;
        }

        Invoke("restartRecording", 1);

        yield return null;

    }

    IEnumerator GetWenxinToken()
    {
        string url =$"https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id={wenXinAPI}&client_secret={wenXinSECRET}";
        UnityWebRequest webRequest = UnityWebRequest.Get(url);
        webRequest.timeout = 5000;

        yield return webRequest.SendWebRequest();

        if (webRequest.result == UnityWebRequest.Result.Success)
        {
            string response = webRequest.downloadHandler.text;
            var result = JsonUtility.FromJson<AccessTokenResponse>(response);
            wenXinToken = result.access_token;
        }
        else
        {
            Debug.LogError("Failed to get access token: " + webRequest.error);
        }
    }

    IEnumerator ChatCompletions(string content)
    {
        string url = $"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={wenXinToken}";
        WenXinPostData postData = new WenXinPostData();
        postData.messages = new WenXinMessage[1];
        postData.messages[0] = new WenXinMessage();
        postData.messages[0].content = content + "30字以内";
        string data = JsonMapper.ToJson(postData);

        UnityWebRequest webRequest = UnityWebRequest.Post(url, data, "application/json");
        webRequest.timeout = 5000;
        yield return webRequest.SendWebRequest();

        if (webRequest.result == UnityWebRequest.Result.Success)
        {
            string response = webRequest.downloadHandler.text;
            WenXinRequest requestData = JsonMapper.ToObject<WenXinRequest>(response);
            DriveTask task = new DriveTask();
            task.player = player;
            task.text = requestData.result;
            TextDrive.GetDrive(task);
        }
        else
        {
            Debug.LogError("Chat completions request failed: " + webRequest.error);
        }
    }

    private IEnumerator RequestChat(string content)
    {
        using (UnityWebRequest webRequest = new UnityWebRequest(chatUrl, "POST"))
        {
            webRequest.SetRequestHeader("Content-Type", "application/json");
            ChatGPTPostData postData = new ChatGPTPostData();
            postData.key = chatGptKey;
            postData.messages = new PostMessage[1];
            postData.messages[0] = new PostMessage();
            postData.messages[0].content = content + "30字以内"; ;
            string data = JsonMapper.ToJson(postData);
            byte[] jsonToSend = new UTF8Encoding().GetBytes(data);
            webRequest.uploadHandler = new UploadHandlerRaw(jsonToSend);
            webRequest.downloadHandler = new DownloadHandlerBuffer();
            yield return webRequest.SendWebRequest();

            if (webRequest.result != UnityWebRequest.Result.Success)
            {
                Debug.LogError("ChatGPT request error: " + content + webRequest.error);
            }
            else
            {
                string response = webRequest.downloadHandler.text;
                ChatGPTRequestData requestData = JsonMapper.ToObject<ChatGPTRequestData>(response);
                DriveTask task = new DriveTask();
                task.player = player;
                task.text = requestData.choices[0].message.content;
                TextDrive.GetDrive(task);
            }
        }
    }
}

5. 新建脚本RealtimeAsrManager，代码如下：

using System;
using System.Collections;
using UnityEngine;
using UnityEngine.UI;
using UnityWebSocket;


namespace Motionverse
{
    public class RealtimeAsrManager : MonoBehaviour
    {
        private IWebSocket webSocket;
        private Status status = Status.FirstFrame;
        private bool lockReconnect = false;
        [HideInInspector]
        public static Action<string> Asr;

        [SerializeField]
        private GameObject TextBG;
        //百度
        public int AsrAppId;
        public string AsrAppkey = "";


        void Start()
        {
            CreateWebSocket();
            RecorderManager.DataAvailable += OnDataAvailable;
        }
        private void OnDisable()
        {
            RecorderManager.DataAvailable -= OnDataAvailable;
        }
        void CreateWebSocket()
        {
            try
            {
                string uril = "wss://vop.baidu.com/realtime_asr?sn=" + Guid.NewGuid().ToString();
                webSocket = new WebSocket(uril);
                InitHandle();
                webSocket.ConnectAsync();
            }
            catch (Exception e)
            {
                Debug.Log("websocket连接异常:" + e.Message);
                ReConnect();
            }

        }


        private void InitHandle()
        {
            RemoveHandle();
            webSocket.OnOpen += OnOpen;
            webSocket.OnMessage += OnMessage;
            webSocket.OnClose += OnClose;
            webSocket.OnError += OnError;


        }
        void RemoveHandle()
        {
            webSocket.OnOpen -= OnOpen;
            webSocket.OnMessage -= OnMessage;
            webSocket.OnClose -= OnClose;
            webSocket.OnError -= OnError;
        }
        //请求开始
        private void OnDataAvailable(byte[] data)
        {
            if (webSocket == null || (webSocket != null && webSocket.ReadyState != WebSocketState.Open))
                return;

            switch (status)
            {
                case Status.FirstFrame://握手
                    {
                        var firstFrame = new FirstFrame();
                        firstFrame.data.appid= AsrAppId;
                        firstFrame.data.appkey = AsrAppkey;

                        webSocket.SendAsync(JsonUtility.ToJson(firstFrame));
                        status = Status.ContinueFrame;
                    }
                    break;
                case Status.ContinueFrame://开始发送
                    {
                        if (data.Length > 0)
                        {
                            webSocket.SendAsync(data);
                        }
                    }
                    break;
                case Status.LastFrame://关闭
                    {
                        webSocket.SendAsync(JsonUtility.ToJson(new LastFrame()));
                    }
                    break;
                default:
                    break;
            }
        }

        void ReConnect()
        {
            if (this.lockReconnect)
                return;
            this.lockReconnect = true;
            StartCoroutine(SetReConnect());
        }

        private IEnumerator SetReConnect()
        {
            yield return new WaitForSeconds(1);
            CreateWebSocket();
            lockReconnect = false;
        }

        #region WebSocket Event Handlers

        private void OnOpen(object sender, OpenEventArgs e)
        {
            status = Status.FirstFrame;
        }

        private void OnMessage(object sender, MessageEventArgs e)
        {
            var err_msg = Utils.GetJsonValue(e.Data, "err_msg");
            var type = Utils.GetJsonValue(e.Data, "type");
           
            if (err_msg == "OK" && type == "MID_TEXT")
            {
                var result = Utils.GetJsonValue(e.Data, "result");
                TextBG.GetComponentInChildren<Text>().text = result;
            }

            if (err_msg == "OK" && type == "FIN_TEXT")
            {
                var result = Utils.GetJsonValue(e.Data, "result");
                TextBG.GetComponentInChildren<Text>().text = result;
                if (result.Length > 1)
                {
                    RecorderManager.Instance.EndRecording();
                    Asr?.Invoke(result);
                }
               
            }

        }

        private void OnClose(object sender, CloseEventArgs e)
        {
            Debug.Log("websocket关闭," + string.Format("Closed: StatusCode: {0}, Reason: {1}", e.StatusCode, e.Reason));
            webSocket = null;
            ReConnect();
        }

        private void OnError(object sender, ErrorEventArgs e)
        {
            if (e != null)
                Debug.Log("websocket连接异常:" + e.Message);
            webSocket = null;
            ReConnect();
        }

        #endregion
    }
}

6. 新建脚本RecorderManager，代码如下：

using UnityEngine;
using System;
using System.Collections;
using UnityEngine.UI;
using Unity.VisualScripting;
namespace Motionverse
{
    public class RecorderManager : Singleton<RecorderManager>
    {
        //标记是否有麦克风
        private bool isHaveMic = false;
        //当前录音设备名称
        private string currentDeviceName = string.Empty;
        //表示录音的最大时长
        int recordMaxLength = 600;
        //录音频率,控制录音质量(16000)
        int recordFrequency = 16000;
        [HideInInspector]
        public static Action<byte[]> DataAvailable;
        [SerializeField]
        private GameObject micON;
        [SerializeField]
        private GameObject micOFF;
        [SerializeField]
        private Image micAmount;
        [SerializeField]
        private GameObject TextBG;
        private AudioClip saveAudioClip;

        int offsetSamples = 0;
        private void Start()
        {
            Debug.Log(Microphone.devices[0]);
            if (Microphone.devices.Length > 0)
            {
                isHaveMic = true;
                currentDeviceName = Microphone.devices[0];
                StartCoroutine(GetAudioFrames());
                StartRecording();
            }

        }

        /// <summary>
        /// 开始录音
        /// </summary>
        /// <returns></returns>
        public void StartRecording() //16000
        {
            if (isHaveMic == false || Microphone.IsRecording(currentDeviceName))
            {
                return;
            }
            micOFF.gameObject.SetActive(false);
            micON.gameObject.SetActive(true);
            TextBG.GetComponentInChildren<Text>().text = null;
            offsetSamples = 0;
            saveAudioClip = Microphone.Start(currentDeviceName, true, recordMaxLength, recordFrequency);

        }
        public void OnButtonClick()
        {
            if (Microphone.IsRecording(currentDeviceName))
            {
                EndRecording();
            }
            else
            {
                StartRecording();
            }
        }
        public void EndRecording()
        {

            if (isHaveMic == false || !Microphone.IsRecording(currentDeviceName))
            {
                return;
            }
            micOFF.gameObject.SetActive(true);
            micON.gameObject.SetActive(false);
            //结束录音
            Microphone.End(currentDeviceName);
        }
        public bool IsRecording()
        {
            return Microphone.IsRecording(currentDeviceName);
        }
        IEnumerator GetAudioFrames()
        {

            while (true)
            {
                if (Microphone.IsRecording(currentDeviceName))
                {
                    int lenght = Microphone.GetPosition(currentDeviceName) * saveAudioClip.channels - offsetSamples;
                    if (lenght > 0)
                    {
                        float[] samples = new float[lenght];
                        saveAudioClip.GetData(samples, offsetSamples);

                        var samplesShort = new short[samples.Length];
                        for (var index = 0; index < samples.Length; index++)
                        {
                            samplesShort[index] = (short)(samples[index] * short.MaxValue);
                        }
                        byte[] binaryData = new byte[samplesShort.Length * 2];
                        Buffer.BlockCopy(samplesShort, 0, binaryData, 0, binaryData.Length);

                        offsetSamples += lenght;
                        DataAvailable?.Invoke(binaryData);
                    }

                    yield return new WaitForSeconds(0.16f);
                }
                else
                {
                    yield return new WaitForSeconds(0.16f);
                    byte[] binaryData = new byte[2];
                    DataAvailable?.Invoke(binaryData);
                }

            }
        }

        /// <summary>
        /// 获取毫秒级别的时间戳,用于计算按下录音时长
        /// </summary>
        /// <returns></returns>
        public double GetTimestampOfNowWithMillisecond()
        {
            return (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000;
        }

        private void LateUpdate()
        {
            if (isHaveMic == true && Microphone.IsRecording(currentDeviceName))
            {
                micAmount.fillAmount = Volume;
            }
        }
        public float Volume
        {
            get
            {
                if (Microphone.IsRecording(currentDeviceName))
                {
                    // 采样数
                    int sampleSize = 128;
                    float[] samples = new float[sampleSize];
                    int startPosition = Microphone.GetPosition(currentDeviceName) - (sampleSize + 1);
                    // 得到数据
                    if (startPosition < 0)
                        return 0;

                    saveAudioClip.GetData(samples, startPosition);

                    // Getting a peak on the last 128 samples
                    float levelMax = 0;
                    for (int i = 0; i < sampleSize; ++i)
                    {
                        float wavePeak = samples[i];
                        if (levelMax < wavePeak)
                            levelMax = wavePeak;
                    }

                    return levelMax;
                }
                return 0;
            }
        }
        void OnGUI()
        {
            GUIStyle guiStyle = GUIStyle.none;
            guiStyle.fontSize = 10;
            guiStyle.normal.textColor = Color.white;
            guiStyle.alignment = TextAnchor.UpperLeft;
            Rect tr = new Rect(0, 0, 100, 100);
            GUI.Label(tr, currentDeviceName, guiStyle);
        }
    }
}

7. 新建空物体，挂载脚本RecorderManager和RealtimeAsrManager
8. 输入百度asr的appId和secretKey：

9. 输入GPTkey、问心一眼appId和secretKey：

10. 根据需求选择相应的NLP方式：

注意：如果缺少Singleton文件，可使用如下代码：

using UnityEngine;
namespace Motionverse
{
    public abstract class Singleton<T> : MonoBehaviour where T : Singleton<T>
    {

        private static T sInstance = null;

        public static T Instance
        {
            get
            {
                if (sInstance == null)
                {
                    GameObject gameObject = new(typeof(T).FullName);
                    sInstance = gameObject.AddComponent<T>();
                }

                return sInstance;
            }
        }

        public static void Clear()
        {
            sInstance = null;
        }

        protected virtual void Awake()
        {
            if (sInstance != null) Debug.LogError(name + "error: already initialized", this);

            sInstance = (T)this;
        }
    }
}

若有收获，就点个赞吧文章来源地址https://www.toymoban.com/news/detail-694401.html

到了这里，关于数字人系列四：Motionverse 接入chatgpt、文心一言等国内外大语言模型的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！