danieloquelis · Dev-develope · Jun 12, 2026
diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs b/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs
@@ -0,0 +1,58 @@
+using System;
+using UnityEngine;
+
+public static class ImageEncodingUtil
+{
+    /// <summary>
+    /// Captures a frame from a <see cref="WebCamTexture"/>, downsizes it to fit within maxSize while preserving
+    /// aspect ratio, and returns a data URL (PNG or JPEG) as a string. Must be called on the main thread.
+    /// </summary>
+    public static string CaptureDataUrlFromWebCam(WebCamTexture cam, int maxSize, bool useJpeg, int jpegQuality)
+    {
+        if (cam == null || !cam.isPlaying) return string.Empty;
+
+        var srcW = cam.width;
+        var srcH = cam.height;
+        if (srcW <= 0 || srcH <= 0) return string.Empty;
+
+        var scale = 1f;
+        var maxDim = Mathf.Max(srcW, srcH);
+        if (maxDim > maxSize)
+        {
+            scale = (float)maxSize / maxDim;
+        }
+
+        var dstW = Mathf.Max(1, Mathf.RoundToInt(srcW * scale));
+        var dstH = Mathf.Max(1, Mathf.RoundToInt(srcH * scale));
+
+        RenderTexture rt = null;
+        Texture2D tex = null;
+        try
+        {
+            rt = new RenderTexture(dstW, dstH, 0, RenderTextureFormat.ARGB32);
+            Graphics.Blit(cam, rt);
+
+            var prev = RenderTexture.active;
+            RenderTexture.active = rt;
+            tex = new Texture2D(dstW, dstH, TextureFormat.RGBA32, false);
+            tex.ReadPixels(new Rect(0, 0, dstW, dstH), 0, 0);
+            tex.Apply();
+            RenderTexture.active = prev;
+
+            var bytes = useJpeg ? tex.EncodeToJPG(jpegQuality) : tex.EncodeToPNG();
+            var mime = useJpeg ? "image/jpeg" : "image/png";
+            return $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
+        }
+        finally
+        {
+            if (tex != null) UnityEngine.Object.Destroy(tex);
+            if (rt != null)
+            {
+                rt.Release();
+                UnityEngine.Object.Destroy(rt);
+            }
+        }
+    }
+}
+
+
diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs
@@ -0,0 +1,142 @@
+using System;
+using UnityEngine;
+
+/// <summary>
+/// Streams microphone audio as Base64-encoded 16-bit mono PCM chunks.
+/// Each chunk is 1,024 samples (~64 ms at 16 kHz).
+/// </summary>
+public class MicrophoneStreamer : MonoBehaviour
+{
+    public Action<string> OnAudioChunk;
+
+    private const int SampleRateOut   = 16_000; 
+    private const int ChunkSamplesOut = 1_024;
+
+    private AudioClip _microphoneClip;
+    private string _micDevice;
+    private int _micSampleRate;   
+    private int _chunkSamplesIn;  
+    private int _lastSamplePos;
+
+    #region Unity Lifecycle
+
+    private void Start()
+    {
+        if (Microphone.devices.Length == 0)
+        {
+            Debug.LogError("MicrophoneStreamer: no microphone devices.");
+            enabled = false;
+            return;
+        }
+
+        _micDevice = Microphone.devices[0];
+    }
+
+    private void Update()
+    {
+        if (!_microphoneClip) return;
+
+        var currentPos       = Microphone.GetPosition(_micDevice);
+        var samplesAvailable = currentPos - _lastSamplePos;
+        if (samplesAvailable < 0) 
+            samplesAvailable += _microphoneClip.samples;
+
+        if (samplesAvailable < _chunkSamplesIn) return;
+
+        var inBuf = new float[_chunkSamplesIn];
+        ReadCircular(_microphoneClip, _lastSamplePos, inBuf);
+        _lastSamplePos = (_lastSamplePos + _chunkSamplesIn) % _microphoneClip.samples;
+        var pcm16 = DownsampleAndConvert(inBuf, _micSampleRate, SampleRateOut);
+        OnAudioChunk?.Invoke(Convert.ToBase64String(pcm16));
+    }
+
+    #endregion
+
+    #region Public Methods
+
+    public void StartStreaming()
+    {
+        _microphoneClip = Microphone.Start(_micDevice, true, 1, SampleRateOut);
+        _micSampleRate  = _microphoneClip.frequency;
+        _chunkSamplesIn = Mathf.RoundToInt(ChunkSamplesOut * (float)_micSampleRate / SampleRateOut);
+        _lastSamplePos  = 0;
+
+        Debug.Log($"[MicrophoneStreamer] device={_micDevice}, " +
+                  $"realRate={_micSampleRate} Hz, chunkIn={_chunkSamplesIn} samples");
+    }
+
+    public void StopStreaming()
+    {
+        if (Microphone.IsRecording(_micDevice))
+            Microphone.End(_micDevice);
+
+        _microphoneClip = null;
+    }
+
+    #endregion
+
+    #region Helpers
+
+    private static void ReadCircular(AudioClip clip, int start, float[] buffer)
+    {
+        var len         = buffer.Length;
+        var clipSamples = clip.samples;
+        var tail        = clipSamples - start;
+
+        if (len <= tail)
+        {
+            clip.GetData(buffer, start);
+        }
+        else
+        {
+            var tempTail = new float[tail];
+            var tempHead = new float[len - tail];
+
+            clip.GetData(tempTail, start);
+            clip.GetData(tempHead, 0);
+
+            Array.Copy(tempTail, 0, buffer, 0, tail);
+            Array.Copy(tempHead, 0, buffer, tail, tempHead.Length);
+        }
+    }
+
+    private static byte[] DownsampleAndConvert(float[] inBuf, int inRate, int outRate)
+    {
+        if (inRate == outRate) 
+            return ConvertToPcm16(inBuf);
+
+        var ratio   = (float)inRate / outRate;
+        var outLen    = Mathf.RoundToInt(inBuf.Length / ratio);
+        var pcmOut = new byte[outLen * 2];
+
+        var pos = 0f;
+        for (var o = 0; o < outLen; o++, pos += ratio)
+        {
+            var i0 = Mathf.Clamp((int)pos, 0, inBuf.Length - 1);
+            var i1 = Mathf.Min(i0 + 1, inBuf.Length - 1);
+            var frac = pos - i0;
+
+            var sample = Mathf.Lerp(inBuf[i0], inBuf[i1], frac);
+            var s16 = (short)Mathf.Clamp(sample * 32767f, short.MinValue, short.MaxValue);
+
+            pcmOut[o * 2]     = (byte)(s16 & 0xFF);
+            pcmOut[o * 2 + 1] = (byte)((s16 >> 8) & 0xFF);
+        }
+
+        return pcmOut;
+    }
+
+    private static byte[] ConvertToPcm16(float[] buf)
+    {
+        var pcm = new byte[buf.Length * 2];
+        for (var i = 0; i < buf.Length; i++)
+        {
+            var s = (short)Mathf.Clamp(buf[i] * 32767f, short.MinValue, short.MaxValue);
+            pcm[i * 2]     = (byte)(s & 0xFF);
+            pcm[i * 2 + 1] = (byte)((s >> 8) & 0xFF);
+        }
+        return pcm;
+    }
+
+    #endregion
+}
diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs
@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+using UnityEngine;
+using UnityEngine.Assertions;
+
+/// <summary>
+/// Handles playback of PCM audio received as Base64-encoded strings.
+/// Uses an internal queue to play audio clips sequentially through an AudioSource.
+/// </summary>
+public class PcmAudioPlayer : MonoBehaviour
+{
+    [SerializeField] private AudioSource audioSource;
+    [SerializeField] private int sampleRate = 16000;
+
+    private readonly Queue<AudioClip> _clipQueue = new();
+
+    #region Unity Lifecycle
+
+    private void Start()
+    {
+        Assert.IsNotNull(audioSource, "Audio source is required");
+    }
+
+    private void Update()
+    {
+        if (audioSource.isPlaying || _clipQueue.Count == 0)
+            return;
+
+        audioSource.clip = _clipQueue.Dequeue();
+        audioSource.Play();
+    }
+
+    #endregion
+
+    #region Public Methods
+
+    /// <summary>
+    /// Converts Base64-encoded PCM audio into a Unity AudioClip 
+    /// and adds it to the playback queue.
+    /// </summary>
+    /// <param name="base64Audio">Base64-encoded PCM16 audio data.</param>
+    public void EnqueueBase64Audio(string base64Audio)
+    {
+        // Decode Base64 string into raw bytes
+        var bytes = System.Convert.FromBase64String(base64Audio);
+
+        // Each PCM16 sample uses 2 bytes
+        var sampleCount = bytes.Length / 2;
+        var samples = new float[sampleCount];
+
+        for (var i = 0; i < sampleCount; i++)
+        {
+            var sample = (short)((bytes[i * 2 + 1] << 8) | bytes[i * 2]);
+            samples[i] = sample / 32768f;
+        }
+
+        var clip = AudioClip.Create("AIConversationalClip", sampleCount, 1, sampleRate, false);
+        clip.SetData(samples, 0);
+
+        _clipQueue.Enqueue(clip);
+    }
+
+    /// <summary>
+    /// Stops playback immediately and clears any queued audio.
+    /// </summary>
+    public void StopImmediately()
+    {
+        _clipQueue.Clear();
+        audioSource.Stop();
+    }
+
+    #endregion
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs
@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using UnityEngine;
+using UnityEngine.Networking;
+
+namespace SixtyDb
+{
+    // REST client for POST /v1/chat/completions. OpenAI-compatible payload,
+    // so the tool array is passed through verbatim.
+    //
+    // Returns a small (content, toolCalls) tuple; SixtyDbConvManager owns the
+    // tool-execution loop (dispatch → tool message → re-call until content).
+    public sealed class SixtyDbChatClient
+    {
+        private readonly SixtyDbConfig _config;
+
+        public SixtyDbChatClient(SixtyDbConfig config)
+        {
+            _config = config ?? throw new ArgumentNullException(nameof(config));
+        }
+
+        public struct Result
+        {
+            public string Content;
+            public List<ToolCall> ToolCalls;
+        }
+
+        public struct ToolCall
+        {
+            public string Id;
+            public string Name;
+            public string ArgumentsJson;
+        }
+
+        public async Task<Result> CompleteAsync(
+            List<JObject> messages,
+            JArray tools,
+            string model = null,
+            float temperature = 0.7f,
+            int topK = 20)
+        {
+            if (string.IsNullOrWhiteSpace(_config.apiKey))
+                throw new InvalidOperationException("60db apiKey is not set in the config asset.");
+
+            var body = new JObject
+            {
+                ["model"]       = model ?? _config.llmModel,
+                ["messages"]    = new JArray(messages),
+                ["stream"]      = false,
+                ["temperature"] = temperature,
+                ["top_k"]       = topK,
+                ["chat_template_kwargs"] = new JObject { ["enable_thinking"] = false },
+            };
+            if (tools != null && tools.Count > 0) body["tool"] = tools;
+
+            var url = $"{_config.apiBase.TrimEnd('/')}/v1/chat/completions";
+            using var req = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPOST);
+            var raw = Encoding.UTF8.GetBytes(body.ToString(Formatting.None));
+            req.uploadHandler = new UploadHandlerRaw(raw) { contentType = "application/json" };
+            req.downloadHandler = new DownloadHandlerBuffer();
+            req.SetRequestHeader("Authorization", $"Bearer {_config.apiKey}");
+            req.SetRequestHeader("Accept", "application/json");
+
+            var op = req.SendWebRequest();
+            while (!op.isDone) await Task.Yield();
+
+            if (req.result != UnityWebRequest.Result.Success)
+                throw new InvalidOperationException(
+                    $"[SixtyDbChatClient] HTTP {(int)req.responseCode}: {req.error} / {req.downloadHandler?.text}");
+
+            var json = JObject.Parse(req.downloadHandler.text);
+            var msg = json["choices"]?[0]?["message"] as JObject ?? new JObject();
+
+            var result = new Result
+            {
+                Content = msg["content"]?.ToString() ?? string.Empty,
+                ToolCalls = new List<ToolCall>(),
+            };
+
+            if (msg["tool_calls"] is JArray toolCalls)
+            {
+                foreach (var tc in toolCalls)
+                {
+                    if (tc is not JObject obj) continue;
+                    var fn = obj["function"] as JObject;
+                    result.ToolCalls.Add(new ToolCall
+                    {
+                        Id            = obj["id"]?.ToString(),
+                        Name          = fn?["name"]?.ToString(),
+                        ArgumentsJson = fn?["arguments"]?.ToString() ?? "{}",
+                    });
+                }
+            }
+
+            return result;
+        }
+
+        // Convenience: build a {role, content} message in the shape /v1/chat/completions expects.
+        public static JObject UserMessage(string text)   => new() { ["role"] = "user",   ["content"] = text };
+        public static JObject SystemMessage(string text) => new() { ["role"] = "system", ["content"] = text };
+        public static JObject AssistantMessage(string text) => new() { ["role"] = "assistant", ["content"] = text };
+        public static JObject ToolMessage(string toolCallId, string name, string resultJson) =>
+            new() { ["role"] = "tool", ["tool_call_id"] = toolCallId, ["name"] = name, ["content"] = resultJson };
+    }
+}