From 3da278ea2ca684c2f18099d956ca82af242ae9b1 Mon Sep 17 00:00:00 2001 From: Frederick <1814239758@qq.com> Date: Sun, 9 Aug 2026 19:47:26 +0800 Subject: [PATCH 1/3] feat: export Sogou Pinyin backup dictionaries --- src/IME WL Converter Win/Forms/MainForm.cs | 9 +- .../Contracts/IConversionPipeline.cs | 3 + .../Pipeline/ConversionPipeline.cs | 36 ++- .../SougouBin/SougouBinExporter.cs | 255 ++++++++++++++++++ .../SougouBin/SougouBinParser.cs | 2 +- .../PipelineBinaryExportTest.cs | 103 +++++++ .../SougouBinExportTest.cs | 90 +++++++ .../ViewModels/MainWindowViewModel.cs | 10 +- 8 files changed, 495 insertions(+), 13 deletions(-) create mode 100644 src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs create mode 100644 src/ImeWlConverterCoreTest/PipelineBinaryExportTest.cs create mode 100644 src/ImeWlConverterCoreTest/SougouBinExportTest.cs diff --git a/src/IME WL Converter Win/Forms/MainForm.cs b/src/IME WL Converter Win/Forms/MainForm.cs index 39671d61..1bddfc0f 100644 --- a/src/IME WL Converter Win/Forms/MainForm.cs +++ b/src/IME WL Converter Win/Forms/MainForm.cs @@ -56,6 +56,7 @@ public partial class MainForm : Form private int _convertedCount; private string? _exportContent; + private byte[]? _exportData; private CancellationTokenSource? _cts; @@ -382,6 +383,7 @@ private async void btnConvert_Click(object? sender, EventArgs e) richTextBox1.Clear(); _exportContent = null; + _exportData = null; _cts = new CancellationTokenSource(); SetConvertingState(true); @@ -426,6 +428,7 @@ private async void btnConvert_Click(object? sender, EventArgs e) { _convertedCount = result.Value.ExportedCount; _exportContent = result.Value.ExportContent; + _exportData = result.Value.ExportData; HandleConversionCompleted(result.Value); } else @@ -552,7 +555,11 @@ private void HandleConversionCompleted(ConversionResult result) if (saveFileDialog1.ShowDialog() == DialogResult.OK) { - if (_exportContent != null) + if (_exportData != null) + { + File.WriteAllBytes(saveFileDialog1.FileName, _exportData); + } + else if (_exportContent != null) { File.WriteAllText(saveFileDialog1.FileName, _exportContent); } diff --git a/src/ImeWlConverter.Abstractions/Contracts/IConversionPipeline.cs b/src/ImeWlConverter.Abstractions/Contracts/IConversionPipeline.cs index 43e79bc7..93a09c99 100644 --- a/src/ImeWlConverter.Abstractions/Contracts/IConversionPipeline.cs +++ b/src/ImeWlConverter.Abstractions/Contracts/IConversionPipeline.cs @@ -62,6 +62,9 @@ public sealed record ConversionResult /// Export content as string (for GUI display). Only populated when OutputStream is used. public string? ExportContent { get; init; } + /// Export data for binary formats. Only populated when OutputStream is used. + public byte[]? ExportData { get; init; } + /// Accumulated error messages from individual file processing. public string? ErrorMessages { get; init; } } diff --git a/src/ImeWlConverter.Core/Pipeline/ConversionPipeline.cs b/src/ImeWlConverter.Core/Pipeline/ConversionPipeline.cs index 56059d1f..235cfff1 100644 --- a/src/ImeWlConverter.Core/Pipeline/ConversionPipeline.cs +++ b/src/ImeWlConverter.Core/Pipeline/ConversionPipeline.cs @@ -141,20 +141,37 @@ e.Code is not null && progress?.Report(new ProgressInfo(exportedCount, exportedCount, $"正在导出 {exportedCount} 条词条...")); string? exportContent = null; + byte[]? exportData = null; if (request.OutputStream is not null) { - // GUI mode: write to provided stream, capture content - await exporter.ExportAsync(entries, request.OutputStream, request.Options.Export, ct); - request.OutputStream.Position = 0; - using var reader = new StreamReader(request.OutputStream, exporter.OutputEncoding, leaveOpen: true); - exportContent = await reader.ReadToEndAsync(ct); + // GUI mode: write to provided stream and retain the exact output for saving. + var exportResult = await exporter.ExportAsync(entries, request.OutputStream, request.Options.Export, ct); + exportedCount = exportResult.EntryCount; + filteredCount = importedCount - exportedCount; + if (request.OutputStream.CanSeek && request.OutputStream.CanRead) + { + request.OutputStream.Position = 0; + if (exporter.Metadata.IsBinary) + { + using var copy = new MemoryStream(); + await request.OutputStream.CopyToAsync(copy, ct); + exportData = copy.ToArray(); + } + else + { + using var reader = new StreamReader(request.OutputStream, exporter.OutputEncoding, leaveOpen: true); + exportContent = await reader.ReadToEndAsync(ct); + } + } } else if (request.OutputPath is not null) { // CLI/file mode: write directly to file using var outputStream = File.Create(request.OutputPath); - await exporter.ExportAsync(entries, outputStream, request.Options.Export, ct); + var exportResult = await exporter.ExportAsync(entries, outputStream, request.Options.Export, ct); + exportedCount = exportResult.EntryCount; + filteredCount = importedCount - exportedCount; } var errorStr = errors.Length > 0 ? errors.ToString() : null; @@ -165,6 +182,7 @@ e.Code is not null && ExportedCount = exportedCount, FilteredCount = filteredCount, ExportContent = exportContent, + ExportData = exportData, ErrorMessages = errorStr }); } @@ -212,11 +230,11 @@ private async Task> ExecutePerFileAsync( var outputFile = Path.Combine( request.OutputDirectory ?? ".", - Path.GetFileNameWithoutExtension(file) + ".txt"); + Path.GetFileNameWithoutExtension(file) + exporter.Metadata.FileExtension); using var outStream = File.Create(outputFile); - await exporter.ExportAsync(fileEntries, outStream, request.Options.Export, ct); + var exportResult = await exporter.ExportAsync(fileEntries, outStream, request.Options.Export, ct); - totalConverted += fileEntries.Count; + totalConverted += exportResult.EntryCount; progress?.Report(new ProgressInfo(i + 1, totalFiles, $"已导出: {outputFile}")); } catch (OperationCanceledException) { throw; } diff --git a/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs new file mode 100644 index 00000000..64bb5f1a --- /dev/null +++ b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs @@ -0,0 +1,255 @@ +namespace ImeWlConverter.Formats.SougouBin; + +using System.Buffers.Binary; +using System.Text; +using ImeWlConverter.Abstractions; +using ImeWlConverter.Abstractions.Contracts; +using ImeWlConverter.Abstractions.Models; +using ImeWlConverter.Abstractions.Options; +using ImeWlConverter.Abstractions.Results; + +/// +/// Writes Sougou Pinyin SGPU backup dictionaries. +/// +[FormatPlugin("sgpybin", "搜狗拼音备份词库bin", 30, IsBinary = true, FileExtension = ".bin")] +public sealed partial class SougouBinExporter : IFormatExporter +{ + private const int HeaderSize = 0x8C; + private const int DefaultIndexCapacity = 10_000; + private const int DefaultDictionaryCapacity = 400_000; + + public async Task ExportAsync( + IReadOnlyList entries, + Stream output, + ExportOptions? options = null, + CancellationToken ct = default) + { + var pendingRecords = new List(entries.Count); + var pinyinToIndex = BuildPinyinIndex(); + + foreach (var entry in entries) + { + ct.ThrowIfCancellationRequested(); + + if (TryCreateRecord(entry, pinyinToIndex, out var record)) + pendingRecords.Add(record); + } + + var seenKeys = new HashSet(StringComparer.Ordinal); + var records = new List(pendingRecords.Count); + for (var i = 0; i < pendingRecords.Count; i++) + { + var pending = pendingRecords[i]; + var key = pending.Word + "\0" + string.Join(',', pending.PinyinIndices); + var isDuplicate = !seenKeys.Add(key); + var id = checked((ushort)(i + 2)); + records.Add(BuildRecord(pending, id, isDuplicate)); + } + + var dictionaryUsed = records.Sum(record => record.Data.Length); + var indexCapacity = RoundUp(Math.Max(DefaultIndexCapacity, records.Count), DefaultIndexCapacity); + var dictionaryCapacity = RoundUp( + Math.Max(DefaultDictionaryCapacity, dictionaryUsed), + DefaultDictionaryCapacity); + var dictionaryBegin = checked(HeaderSize + indexCapacity * sizeof(uint)); + var data = new byte[checked(dictionaryBegin + dictionaryCapacity)]; + + WriteHeader(data, records, indexCapacity, dictionaryBegin, dictionaryCapacity, dictionaryUsed); + + var dictionaryOffset = 0; + foreach (var record in records) + { + record.DictionaryOffset = dictionaryOffset; + record.Data.CopyTo(data, dictionaryBegin + dictionaryOffset); + dictionaryOffset += record.Data.Length; + } + + var logicalIndex = records + .OrderBy(record => record, SougouRecordComparer.Instance) + .ToList(); + for (var i = 0; i < logicalIndex.Count; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian( + data.AsSpan(HeaderSize + i * sizeof(uint), sizeof(uint)), + checked((uint)logicalIndex[i].DictionaryOffset)); + } + + await output.WriteAsync(data, ct); + await output.FlushAsync(ct); + + return new ExportResult + { + EntryCount = records.Count, + ErrorCount = entries.Count - records.Count + }; + } + + private static Dictionary BuildPinyinIndex() + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < SougouBinParser.PinyinData.Length; i++) + result.TryAdd(SougouBinParser.PinyinData[i], checked((ushort)i)); + + return result; + } + + private static bool TryCreateRecord( + WordEntry entry, + IReadOnlyDictionary pinyinToIndex, + out PendingRecord record) + { + record = default!; + if (string.IsNullOrWhiteSpace(entry.Word) || entry.Code?.Segments.Count is not > 0) + return false; + + var pinyinIndices = new ushort[entry.Code.Segments.Count]; + for (var i = 0; i < entry.Code.Segments.Count; i++) + { + var segment = entry.Code.Segments[i]; + if (segment.Count == 0 || !pinyinToIndex.TryGetValue(NormalizePinyin(segment[0]), out pinyinIndices[i])) + return false; + } + + var wordBytes = Encoding.Unicode.GetBytes(entry.Word); + var pinyinByteCount = checked(pinyinIndices.Length * sizeof(ushort)); + if (wordBytes.Length > ushort.MaxValue || pinyinByteCount > ushort.MaxValue) + return false; + + record = new PendingRecord( + entry.Word, + pinyinIndices, + wordBytes, + Math.Clamp(entry.Rank, 0, ushort.MaxValue)); + return true; + } + + private static SougouRecord BuildRecord(PendingRecord pending, ushort id, bool isDuplicate) + { + var pinyinByteCount = checked(pending.PinyinIndices.Length * sizeof(ushort)); + var trailingData = isDuplicate + ? BuildPinyinTail(pending.PinyinIndices) + : new byte[] { 0x02, 0x00, 0xA0, 0x03 }; + var data = new byte[checked(15 + pinyinByteCount + pending.WordBytes.Length + trailingData.Length)]; + + BinaryPrimitives.WriteUInt16LittleEndian(data, checked((ushort)pending.Rank)); + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan(2), id); + data[4] = 0; + data[5] = 0; + data[6] = 1; + data[7] = 0; + data[8] = checked((byte)Math.Min(pending.PinyinIndices.Length, byte.MaxValue)); + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan(9), checked((ushort)pinyinByteCount)); + + for (var i = 0; i < pending.PinyinIndices.Length; i++) + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan(11 + i * sizeof(ushort)), pending.PinyinIndices[i]); + + var wordInfoOffset = 11 + pinyinByteCount; + BinaryPrimitives.WriteUInt16LittleEndian( + data.AsSpan(wordInfoOffset), + checked((ushort)(sizeof(ushort) + pending.WordBytes.Length + trailingData.Length))); + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan(wordInfoOffset + 2), checked((ushort)pending.WordBytes.Length)); + pending.WordBytes.CopyTo(data, wordInfoOffset + 4); + trailingData.CopyTo(data, wordInfoOffset + 4 + pending.WordBytes.Length); + + return new SougouRecord(pending.Word, pending.PinyinIndices, id, data); + } + + private static byte[] BuildPinyinTail(IReadOnlyList pinyinIndices) + { + var data = new byte[checked(sizeof(ushort) + pinyinIndices.Count * sizeof(ushort))]; + BinaryPrimitives.WriteUInt16LittleEndian(data, checked((ushort)(pinyinIndices.Count * sizeof(ushort)))); + for (var i = 0; i < pinyinIndices.Count; i++) + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan(sizeof(ushort) + i * sizeof(ushort)), pinyinIndices[i]); + return data; + } + + private static string NormalizePinyin(string value) + { + return value + .Trim() + .ToLowerInvariant() + .TrimEnd('0', '1', '2', '3', '4', '5') + .Replace("u:", "v", StringComparison.Ordinal) + .Replace('ü', 'v'); + } + + private static void WriteHeader( + byte[] data, + IReadOnlyList records, + int indexCapacity, + int dictionaryBegin, + int dictionaryCapacity, + int dictionaryUsed) + { + BinaryPrimitives.WriteUInt32LittleEndian(data, 0x55504753); // SGPU + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x04), 0x28); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x08), 0x0133A009); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x10), checked((uint)data.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x14), 1); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x18), 16); + // Sogou validates this value when importing. It is the base marker plus + // dictionary data, per-entry bookkeeping, and the zero-frequency count. + var zeroRankCount = records.Count(record => record.Rank == 0); + var marker = checked(0x5691F359u + (uint)dictionaryUsed + (uint)Math.Max(records.Count - 1, 0) + (uint)zeroRankCount); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x20), marker); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x24), 84); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x28), checked((uint)(records.Count + 1))); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x30), checked((uint)records.Count(record => record.Rank > 0))); + BinaryPrimitives.WriteUInt32LittleEndian( + data.AsSpan(0x34), + checked((uint)records.Where(record => record.Rank > 0).Sum(record => (long)record.Id))); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x38), HeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x3C), checked((uint)(indexCapacity * sizeof(uint)))); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x40), checked((uint)records.Count)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x44), checked((uint)dictionaryBegin)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x48), checked((uint)dictionaryCapacity)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x4C), checked((uint)dictionaryUsed)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x50), 9); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x54), checked((uint)zeroRankCount)); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x64), uint.MaxValue); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x68), 4); + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x6C), uint.MaxValue); + } + + private static int RoundUp(int value, int unit) + { + return checked(((value + unit - 1) / unit) * unit); + } + + private sealed record PendingRecord(string Word, ushort[] PinyinIndices, byte[] WordBytes, int Rank); + + private sealed class SougouRecord(string word, ushort[] pinyinIndices, ushort id, byte[] data) + { + public string Word { get; } = word; + public ushort[] PinyinIndices { get; } = pinyinIndices; + public ushort Id { get; } = id; + public int Rank => BinaryPrimitives.ReadUInt16LittleEndian(Data); + public byte[] Data { get; } = data; + public int DictionaryOffset { get; set; } + } + + private sealed class SougouRecordComparer : IComparer + { + public static SougouRecordComparer Instance { get; } = new(); + + public int Compare(SougouRecord? x, SougouRecord? y) + { + if (ReferenceEquals(x, y)) return 0; + if (x is null) return -1; + if (y is null) return 1; + + var length = Math.Min(x.PinyinIndices.Length, y.PinyinIndices.Length); + for (var i = 0; i < length; i++) + { + var result = x.PinyinIndices[i].CompareTo(y.PinyinIndices[i]); + if (result != 0) return result; + } + + var countResult = x.PinyinIndices.Length.CompareTo(y.PinyinIndices.Length); + if (countResult != 0) return countResult; + + var wordResult = StringComparer.Ordinal.Compare(x.Word, y.Word); + return wordResult != 0 ? wordResult : y.Id.CompareTo(x.Id); + } + } +} diff --git a/src/ImeWlConverter.Formats/SougouBin/SougouBinParser.cs b/src/ImeWlConverter.Formats/SougouBin/SougouBinParser.cs index a2860900..5651798a 100644 --- a/src/ImeWlConverter.Formats/SougouBin/SougouBinParser.cs +++ b/src/ImeWlConverter.Formats/SougouBin/SougouBinParser.cs @@ -619,7 +619,7 @@ private static uint ReadUInt32(Stream fs) #region Pinyin data - private static readonly string[] PinyinData = + internal static readonly string[] PinyinData = { "a", "ai", "an", "ang", "ao", "ba", "bai", "ban", "bang", "bao", "bei", "ben", "beng", "bi", "bian", "biao", "bie", "bin", "bing", "bo", diff --git a/src/ImeWlConverterCoreTest/PipelineBinaryExportTest.cs b/src/ImeWlConverterCoreTest/PipelineBinaryExportTest.cs new file mode 100644 index 00000000..bf76a35b --- /dev/null +++ b/src/ImeWlConverterCoreTest/PipelineBinaryExportTest.cs @@ -0,0 +1,103 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ImeWlConverter.Abstractions.Contracts; +using ImeWlConverter.Abstractions.Models; +using ImeWlConverter.Abstractions.Options; +using ImeWlConverter.Abstractions.Results; +using ImeWlConverter.Core.Pipeline; +using Xunit; + +namespace ImeWlConverterCoreTest; + +public class PipelineBinaryExportTest +{ + [Fact] + public async Task ExecuteAsync_KeepsBinaryOutputAsBytes() + { + var inputPath = Path.GetTempFileName(); + try + { + var expected = new byte[] { 0x53, 0x47, 0x50, 0x55, 0xA0, 0x80, 0xFF }; + var pipeline = new ConversionPipeline( + [new TestImporter()], + [new TestBinaryExporter(expected)]); + using var output = new MemoryStream(); + + var result = await pipeline.ExecuteAsync(new ConversionRequest + { + InputFormatId = "test-input", + OutputFormatId = "test-binary", + InputPaths = [inputPath], + OutputStream = output + }); + + Assert.True(result.IsSuccess); + Assert.Equal(expected, result.Value.ExportData); + Assert.Null(result.Value.ExportContent); + } + finally + { + File.Delete(inputPath); + } + } + + [Fact] + public async Task ExecuteAsync_ReportsExporterEntryCount() + { + var inputPath = Path.GetTempFileName(); + try + { + var pipeline = new ConversionPipeline( + [new TestImporter()], + [new TestBinaryExporter([0x00], entryCount: 0)]); + using var output = new MemoryStream(); + + var result = await pipeline.ExecuteAsync(new ConversionRequest + { + InputFormatId = "test-input", + OutputFormatId = "test-binary", + InputPaths = [inputPath], + OutputStream = output + }); + + Assert.True(result.IsSuccess); + Assert.Equal(0, result.Value.ExportedCount); + Assert.Equal(1, result.Value.FilteredCount); + } + finally + { + File.Delete(inputPath); + } + } + + private sealed class TestImporter : IFormatImporter + { + public FormatMetadata Metadata { get; } = new("test-input", "Test input", 0, true, false); + + public Task ImportAsync(Stream input, ImportOptions? options = null, CancellationToken ct = default) => + Task.FromResult(new ImportResult + { + Entries = [new WordEntry { Word = "test" }] + }); + } + + private sealed class TestBinaryExporter(byte[] data, int? entryCount = null) : IFormatExporter + { + public FormatMetadata Metadata { get; } = new("test-binary", "Test binary", 0, false, true, true, ".bin"); + + public async Task ExportAsync( + IReadOnlyList entries, + Stream output, + ExportOptions? options = null, + CancellationToken ct = default) + { + await output.WriteAsync(data, ct); + return new ExportResult { EntryCount = entryCount ?? entries.Count }; + } + } +} diff --git a/src/ImeWlConverterCoreTest/SougouBinExportTest.cs b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs new file mode 100644 index 00000000..5d5a9846 --- /dev/null +++ b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using ImeWlConverter.Abstractions.Enums; +using ImeWlConverter.Abstractions.Models; +using ImeWlConverter.Formats.SougouBin; +using Xunit; + +namespace ImeWlConverterCoreTest; + +public class SougouBinExportTest +{ + [Fact] + public async Task ExportThenImport_PreservesWordsCodesAndRanks() + { + var entries = new List + { + new() { Word = "\u6df1\u84dd\u8bcd\u5e93", Code = WordCode.FromSingle(new[] { "shen", "lan", "ci", "ku" }), Rank = 17, CodeType = CodeType.Pinyin }, + new() { Word = "\u8f6f\u4ef6\u6d4b\u8bd5", Code = WordCode.FromSingle(new[] { "ruan", "jian", "ce", "shi" }), Rank = 1, CodeType = CodeType.Pinyin }, + new() { Word = "\u5973\u7eff", Code = WordCode.FromSingle(new[] { "nv", "lv" }), Rank = 0, CodeType = CodeType.Pinyin } + }; + + var (result, data) = await Export(entries); + + Assert.Equal(entries.Count, result.EntryCount); + Assert.Equal(0, result.ErrorCount); + Assert.Equal(0x55504753u, BitConverter.ToUInt32(data, 0)); + Assert.Equal((uint)entries.Count, BitConverter.ToUInt32(data, 0x40)); + Assert.Equal((uint)0x8C, BitConverter.ToUInt32(data, 0x38)); + Assert.Equal((uint)data.Length, BitConverter.ToUInt32(data, 0x10)); + + using var stream = new MemoryStream(data); + var imported = (await new SougouBinImporter().ImportAsync(stream)).Entries; + + Assert.Equal(entries.Count, imported.Count); + foreach (var entry in entries) + { + var actual = Assert.Single(imported, item => item.Word == entry.Word); + Assert.Equal(entry.Code!.GetPrimaryCode(), actual.Code!.GetPrimaryCode()); + Assert.Equal(entry.Rank, actual.Rank); + } + } + + [Fact] + public async Task Export_SkipsEntriesWithoutSupportedPinyin() + { + var entries = new List + { + new() { Word = "\u6b63\u5e38", Code = WordCode.FromSingle(new[] { "zheng", "chang" }), Rank = 1 }, + new() { Word = "\u65e0\u6548", Code = WordCode.FromSingle(new[] { "not-pinyin" }), Rank = 1 }, + new() { Word = "\u7f3a\u5c11", Rank = 1 } + }; + + var (result, data) = await Export(entries); + + Assert.Equal(1, result.EntryCount); + Assert.Equal(2, result.ErrorCount); + Assert.Equal(1u, BitConverter.ToUInt32(data, 0x40)); + } + + [Fact] + public async Task Export_WritesSogouMarkerFromDictionarySizeAndZeroRanks() + { + var entries = new List + { + new() { Word = "\u4ec4\u963f\u7532", Code = WordCode.FromSingle(new[] { "ze", "a", "jia" }), Rank = 1 }, + new() { Word = "\u4ec4\u963f\u5047", Code = WordCode.FromSingle(new[] { "ze", "a", "jia" }), Rank = 0 } + }; + + var (_, data) = await Export(entries); + + var dictionarySize = BitConverter.ToUInt32(data, 0x4c); + var wordCount = BitConverter.ToUInt32(data, 0x40); + var zeroRankCount = BitConverter.ToUInt32(data, 0x54); + var expected = checked(0x5691F359u + dictionarySize + wordCount - 1 + zeroRankCount); + + Assert.Equal(expected, BitConverter.ToUInt32(data, 0x20)); + } + + private static async Task<(ImeWlConverter.Abstractions.Results.ExportResult Result, byte[] Data)> Export( + IReadOnlyList entries) + { + var exporter = new SougouBinExporter(); + using var stream = new MemoryStream(); + var result = await exporter.ExportAsync(entries, stream); + return (result, stream.ToArray()); + } +} diff --git a/src/ImeWlConverterMac/ViewModels/MainWindowViewModel.cs b/src/ImeWlConverterMac/ViewModels/MainWindowViewModel.cs index f1491ba8..ee5143df 100644 --- a/src/ImeWlConverterMac/ViewModels/MainWindowViewModel.cs +++ b/src/ImeWlConverterMac/ViewModels/MainWindowViewModel.cs @@ -41,6 +41,7 @@ public class MainWindowViewModel : ViewModelBase private FilterConfig _filterConfig = new(); private CodeGenerationOptions _codeGenOptions = new(); private string? _lastExportContent; + private byte[]? _lastExportData; private CancellationTokenSource? _cts; public MainWindowViewModel(IServiceProvider serviceProvider) @@ -306,6 +307,7 @@ private async Task ConvertAsync() Progress = 0; ResultText = ""; _lastExportContent = null; + _lastExportData = null; _cts = new CancellationTokenSource(); try @@ -352,6 +354,7 @@ private async Task ConvertAsync() if (result.IsSuccess) { _lastExportContent = result.Value.ExportContent; + _lastExportData = result.Value.ExportData; StatusMessage = $"转换完成，导入 {result.Value.ImportedCount} 条，导出 {result.Value.ExportedCount} 条"; if (_lastExportContent != null) @@ -575,7 +578,7 @@ private async void ShowMergeWL() private async Task ShowSaveDialog() { - if (string.IsNullOrEmpty(_lastExportContent)) return; + if (string.IsNullOrEmpty(_lastExportContent) && _lastExportData is null) return; try { @@ -607,7 +610,10 @@ private async Task ShowSaveDialog() if (file != null) { var filePath = file.Path.LocalPath; - await File.WriteAllTextAsync(filePath, _lastExportContent); + if (_lastExportData is not null) + await File.WriteAllBytesAsync(filePath, _lastExportData); + else + await File.WriteAllTextAsync(filePath, _lastExportContent!); StatusMessage = $"保存成功，词库路径：{filePath}"; } else From 11d813360acb51e4bcf8aba19d8da1ff39b23208 Mon Sep 17 00:00:00 2001 From: Frederick <1814239758@qq.com> Date: Sun, 9 Aug 2026 20:00:24 +0800 Subject: [PATCH 2/3] feat: support large Sogou backup dictionaries --- .../SougouBin/SougouBinExporter.cs | 52 ++++++++++---- .../SougouBinExportTest.cs | 72 +++++++++++++++++++ 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs index 64bb5f1a..f02bb730 100644 --- a/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs +++ b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs @@ -17,6 +17,9 @@ public sealed partial class SougouBinExporter : IFormatExporter private const int HeaderSize = 0x8C; private const int DefaultIndexCapacity = 10_000; private const int DefaultDictionaryCapacity = 400_000; + private const int LargeDictionaryCapacityUnit = 10_000; + private const int LargeDictionaryCapacityReserve = 90_000; + private const uint LargeDictionaryMarkerStride = 0x000B4AA0; public async Task ExportAsync( IReadOnlyList entries, @@ -35,6 +38,7 @@ public async Task ExportAsync( pendingRecords.Add(record); } + var isLargeDictionary = pendingRecords.Count > DefaultIndexCapacity; var seenKeys = new HashSet(StringComparer.Ordinal); var records = new List(pendingRecords.Count); for (var i = 0; i < pendingRecords.Count; i++) @@ -42,19 +46,28 @@ public async Task ExportAsync( var pending = pendingRecords[i]; var key = pending.Word + "\0" + string.Join(',', pending.PinyinIndices); var isDuplicate = !seenKeys.Add(key); - var id = checked((ushort)(i + 2)); + var id = isLargeDictionary ? (ushort)0 : checked((ushort)(i + 2)); records.Add(BuildRecord(pending, id, isDuplicate)); } var dictionaryUsed = records.Sum(record => record.Data.Length); var indexCapacity = RoundUp(Math.Max(DefaultIndexCapacity, records.Count), DefaultIndexCapacity); - var dictionaryCapacity = RoundUp( - Math.Max(DefaultDictionaryCapacity, dictionaryUsed), - DefaultDictionaryCapacity); + var dictionaryCapacity = isLargeDictionary + ? checked(RoundUp(dictionaryUsed, LargeDictionaryCapacityUnit) + LargeDictionaryCapacityReserve) + : RoundUp( + Math.Max(DefaultDictionaryCapacity, dictionaryUsed), + DefaultDictionaryCapacity); var dictionaryBegin = checked(HeaderSize + indexCapacity * sizeof(uint)); var data = new byte[checked(dictionaryBegin + dictionaryCapacity)]; - WriteHeader(data, records, indexCapacity, dictionaryBegin, dictionaryCapacity, dictionaryUsed); + WriteHeader( + data, + records, + indexCapacity, + dictionaryBegin, + dictionaryCapacity, + dictionaryUsed, + isLargeDictionary); var dictionaryOffset = 0; foreach (var record in records) @@ -179,7 +192,8 @@ private static void WriteHeader( int indexCapacity, int dictionaryBegin, int dictionaryCapacity, - int dictionaryUsed) + int dictionaryUsed, + bool isLargeDictionary) { BinaryPrimitives.WriteUInt32LittleEndian(data, 0x55504753); // SGPU BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x04), 0x28); @@ -187,17 +201,31 @@ private static void WriteHeader( BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x10), checked((uint)data.Length)); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x14), 1); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x18), 16); - // Sogou validates this value when importing. It is the base marker plus - // dictionary data, per-entry bookkeeping, and the zero-frequency count. + // Sogou validates this value when importing. Each index block after the + // first 10,000 records adds the allocator stride used by Sogou itself. var zeroRankCount = records.Count(record => record.Rank == 0); - var marker = checked(0x5691F359u + (uint)dictionaryUsed + (uint)Math.Max(records.Count - 1, 0) + (uint)zeroRankCount); + var additionalBlocks = isLargeDictionary + ? Math.Max(records.Count - 1, 0) / DefaultIndexCapacity + : 0; + var marker = checked( + 0x5691F359u + + (uint)dictionaryUsed + + (uint)Math.Max(records.Count - 1, 0) + + (uint)zeroRankCount + + checked((uint)additionalBlocks * LargeDictionaryMarkerStride)); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x20), marker); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x24), 84); - BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x28), checked((uint)(records.Count + 1))); - BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x30), checked((uint)records.Count(record => record.Rank > 0))); + BinaryPrimitives.WriteUInt32LittleEndian( + data.AsSpan(0x28), + isLargeDictionary ? 1u : checked((uint)(records.Count + 1))); + BinaryPrimitives.WriteUInt32LittleEndian( + data.AsSpan(0x30), + isLargeDictionary ? 0u : checked((uint)records.Count(record => record.Rank > 0))); BinaryPrimitives.WriteUInt32LittleEndian( data.AsSpan(0x34), - checked((uint)records.Where(record => record.Rank > 0).Sum(record => (long)record.Id))); + isLargeDictionary + ? 0u + : checked((uint)records.Where(record => record.Rank > 0).Sum(record => (long)record.Id))); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x38), HeaderSize); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x3C), checked((uint)(indexCapacity * sizeof(uint)))); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x40), checked((uint)records.Count)); diff --git a/src/ImeWlConverterCoreTest/SougouBinExportTest.cs b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs index 5d5a9846..0af7ac31 100644 --- a/src/ImeWlConverterCoreTest/SougouBinExportTest.cs +++ b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs @@ -79,6 +79,78 @@ public async Task Export_WritesSogouMarkerFromDictionarySizeAndZeroRanks() Assert.Equal(expected, BitConverter.ToUInt32(data, 0x20)); } + [Fact] + public async Task Export_AtTenThousandEntries_UsesSingleBlockLayout() + { + var (_, data) = await Export(CreateEntries(10_000)); + + var count = BitConverter.ToUInt32(data, 0x40); + var dictionaryBegin = BitConverter.ToUInt32(data, 0x44); + + Assert.Equal(10_000u, count); + Assert.Equal(10_001u, BitConverter.ToUInt32(data, 0x28)); + Assert.Equal(10_000u, BitConverter.ToUInt32(data, 0x30)); + Assert.NotEqual(0u, BitConverter.ToUInt32(data, 0x34)); + Assert.Equal((ushort)2, ReadRecordId(data, dictionaryBegin, 0)); + Assert.Equal((ushort)10_001, ReadRecordId(data, dictionaryBegin, (int)count - 1)); + } + + [Theory] + [InlineData(10_001, 1)] + [InlineData(20_000, 1)] + [InlineData(20_001, 2)] + [InlineData(30_000, 2)] + [InlineData(30_001, 3)] + [InlineData(40_000, 3)] + public async Task Export_LargeDictionary_UsesSogouBatchStorageLayout( + int entryCount, + uint expectedAdditionalBlocks) + { + var entries = CreateEntries(entryCount); + + var (_, data) = await Export(entries); + + var count = BitConverter.ToUInt32(data, 0x40); + var used = BitConverter.ToUInt32(data, 0x4c); + var expectedCapacity = checked(((used + 9_999u) / 10_000u) * 10_000u + 90_000u); + var expectedMarker = checked( + 0x5691F359u + + used + + count + - 1 + + expectedAdditionalBlocks * 0x000B4AA0u); + + Assert.Equal((uint)entryCount, count); + Assert.Equal(1u, BitConverter.ToUInt32(data, 0x28)); + Assert.Equal(0u, BitConverter.ToUInt32(data, 0x30)); + Assert.Equal(0u, BitConverter.ToUInt32(data, 0x34)); + Assert.Equal(expectedCapacity, BitConverter.ToUInt32(data, 0x48)); + Assert.Equal(expectedMarker, BitConverter.ToUInt32(data, 0x20)); + + var dictionaryBegin = BitConverter.ToUInt32(data, 0x44); + Assert.Equal((ushort)0, ReadRecordId(data, dictionaryBegin, 0)); + Assert.Equal((ushort)0, ReadRecordId(data, dictionaryBegin, (int)count / 2)); + Assert.Equal((ushort)0, ReadRecordId(data, dictionaryBegin, (int)count - 1)); + } + + private static List CreateEntries(int count) + { + return Enumerable.Range(0, count) + .Select(index => new WordEntry + { + Word = $"\u8bcd{index}", + Code = WordCode.FromSingle(new[] { "ci" }), + Rank = 1 + }) + .ToList(); + } + + private static ushort ReadRecordId(byte[] data, uint dictionaryBegin, int index) + { + var offset = BitConverter.ToUInt32(data, 0x8c + index * sizeof(uint)); + return BitConverter.ToUInt16(data, (int)dictionaryBegin + (int)offset + 2); + } + private static async Task<(ImeWlConverter.Abstractions.Results.ExportResult Result, byte[] Data)> Export( IReadOnlyList entries) { From 550d8a3c251a79d088930b5e449b466f23f49ef6 Mon Sep 17 00:00:00 2001 From: Frederick <1814239758@qq.com> Date: Sun, 9 Aug 2026 21:58:35 +0800 Subject: [PATCH 3/3] fix: match Sogou large dictionary allocator --- README.md | 4 +-- .../SougouBin/SougouBinExporter.cs | 36 ++++++++++++++----- .../SougouBinExportTest.cs | 23 ++++++------ 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b3414cf5..6c117820 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ imewlconverter -i <输入格式> -o <输出格式> -O <输出文件> <输入文 |---------|------|-----------|:-------:|:-------:| | `scel` | 搜狗拼音细胞词库 | .scel | ✅ | ✅ | | `sgpy` | 搜狗拼音文本格式 | .txt | ✅ | ✅ | -| `sgpybin` | 搜狗拼音备份词库 | .bin | ✅ | ❌ | +| `sgpybin` | 搜狗拼音备份词库 | .bin | ✅ | ✅ | | `qqpy` | QQ 拼音文本格式 | .txt | ✅ | ✅ | | `qpyd` | QQ 拼音分类词库 | .qpyd | ✅ | ❌ | | `qcel` | QQ 拼音细胞词库 | .qcel | ✅ | ❌ | @@ -251,4 +251,4 @@ cd ../../tests/integration ## Star History -[![Star History Chart](https://api.star-history.com/svg?repos=studyzy/imewlconverter&type=Date)](https://star-history.com/#studyzy/imewlconverter&Date) \ No newline at end of file +[![Star History Chart](https://api.star-history.com/svg?repos=studyzy/imewlconverter&type=Date)](https://star-history.com/#studyzy/imewlconverter&Date) diff --git a/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs index f02bb730..20337be8 100644 --- a/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs +++ b/src/ImeWlConverter.Formats/SougouBin/SougouBinExporter.cs @@ -18,8 +18,10 @@ public sealed partial class SougouBinExporter : IFormatExporter private const int DefaultIndexCapacity = 10_000; private const int DefaultDictionaryCapacity = 400_000; private const int LargeDictionaryCapacityUnit = 10_000; - private const int LargeDictionaryCapacityReserve = 90_000; - private const uint LargeDictionaryMarkerStride = 0x000B4AA0; + private const int LargeDictionaryBytesPerIndexSlot = 31; + private const int LargeDictionaryReserve = 90_000; + private const int LargeDictionaryReserveGrowthPeriod = 4; + private const int LargeDictionaryReserveGrowth = 10_000; public async Task ExportAsync( IReadOnlyList entries, @@ -53,7 +55,9 @@ public async Task ExportAsync( var dictionaryUsed = records.Sum(record => record.Data.Length); var indexCapacity = RoundUp(Math.Max(DefaultIndexCapacity, records.Count), DefaultIndexCapacity); var dictionaryCapacity = isLargeDictionary - ? checked(RoundUp(dictionaryUsed, LargeDictionaryCapacityUnit) + LargeDictionaryCapacityReserve) + ? Math.Max( + CalculateLargeDictionaryCapacity(indexCapacity), + RoundUp(dictionaryUsed, LargeDictionaryCapacityUnit)) : RoundUp( Math.Max(DefaultDictionaryCapacity, dictionaryUsed), DefaultDictionaryCapacity); @@ -201,18 +205,20 @@ private static void WriteHeader( BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x10), checked((uint)data.Length)); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x14), 1); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x18), 16); - // Sogou validates this value when importing. Each index block after the - // first 10,000 records adds the allocator stride used by Sogou itself. + // Sogou validates this value when importing. Its allocator contributes + // two times the dictionary growth and three times the index growth. var zeroRankCount = records.Count(record => record.Rank == 0); - var additionalBlocks = isLargeDictionary - ? Math.Max(records.Count - 1, 0) / DefaultIndexCapacity - : 0; + var allocationMarker = isLargeDictionary + ? checked((uint)( + 2L * (dictionaryCapacity - DefaultDictionaryCapacity) + + 3L * ((long)indexCapacity * sizeof(uint) - DefaultIndexCapacity * sizeof(uint)))) + : 0u; var marker = checked( 0x5691F359u + (uint)dictionaryUsed + (uint)Math.Max(records.Count - 1, 0) + (uint)zeroRankCount - + checked((uint)additionalBlocks * LargeDictionaryMarkerStride)); + + allocationMarker); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x20), marker); BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(0x24), 84); BinaryPrimitives.WriteUInt32LittleEndian( @@ -244,6 +250,18 @@ private static int RoundUp(int value, int unit) return checked(((value + unit - 1) / unit) * unit); } + private static int CalculateLargeDictionaryCapacity(int indexCapacity) + { + // Sogou allocates 31 bytes per index slot plus a 90 KB reserve. The + // reserve gains another 10 KB after every four 10,000-entry blocks. + var blockCount = indexCapacity / DefaultIndexCapacity; + var reserveGrowthCount = (blockCount - 1) / LargeDictionaryReserveGrowthPeriod; + return checked( + indexCapacity * LargeDictionaryBytesPerIndexSlot + + LargeDictionaryReserve + + reserveGrowthCount * LargeDictionaryReserveGrowth); + } + private sealed record PendingRecord(string Word, ushort[] PinyinIndices, byte[] WordBytes, int Rank); private sealed class SougouRecord(string word, ushort[] pinyinIndices, ushort id, byte[] data) diff --git a/src/ImeWlConverterCoreTest/SougouBinExportTest.cs b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs index 0af7ac31..3e21eaf7 100644 --- a/src/ImeWlConverterCoreTest/SougouBinExportTest.cs +++ b/src/ImeWlConverterCoreTest/SougouBinExportTest.cs @@ -96,15 +96,17 @@ public async Task Export_AtTenThousandEntries_UsesSingleBlockLayout() } [Theory] - [InlineData(10_001, 1)] - [InlineData(20_000, 1)] - [InlineData(20_001, 2)] - [InlineData(30_000, 2)] - [InlineData(30_001, 3)] - [InlineData(40_000, 3)] - public async Task Export_LargeDictionary_UsesSogouBatchStorageLayout( + [InlineData(10_001, 710_000)] + [InlineData(20_000, 710_000)] + [InlineData(20_001, 1_020_000)] + [InlineData(30_000, 1_020_000)] + [InlineData(30_001, 1_330_000)] + [InlineData(40_000, 1_330_000)] + [InlineData(40_001, 1_650_000)] + [InlineData(48_000, 1_650_000)] + public async Task Export_LargeDictionary_UsesSogouAllocatorLayout( int entryCount, - uint expectedAdditionalBlocks) + uint expectedCapacity) { var entries = CreateEntries(entryCount); @@ -112,13 +114,14 @@ public async Task Export_LargeDictionary_UsesSogouBatchStorageLayout( var count = BitConverter.ToUInt32(data, 0x40); var used = BitConverter.ToUInt32(data, 0x4c); - var expectedCapacity = checked(((used + 9_999u) / 10_000u) * 10_000u + 90_000u); + var indexSize = BitConverter.ToUInt32(data, 0x3c); var expectedMarker = checked( 0x5691F359u + used + count - 1 - + expectedAdditionalBlocks * 0x000B4AA0u); + + checked(2u * (expectedCapacity - 400_000u)) + + checked(3u * (indexSize - 40_000u))); Assert.Equal((uint)entryCount, count); Assert.Equal(1u, BitConverter.ToUInt32(data, 0x28));