diff --git a/UnicodeCharacterInspector/App.config b/UnicodeCharacterInspector/App.config index 8e15646..d1428ad 100644 --- a/UnicodeCharacterInspector/App.config +++ b/UnicodeCharacterInspector/App.config @@ -1,6 +1,6 @@ - + - + - \ No newline at end of file + diff --git a/UnicodeCharacterInspector/App.manifest b/UnicodeCharacterInspector/App.manifest new file mode 100644 index 0000000..e74ab2a --- /dev/null +++ b/UnicodeCharacterInspector/App.manifest @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/UnicodeCharacterInspector/UnicodeCharacterInspector.csproj b/UnicodeCharacterInspector/UnicodeCharacterInspector.csproj index 426225d..b767a37 100644 --- a/UnicodeCharacterInspector/UnicodeCharacterInspector.csproj +++ b/UnicodeCharacterInspector/UnicodeCharacterInspector.csproj @@ -14,6 +14,7 @@ {60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 4 f1bfc594 + AnyCPU @@ -34,6 +35,9 @@ prompt 4 + + App.manifest + @@ -80,6 +84,7 @@ + diff --git a/UnicodeInformation.Builder/FileUcdSource.cs b/UnicodeInformation.Builder/FileDataSource.cs similarity index 83% rename from UnicodeInformation.Builder/FileUcdSource.cs rename to UnicodeInformation.Builder/FileDataSource.cs index d452172..f3ba7b1 100644 --- a/UnicodeInformation.Builder/FileUcdSource.cs +++ b/UnicodeInformation.Builder/FileDataSource.cs @@ -8,11 +8,11 @@ using System.Unicode; namespace System.Unicode.Builder { - public sealed class FileUcdSource : IUcdSource + public sealed class FileDataSource : IDataSource { private readonly string baseDirectory; - public FileUcdSource(string baseDirectory) + public FileDataSource(string baseDirectory) { this.baseDirectory = Path.GetFullPath(baseDirectory); } diff --git a/UnicodeInformation.Builder/HttpUcdSource.cs b/UnicodeInformation.Builder/HttpDataSource.cs similarity index 87% rename from UnicodeInformation.Builder/HttpUcdSource.cs rename to UnicodeInformation.Builder/HttpDataSource.cs index 0133ce7..129dbd9 100644 --- a/UnicodeInformation.Builder/HttpUcdSource.cs +++ b/UnicodeInformation.Builder/HttpDataSource.cs @@ -8,19 +8,19 @@ using System.Threading.Tasks; namespace System.Unicode.Builder { - public class HttpUcdSource : IUcdSource + public class HttpDataSource : IDataSource { public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute); private readonly HttpClient httpClient; private readonly Uri baseUri; - public HttpUcdSource() + public HttpDataSource() : this(UnicodeCharacterDataUri) { } - public HttpUcdSource(Uri baseUri) + public HttpDataSource(Uri baseUri) { this.httpClient = new HttpClient(); this.baseUri = baseUri; diff --git a/UnicodeInformation.Builder/IUcdSource.cs b/UnicodeInformation.Builder/IDataSource.cs similarity index 84% rename from UnicodeInformation.Builder/IUcdSource.cs rename to UnicodeInformation.Builder/IDataSource.cs index 2c51a4b..3e807b8 100644 --- a/UnicodeInformation.Builder/IUcdSource.cs +++ b/UnicodeInformation.Builder/IDataSource.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; namespace System.Unicode.Builder { - public interface IUcdSource : IDisposable + public interface IDataSource : IDisposable { Task OpenDataFileAsync(string fileName); } diff --git a/UnicodeInformation.Builder/Program.cs b/UnicodeInformation.Builder/Program.cs index f836a3e..d52d240 100644 --- a/UnicodeInformation.Builder/Program.cs +++ b/UnicodeInformation.Builder/Program.cs @@ -13,7 +13,7 @@ namespace System.Unicode.Builder { public const string UnihanDirectoryName = "Unihan"; public const string UnihanArchiveName = "Unihan.zip"; - public const string UcdDirectoryName = "UCD"; + public const string directoryName = "UCD"; public const string UcdArchiveName = "UCD.zip"; public static readonly string[] ucdRequiredFiles = new[] @@ -22,32 +22,39 @@ namespace System.Unicode.Builder "PropList.txt", "DerivedCoreProperties.txt", "Blocks.txt", - "Jamo.txt" + "Jamo.txt", }; - private static byte[] DownloadUcdArchive() + public static readonly string[] unihanRequiredFiles = new[] + { + "Unihan_NumericValues.txt", + "Unihan_Readings.txt", + "Unihan_Variants.txt", + }; + + private static byte[] DownloadDataArchive(string archiveName) { using (var httpClient = new HttpClient()) { - return httpClient.GetByteArrayAsync(HttpUcdSource.UnicodeCharacterDataUri + UcdArchiveName).Result; + return httpClient.GetByteArrayAsync(HttpDataSource.UnicodeCharacterDataUri + archiveName).Result; } } - internal static IUcdSource GetUcdSource(bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract) + internal static IDataSource GetDataSource(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract) { string baseDirectory = Environment.CurrentDirectory; - string ucdDirectory = Path.Combine(baseDirectory, UcdDirectoryName); - string ucdArchiveFileName = Path.Combine(baseDirectory, UcdArchiveName); + string dataDirectory = Path.Combine(baseDirectory, Program.directoryName); + string dataArchiveFileName = Path.Combine(baseDirectory, archiveName); if (shouldDownload != true) { - bool hasValidDirectory = Directory.Exists(ucdDirectory); + bool hasValidDirectory = Directory.Exists(dataDirectory); if (hasValidDirectory) { - foreach (string ucdRequiredFile in ucdRequiredFiles) + foreach (string requiredFile in requiredFiles) { - if (!File.Exists(Path.Combine(ucdDirectory, ucdRequiredFile))) + if (!File.Exists(Path.Combine(dataDirectory, requiredFile))) { hasValidDirectory = false; break; @@ -57,47 +64,47 @@ namespace System.Unicode.Builder if (hasValidDirectory) { - return new FileUcdSource(ucdDirectory); + return new FileDataSource(dataDirectory); } - if (File.Exists(ucdArchiveFileName)) + if (File.Exists(dataArchiveFileName)) { if (shouldExtract == true) { - ZipFile.ExtractToDirectory(ucdArchiveFileName, ucdDirectory); - return new FileUcdSource(ucdDirectory); + ZipFile.ExtractToDirectory(dataArchiveFileName, dataDirectory); + return new FileDataSource(dataDirectory); } else { - return new ZipUcdSource(File.OpenRead(ucdArchiveFileName)); + return new ZipDataSource(File.OpenRead(dataArchiveFileName)); } } } if (shouldDownload != false) { - var ucdArchiveData = DownloadUcdArchive(); + var dataArchiveData = DownloadDataArchive(archiveName); if (shouldSaveArchive == true) { - var stream = File.Open(ucdArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read); + var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read); try { - stream.Write(ucdArchiveData, 0, ucdArchiveData.Length); - ucdArchiveData = null; // Release the reference now, since we won't need it anymore. + stream.Write(dataArchiveData, 0, dataArchiveData.Length); + dataArchiveData = null; // Release the reference now, since we won't need it anymore. if (shouldExtract == true) { using (var archive = new ZipArchive(stream, ZipArchiveMode.Read, false)) { - archive.ExtractToDirectory(ucdDirectory); + archive.ExtractToDirectory(dataDirectory); - return new FileUcdSource(ucdDirectory); + return new FileDataSource(dataDirectory); } } else { - return new ZipUcdSource(stream); + return new ZipDataSource(stream); } } catch @@ -108,7 +115,7 @@ namespace System.Unicode.Builder } else { - return new ZipUcdSource(new MemoryStream(ucdArchiveData)); + return new ZipDataSource(new MemoryStream(dataArchiveData)); } } @@ -119,9 +126,10 @@ namespace System.Unicode.Builder { UnicodeInfoBuilder data; - using (var ucdSource = GetUcdSource(null, null, null)) + using (var ucdSource = GetDataSource(UcdArchiveName, directoryName, ucdRequiredFiles, null, null, null)) + using (var unihanSource = GetDataSource(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null)) { - data = UnicodeDataProcessor.BuildDataAsync(ucdSource).Result; + data = UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource).Result; } using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false)) diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs index 29858d2..93a90a9 100644 --- a/UnicodeInformation.Builder/UnicodeDataProcessor.cs +++ b/UnicodeInformation.Builder/UnicodeDataProcessor.cs @@ -27,7 +27,7 @@ namespace System.Unicode.Builder return string.IsNullOrEmpty(s) ? null : s; } - public static async Task BuildDataAsync(IUcdSource ucdSource) + public static async Task BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource) { var builder = new UnicodeInfoBuilder(new Version(7, 0)); @@ -39,7 +39,7 @@ namespace System.Unicode.Builder return builder; } - private static async Task ProcessUnicodeDataFile(IUcdSource ucdSource, UnicodeInfoBuilder builder) + private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false))) { @@ -156,7 +156,7 @@ namespace System.Unicode.Builder } } - private static async Task ProcessPropListFile(IUcdSource ucdSource, UnicodeInfoBuilder builder) + private static async Task ProcessPropListFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(PropListFileName).ConfigureAwait(false))) { @@ -173,7 +173,7 @@ namespace System.Unicode.Builder } } - private static async Task ProcessDerivedCorePropertiesFile(IUcdSource ucdSource, UnicodeInfoBuilder builder) + private static async Task ProcessDerivedCorePropertiesFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(DerivedCorePropertiesFileName).ConfigureAwait(false))) { @@ -190,7 +190,7 @@ namespace System.Unicode.Builder } } - private static async Task ProcessBlocksFile(IUcdSource ucdSource, UnicodeInfoBuilder builder) + private static async Task ProcessBlocksFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false))) { diff --git a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj index f8b431d..1294119 100644 --- a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj +++ b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj @@ -54,16 +54,16 @@ - - - + + + - + diff --git a/UnicodeInformation.Builder/ZipUcdSource.cs b/UnicodeInformation.Builder/ZipDataSource.cs similarity index 87% rename from UnicodeInformation.Builder/ZipUcdSource.cs rename to UnicodeInformation.Builder/ZipDataSource.cs index f5faf08..3b0e0a7 100644 --- a/UnicodeInformation.Builder/ZipUcdSource.cs +++ b/UnicodeInformation.Builder/ZipDataSource.cs @@ -8,11 +8,11 @@ using System.Threading.Tasks; namespace System.Unicode.Builder { - public sealed class ZipUcdSource : IUcdSource + public sealed class ZipDataSource : IDataSource { private readonly ZipArchive archive; - public ZipUcdSource(Stream stream) + public ZipDataSource(Stream stream) { archive = new ZipArchive(stream, ZipArchiveMode.Read, false); } diff --git a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs index 1c25bf4..42ffd99 100644 --- a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs +++ b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs @@ -23,12 +23,19 @@ namespace System.Unicode.Tests { Directory.CreateDirectory(directoryName); - var fileName = Path.Combine(directoryName, "UCD.zip"); + string ucdFileName = Path.Combine(directoryName, "UCD.zip"); + string unihanFileName = Path.Combine(directoryName, "Unihan.zip"); - if (!File.Exists(fileName)) + if (!File.Exists(ucdFileName)) { - new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/UCD.zip", fileName); - ZipFile.ExtractToDirectory(fileName, directoryName); + new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/UCD.zip", ucdFileName); + ZipFile.ExtractToDirectory(ucdFileName, directoryName); + } + + if (!File.Exists(unihanFileName)) + { + new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip", unihanFileName); + ZipFile.ExtractToDirectory(unihanFileName, directoryName); } } } @@ -36,9 +43,9 @@ namespace System.Unicode.Tests [TestMethod] public async Task BuildDataAsync() { - var source = new FileUcdSource(UcdDirectoryName); + var source = new FileDataSource(UcdDirectoryName); - var data = (await UnicodeDataProcessor.BuildDataAsync(source)).ToUnicodeData(); + var data = (await UnicodeDataProcessor.BuildDataAsync(source, source)).ToUnicodeData(); Assert.AreEqual((int)'\t', data.GetCharInfo('\t').CodePoint); } @@ -46,9 +53,9 @@ namespace System.Unicode.Tests [TestMethod] public async Task BuildAndWriteDataAsync() { - var source = new FileUcdSource(UcdDirectoryName); + var source = new FileDataSource(UcdDirectoryName); - var data = await UnicodeDataProcessor.BuildDataAsync(source); + var data = await UnicodeDataProcessor.BuildDataAsync(source, source); using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false)) { diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index e089625..34c6758 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -76,7 +76,10 @@ + + + diff --git a/UnicodeInformation/UnihanCharacterData.cs b/UnicodeInformation/UnihanCharacterData.cs new file mode 100644 index 0000000..6a09986 --- /dev/null +++ b/UnicodeInformation/UnihanCharacterData.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + internal sealed class UnihanCharacterData + { + public readonly int CodePoint; + public readonly UnihanNumericType NumericType; + public readonly ulong NumericValue; + public readonly string Definition; + public readonly string MandarinReading; + public readonly string CantoneseReading; + public readonly string JapaneseKunReading; + public readonly string JapaneseOnReading; + public readonly string KoreanReading; + public readonly string HangulReading; + public readonly string VietnameseReading; + public readonly string SimplifiedVariant; + public readonly string TraditionalVariant; + + internal static int PackCodePoint(int codePoint) + { + if (codePoint >= 0x3400) + { + if (codePoint < 0x4E00) return codePoint + 0x1E00; + else if (codePoint < 0xA000) return codePoint - 0x4E00; + else if (codePoint >= 0xF900 && codePoint < 0xFB00) return codePoint + 0xFD00; + else if (codePoint >= 0x20000) + { + if (codePoint < 0x2F800) return codePoint - 0x19400; + else if (codePoint < 0x30000) return codePoint - 0x10000; + } + } + + throw new ArgumentOutOfRangeException("codePoint"); + } + + internal static int UnpackCodePoint(int packedCodePoint) + { + if (packedCodePoint >= 0) + { + if (packedCodePoint < 0x05200) return packedCodePoint + 0x4E00; + else if (packedCodePoint < 0x06C00) return packedCodePoint - 0x1E00; + else if (packedCodePoint < 0x1F600) return packedCodePoint + 0x19400; + else if (packedCodePoint < 0x1F800) return packedCodePoint - 0xFD00; + else if (packedCodePoint < 0x20000) return packedCodePoint + 0x10000; + } + throw new ArgumentOutOfRangeException("packedCodePoint"); + } + } +} diff --git a/UnicodeInformation/UnihanFields.cs b/UnicodeInformation/UnihanFields.cs new file mode 100644 index 0000000..2d52757 --- /dev/null +++ b/UnicodeInformation/UnihanFields.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + [Flags] + internal enum UnihanFields : ushort + { + // NumericType / NumericValue : Not exactly a bit mask here… + AccountingNumeric = 1, + OtherNumeric = 2, + PrimaryNumeric = 3, + + Definition = 4, + MandarinReading = 8, + CantoneseReading = 16, + JapaneseKunReading = 32, + JapaneseOnReading = 64, + KoreanReading = 128, + HangulReading = 256, + VietnameseReading = 512, + + SimplifiedVariant = 1024, + TraditionalVariant = 2048, + } +} diff --git a/UnicodeInformation/UnihanNumericType.cs b/UnicodeInformation/UnihanNumericType.cs new file mode 100644 index 0000000..01db806 --- /dev/null +++ b/UnicodeInformation/UnihanNumericType.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + /// Represents the different numeric types from the Unihan database. + public enum UnihanNumericType : byte + { + None = 0, + [ValueName("kAccountingNumeric")] + Accounting = 1, + [ValueName("kOtherNumeric")] + Other = 2, + [ValueName("kPrimaryNumeric")] + Primary = 3 + } +}