diff --git a/UnicodeInformation.Builder/BinaryWriterExtensions.cs b/UnicodeInformation.Builder/BinaryWriterExtensions.cs new file mode 100644 index 0000000..95faee6 --- /dev/null +++ b/UnicodeInformation.Builder/BinaryWriterExtensions.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode.Builder +{ + internal static class BinaryWriterExtensions + { + //public static void WriteUInt24(this BinaryWriter writer, int value) + //{ + // if (value < 0 || value > 0xFFFFFF) throw new ArgumentOutOfRangeException("value"); + + // writer.Write((byte)(value)); + // writer.Write((byte)(value >> 8)); + // writer.Write((byte)(value >> 16)); + //} + + /// Writes code point in a custom, but compact encoding. + /// + /// Unlike UTF-8, this encoding will consume at most 3 bytes. + /// It could ideally store values between 0x0 and 0x40407F, but this range is useless at the moment. + /// + /// The binary writer to use. + /// The value to write + public static void WriteCodePoint(this BinaryWriter writer, int value) + { + if (value < 0 || value > 0x40407F) throw new ArgumentOutOfRangeException("value"); + + if (value < 0x80) writer.Write((byte)value); + else if (value < 0x4080) + { + value -= 0x80; + writer.Write((byte)((byte)(value >> 8) | 0x80)); + writer.Write((byte)value); + } + else + { + value -= 0x4080; + writer.Write((byte)((byte)(value >> 16) | 0xC0)); + writer.Write((byte)(value >> 8)); + writer.Write((byte)value); + } + } + } +} diff --git a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs index 5ac5f3c..bf8f5ab 100644 --- a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs +++ b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -147,5 +148,46 @@ namespace System.Unicode.Builder relatedCodePoints.Count > 0 ? relatedCodePoints.ToArray() : null ); } + + internal void WriteToFile(BinaryWriter writer) + { + UcdFields fields = default(UcdFields); + + if (!codePointRange.IsSingleCodePoint) fields = UcdFields.CodePointRange; + + if (name != null) fields |= UcdFields.Name; + if (category != UnicodeCategory.OtherNotAssigned) fields |= UcdFields.Category; + if (canonicalCombiningClass != CanonicalCombiningClass.NotReordered) fields |= UcdFields.CanonicalCombiningClass; + /*if (bidirectionalClass != 0)*/fields |= UcdFields.BidirectionalClass; + if (decompositionType != null) fields |= UcdFields.DecompositionType; + fields |= (UcdFields)((int)numericType << 6); + if (bidirectionalMirrored) fields |= UcdFields.BidirectionalMirrored; + if (oldName != null) fields |= UcdFields.OldName; + if (simpleUpperCaseMapping != null) fields |= UcdFields.SimpleUpperCaseMapping; + if (simpleLowerCaseMapping != null) fields |= UcdFields.SimpleLowerCaseMapping; + if (simpleTitleCaseMapping != null) fields |= UcdFields.SimpleTitleCaseMapping; + if (contributoryProperties != 0) fields |= UcdFields.ContributoryProperties; + + writer.Write((ushort)fields); + + writer.WriteCodePoint(codePointRange.FirstCodePoint); + if ((fields & UcdFields.CodePointRange) != 0) writer.WriteCodePoint(CodePointRange.LastCodePoint); + + if ((fields & UcdFields.Name) != 0) writer.Write(name); + if ((fields & UcdFields.Category) != 0) writer.Write((byte)category); + if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass); + if ((fields & UcdFields.BidirectionalClass) != 0) writer.Write((byte)bidirectionalClass); + if ((fields & UcdFields.DecompositionType) != 0) writer.Write(decompositionType); + if ((fields & UcdFields.NumericNumeric) != 0) + { + writer.Write(numericValue.Numerator); + writer.Write(numericValue.Denominator); + } + if ((fields & UcdFields.OldName) != 0) writer.Write(oldName); + if ((fields & UcdFields.SimpleUpperCaseMapping) != 0) writer.Write(simpleUpperCaseMapping); + if ((fields & UcdFields.SimpleLowerCaseMapping) != 0) writer.Write(simpleLowerCaseMapping); + if ((fields & UcdFields.SimpleTitleCaseMapping) != 0) writer.Write(simpleTitleCaseMapping); + if ((fields & UcdFields.ContributoryProperties) != 0) writer.Write((int)contributoryProperties); + } } } diff --git a/UnicodeInformation.Builder/UnicodeDataBuilder.cs b/UnicodeInformation.Builder/UnicodeDataBuilder.cs index 336b183..69d94c2 100644 --- a/UnicodeInformation.Builder/UnicodeDataBuilder.cs +++ b/UnicodeInformation.Builder/UnicodeDataBuilder.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -9,8 +10,8 @@ namespace System.Unicode.Builder public class UnicodeDataBuilder { private readonly Version unicodeVersion; - private UnicodeCharacterDataBuilder[] characterDataBuilders = new UnicodeCharacterDataBuilder[10000]; - private int characterCount; + private UnicodeCharacterDataBuilder[] entries = new UnicodeCharacterDataBuilder[10000]; + private int entryCount; public UnicodeDataBuilder(Version unicodeVersion) { @@ -19,16 +20,16 @@ namespace System.Unicode.Builder private int FindCodePoint(int codePoint) { - if (characterCount == 0) return -1; + if (entryCount == 0) return -1; int minIndex = 0; - int maxIndex = characterCount - 1; + int maxIndex = entryCount - 1; do { int index = (minIndex + maxIndex) >> 1; - int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(codePoint); + int Δ = entries[index].CodePointRange.CompareCodePoint(codePoint); if (Δ == 0) return index; else if (Δ < 0) maxIndex = index - 1; @@ -43,22 +44,22 @@ namespace System.Unicode.Builder int minIndex; int maxIndex; - if (characterCount == 0 || characterDataBuilders[maxIndex = characterCount - 1].CodePointRange.LastCodePoint < startCodePoint) return characterCount; - else if (endCodePoint < characterDataBuilders[minIndex = 0].CodePointRange.FirstCodePoint) return 0; - else if (characterCount == 1) return -1; + if (entryCount == 0 || entries[maxIndex = entryCount - 1].CodePointRange.LastCodePoint < startCodePoint) return entryCount; + else if (endCodePoint < entries[minIndex = 0].CodePointRange.FirstCodePoint) return 0; + else if (entryCount == 1) return -1; do { int index = (minIndex + maxIndex) >> 1; - int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(startCodePoint); + int Δ = entries[index].CodePointRange.CompareCodePoint(startCodePoint); if (Δ == 0) return -1; else if (Δ < 0) maxIndex = index; else minIndex = index; } while (maxIndex - minIndex > 1); - if (characterDataBuilders[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1; + if (entries[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1; else return maxIndex; } @@ -68,25 +69,25 @@ namespace System.Unicode.Builder if (insertionPoint < 0) throw new InvalidOperationException("The specified range overlaps with pre-existing ranges."); - if (insertionPoint >= characterDataBuilders.Length) + if (insertionPoint >= entries.Length) { - Array.Resize(ref characterDataBuilders, characterDataBuilders.Length << 1); + Array.Resize(ref entries, entries.Length << 1); } - if (insertionPoint < characterCount) + if (insertionPoint < entryCount) { - Array.Copy(characterDataBuilders, insertionPoint, characterDataBuilders, insertionPoint + 1, characterCount - insertionPoint); + Array.Copy(entries, insertionPoint, entries, insertionPoint + 1, entryCount - insertionPoint); } - characterDataBuilders[insertionPoint] = data; - ++characterCount; + entries[insertionPoint] = data; + ++entryCount; } public UnicodeCharacterDataBuilder Get(int codePoint) { int index = FindCodePoint(codePoint); - return index >= 0 ? characterDataBuilders[index] : null; + return index >= 0 ? entries[index] : null; } public void SetProperties(ContributoryProperties property, UnicodeCharacterRange codePointRange) @@ -102,8 +103,8 @@ namespace System.Unicode.Builder if (firstIndex < 0 || lastIndex < 0 - || characterDataBuilders[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint - || characterDataBuilders[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint) + || entries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint + || entries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint) { throw new InvalidOperationException(); } @@ -112,7 +113,7 @@ namespace System.Unicode.Builder while (true) { - characterDataBuilders[i].ContributoryProperties |= property; + entries[i].ContributoryProperties |= property; if (i == lastIndex) break; @@ -122,12 +123,35 @@ namespace System.Unicode.Builder public UnicodeData ToUnicodeData() { - var finalData = new UnicodeCharacterData[characterCount]; + var finalData = new UnicodeCharacterData[entryCount]; for (int i = 0; i < finalData.Length; ++i) - finalData[i] = characterDataBuilders[i].ToCharacterData(); + finalData[i] = entries[i].ToCharacterData(); return new UnicodeData(unicodeVersion, finalData); } + + public void WriteToStream(Stream stream) + { + using (var writer = new BinaryWriter(stream, Encoding.UTF8, true)) + { + writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 0 }); + writer.Write((ushort)7); + writer.Write((byte)0); + writer.WriteCodePoint(entryCount); + for (int i = 0; i < entryCount; ++i) + { + entries[i].WriteToFile(writer); + } + } + } + + public void WriteToFile(string fileName) + { + using (var stream = File.Create(fileName)) + { + WriteToStream(stream); + } + } } } diff --git a/UnicodeInformation.Builder/UnicodeDataManager.cs b/UnicodeInformation.Builder/UnicodeDataManager.cs index 4efb6f5..1bb0110 100644 --- a/UnicodeInformation.Builder/UnicodeDataManager.cs +++ b/UnicodeInformation.Builder/UnicodeDataManager.cs @@ -25,7 +25,7 @@ namespace System.Unicode.Builder return string.IsNullOrEmpty(s) ? null : s; } - public static async Task DownloadAndBuildDataAsync(IUcdSource ucdSource) + public static async Task BuildDataAsync(IUcdSource ucdSource) { var builder = new UnicodeDataBuilder(new Version(7, 0)); @@ -153,7 +153,7 @@ namespace System.Unicode.Builder } } - return builder.ToUnicodeData(); + return builder; } } } diff --git a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj index c918bc8..65cb29b 100644 --- a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj +++ b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj @@ -48,6 +48,7 @@ + diff --git a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs index 0b234e9..c96bd17 100644 --- a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs +++ b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs @@ -33,13 +33,33 @@ namespace System.Unicode.Tests } [TestMethod] - public async Task DownloadAndBuildDataAsync() + public async Task BuildDataAsync() { var source = new FileUcdSource(UcdDirectoryName); - var data = await UnicodeDataManager.DownloadAndBuildDataAsync(source); + var data = (await UnicodeDataManager.BuildDataAsync(source)).ToUnicodeData(); Assert.AreEqual((int)'\t', data.GetUnicodeData('\t').CodePointRange.FirstCodePoint); } + + [TestMethod] + public async Task BuildAndWriteDataAsync() + { + var source = new FileUcdSource(UcdDirectoryName); + + var data = await UnicodeDataManager.BuildDataAsync(source); + + using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false)) + { + data.WriteToStream(stream); + } + + using (var stream = new DeflateStream(File.OpenRead("ucd.dat"), CompressionMode.Decompress, false)) + { + var readData = UnicodeData.FromStream(stream); + + Assert.AreEqual((int)'\t', data.Get('\t').CodePointRange.FirstCodePoint); + } + } } } diff --git a/UnicodeInformation/UcdFields.cs b/UnicodeInformation/UcdFields.cs new file mode 100644 index 0000000..cbe07af --- /dev/null +++ b/UnicodeInformation/UcdFields.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + /// Represents the fields available for an UCD entry. + /// Not all the enumeration member directly map to a field. + [Flags] + internal enum UcdFields : ushort + { + // Not really a field, just here to indicate that the entry is a range + CodePointRange = 1, + + Name = 2, + Category = 4, + CanonicalCombiningClass = 8, + BidirectionalClass = 16, + DecompositionType = 32, + + // NumericType / NumericValue : Not exactly a bit mask here… More like [0…3] << 6 + NumericDecimal = 64, + NumericDigit = 128, + NumericNumeric = 192, + + // This is a yes/no field, so obviously, no extra storage is required for this one… + BidirectionalMirrored = 256, + + OldName = 512, + SimpleUpperCaseMapping = 1024, + SimpleLowerCaseMapping = 2048, + SimpleTitleCaseMapping = 4096, + ContributoryProperties = 8192, + } +} diff --git a/UnicodeInformation/UnicodeCharacterData.cs b/UnicodeInformation/UnicodeCharacterData.cs index d749af3..c6e0c9f 100644 --- a/UnicodeInformation/UnicodeCharacterData.cs +++ b/UnicodeInformation/UnicodeCharacterData.cs @@ -10,13 +10,13 @@ namespace System.Unicode public sealed class UnicodeCharacterData { public readonly UnicodeCharacterRange CodePointRange; - private readonly string name; + public readonly string Name; public readonly UnicodeCategory Category; public readonly CanonicalCombiningClass CanonicalCombiningClass; public readonly BidirectionalClass BidirectionalClass; public readonly string DecompositionType; public readonly UnicodeNumericType NumericType; - public readonly UnicodeRationalNumber NumericValue; + private readonly UnicodeRationalNumber numericValue; public readonly bool BidirectionalMirrored; public readonly string OldName; public readonly string SimpleUpperCaseMapping; @@ -46,13 +46,13 @@ namespace System.Unicode ) { this.CodePointRange = codePointRange; - this.name = name; + this.Name = name; this.Category = category; this.CanonicalCombiningClass = canonicalCombiningClass; this.BidirectionalClass = bidirectionalClass; this.DecompositionType = decompositionType; this.NumericType = numericType; - this.NumericValue = numericValue; + this.numericValue = numericValue; this.BidirectionalMirrored = bidirectionalMirrored; this.OldName = oldName; this.SimpleUpperCaseMapping = simpleUpperCaseMapping; @@ -62,8 +62,6 @@ namespace System.Unicode this.RelatedCodePoints = relatedCodePoints; } - public bool IsRange { get { return CodePointRange.FirstCodePoint != CodePointRange.LastCodePoint; } } - - public string Name { get { return !IsRange ? name : null; } } + public UnicodeRationalNumber? NumericValue { get { return NumericType != UnicodeNumericType.None ? numericValue : null as UnicodeRationalNumber?; } } } } diff --git a/UnicodeInformation/UnicodeData.cs b/UnicodeInformation/UnicodeData.cs index 79d6867..7402c2a 100644 --- a/UnicodeInformation/UnicodeData.cs +++ b/UnicodeInformation/UnicodeData.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -12,25 +13,93 @@ namespace System.Unicode private readonly Version unicodeVersion; private readonly UnicodeCharacterData[] characterData; - public static async Task FromStreamAsync(Stream stream) + public static UnicodeData FromStream(Stream stream) { - var buffer = new byte[4096]; + using (var reader = new BinaryReader(stream, Encoding.UTF8)) + { + if (reader.ReadByte() != 'U' + | reader.ReadByte() != 'C' + | reader.ReadByte() != 'D') + throw new InvalidDataException(); - if (await stream.ReadAsync(buffer, 0, 6).ConfigureAwait(false) != 6) - throw new EndOfStreamException(); + byte formatVersion = reader.ReadByte(); - if (buffer[0] != (byte)'U' - || buffer[1] != (byte)'C' - || buffer[2] != (byte)'D') - throw new InvalidDataException(); +#if !DEBUG + if (formatVersion != 1) throw new InvalidDataException(); +#endif - int formatVersion = buffer[3]; + var unicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte()); - if (formatVersion != 1) throw new InvalidDataException(); + var entries = new UnicodeCharacterData[ReadCodePoint(reader)]; - var unicodeVersion = new Version(buffer[4], buffer[6]); + for (int i = 0; i < entries.Length; ++i) + { + entries[i] = ReadEntry(reader); + } - return new UnicodeData(unicodeVersion, null); + return new UnicodeData(unicodeVersion, entries); + } + } + + private static UnicodeCharacterData ReadEntry(BinaryReader reader) + { + var fields = (UcdFields)reader.ReadUInt16(); + + var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCharacterRange(ReadCodePoint(reader)); + + string name = (fields & UcdFields.Name) != 0 ? reader.ReadString() : null; + var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned; + var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered; + var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0; + string decompositionType = (fields & UcdFields.DecompositionType) != 0 ? reader.ReadString() : null; + var numericType = (UnicodeNumericType)((int)(fields & UcdFields.NumericNumeric) >> 6); + UnicodeRationalNumber numericValue = numericType != UnicodeNumericType.None ? + new UnicodeRationalNumber(reader.ReadInt64(), reader.ReadByte()) : + default(UnicodeRationalNumber); + string oldName = (fields & UcdFields.OldName) != 0 ? reader.ReadString() : null; + string simpleUpperCaseMapping = (fields & UcdFields.SimpleUpperCaseMapping) != 0 ? reader.ReadString() : null; + string simpleLowerCaseMapping = (fields & UcdFields.SimpleLowerCaseMapping) != 0 ? reader.ReadString() : null; + string simpleTitleCaseMapping = (fields & UcdFields.SimpleTitleCaseMapping) != 0 ? reader.ReadString() : null; + ContributoryProperties contributoryProperties = (fields & UcdFields.ContributoryProperties) != 0 ? (ContributoryProperties)reader.ReadInt32() : 0; + + return new UnicodeCharacterData + ( + codePointRange, + name, + category, + canonicalCombiningClass, + bidirectionalClass, + decompositionType, + numericType, + numericValue, + (fields & UcdFields.BidirectionalMirrored) != 0, + oldName, + simpleUpperCaseMapping, + simpleLowerCaseMapping, + simpleTitleCaseMapping, + contributoryProperties, + null + ); + } + + //private static int ReadInt24(BinaryReader reader) + //{ + // return reader.ReadByte() | ((reader.ReadByte() | (reader.ReadByte() << 8)) << 8); + //} + + private static int ReadCodePoint(BinaryReader reader) + { + byte b = reader.ReadByte(); + + if (b < 128) return b; + else if ((b & 0x40) == 0) + { + return 128 + (((b & 0x3F) << 8) | reader.ReadByte()); + } + else + { + return 0x4080 + (((((b & 0x3F) << 8) | reader.ReadByte()) << 8) | reader.ReadByte()); + } } internal UnicodeData(Version unicodeVersion, UnicodeCharacterData[] characterData) @@ -64,9 +133,5 @@ namespace System.Unicode { return FindCodePoint(codePoint); } - - private void LoadData(Stream stream) - { - } } } diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index b5dda56..0edcfe8 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -59,6 +59,7 @@ +