diff --git a/UnicodeInformation.Builder/BinaryWriterExtensions.cs b/UnicodeInformation.Builder/BinaryWriterExtensions.cs
new file mode 100644
index 0000000..95faee6
--- /dev/null
+++ b/UnicodeInformation.Builder/BinaryWriterExtensions.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Unicode.Builder
+{
+ internal static class BinaryWriterExtensions
+ {
+ //public static void WriteUInt24(this BinaryWriter writer, int value)
+ //{
+ // if (value < 0 || value > 0xFFFFFF) throw new ArgumentOutOfRangeException("value");
+
+ // writer.Write((byte)(value));
+ // writer.Write((byte)(value >> 8));
+ // writer.Write((byte)(value >> 16));
+ //}
+
+ /// Writes code point in a custom, but compact encoding.
+ ///
+ /// Unlike UTF-8, this encoding will consume at most 3 bytes.
+ /// It could ideally store values between 0x0 and 0x40407F, but this range is useless at the moment.
+ ///
+ /// The binary writer to use.
+ /// The value to write
+ public static void WriteCodePoint(this BinaryWriter writer, int value)
+ {
+ if (value < 0 || value > 0x40407F) throw new ArgumentOutOfRangeException("value");
+
+ if (value < 0x80) writer.Write((byte)value);
+ else if (value < 0x4080)
+ {
+ value -= 0x80;
+ writer.Write((byte)((byte)(value >> 8) | 0x80));
+ writer.Write((byte)value);
+ }
+ else
+ {
+ value -= 0x4080;
+ writer.Write((byte)((byte)(value >> 16) | 0xC0));
+ writer.Write((byte)(value >> 8));
+ writer.Write((byte)value);
+ }
+ }
+ }
+}
diff --git a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs
index 5ac5f3c..bf8f5ab 100644
--- a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs
+++ b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
+using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@@ -147,5 +148,46 @@ namespace System.Unicode.Builder
relatedCodePoints.Count > 0 ? relatedCodePoints.ToArray() : null
);
}
+
+ internal void WriteToFile(BinaryWriter writer)
+ {
+ UcdFields fields = default(UcdFields);
+
+ if (!codePointRange.IsSingleCodePoint) fields = UcdFields.CodePointRange;
+
+ if (name != null) fields |= UcdFields.Name;
+ if (category != UnicodeCategory.OtherNotAssigned) fields |= UcdFields.Category;
+ if (canonicalCombiningClass != CanonicalCombiningClass.NotReordered) fields |= UcdFields.CanonicalCombiningClass;
+ /*if (bidirectionalClass != 0)*/fields |= UcdFields.BidirectionalClass;
+ if (decompositionType != null) fields |= UcdFields.DecompositionType;
+ fields |= (UcdFields)((int)numericType << 6);
+ if (bidirectionalMirrored) fields |= UcdFields.BidirectionalMirrored;
+ if (oldName != null) fields |= UcdFields.OldName;
+ if (simpleUpperCaseMapping != null) fields |= UcdFields.SimpleUpperCaseMapping;
+ if (simpleLowerCaseMapping != null) fields |= UcdFields.SimpleLowerCaseMapping;
+ if (simpleTitleCaseMapping != null) fields |= UcdFields.SimpleTitleCaseMapping;
+ if (contributoryProperties != 0) fields |= UcdFields.ContributoryProperties;
+
+ writer.Write((ushort)fields);
+
+ writer.WriteCodePoint(codePointRange.FirstCodePoint);
+ if ((fields & UcdFields.CodePointRange) != 0) writer.WriteCodePoint(CodePointRange.LastCodePoint);
+
+ if ((fields & UcdFields.Name) != 0) writer.Write(name);
+ if ((fields & UcdFields.Category) != 0) writer.Write((byte)category);
+ if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass);
+ if ((fields & UcdFields.BidirectionalClass) != 0) writer.Write((byte)bidirectionalClass);
+ if ((fields & UcdFields.DecompositionType) != 0) writer.Write(decompositionType);
+ if ((fields & UcdFields.NumericNumeric) != 0)
+ {
+ writer.Write(numericValue.Numerator);
+ writer.Write(numericValue.Denominator);
+ }
+ if ((fields & UcdFields.OldName) != 0) writer.Write(oldName);
+ if ((fields & UcdFields.SimpleUpperCaseMapping) != 0) writer.Write(simpleUpperCaseMapping);
+ if ((fields & UcdFields.SimpleLowerCaseMapping) != 0) writer.Write(simpleLowerCaseMapping);
+ if ((fields & UcdFields.SimpleTitleCaseMapping) != 0) writer.Write(simpleTitleCaseMapping);
+ if ((fields & UcdFields.ContributoryProperties) != 0) writer.Write((int)contributoryProperties);
+ }
}
}
diff --git a/UnicodeInformation.Builder/UnicodeDataBuilder.cs b/UnicodeInformation.Builder/UnicodeDataBuilder.cs
index 336b183..69d94c2 100644
--- a/UnicodeInformation.Builder/UnicodeDataBuilder.cs
+++ b/UnicodeInformation.Builder/UnicodeDataBuilder.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@@ -9,8 +10,8 @@ namespace System.Unicode.Builder
public class UnicodeDataBuilder
{
private readonly Version unicodeVersion;
- private UnicodeCharacterDataBuilder[] characterDataBuilders = new UnicodeCharacterDataBuilder[10000];
- private int characterCount;
+ private UnicodeCharacterDataBuilder[] entries = new UnicodeCharacterDataBuilder[10000];
+ private int entryCount;
public UnicodeDataBuilder(Version unicodeVersion)
{
@@ -19,16 +20,16 @@ namespace System.Unicode.Builder
private int FindCodePoint(int codePoint)
{
- if (characterCount == 0) return -1;
+ if (entryCount == 0) return -1;
int minIndex = 0;
- int maxIndex = characterCount - 1;
+ int maxIndex = entryCount - 1;
do
{
int index = (minIndex + maxIndex) >> 1;
- int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(codePoint);
+ int Δ = entries[index].CodePointRange.CompareCodePoint(codePoint);
if (Δ == 0) return index;
else if (Δ < 0) maxIndex = index - 1;
@@ -43,22 +44,22 @@ namespace System.Unicode.Builder
int minIndex;
int maxIndex;
- if (characterCount == 0 || characterDataBuilders[maxIndex = characterCount - 1].CodePointRange.LastCodePoint < startCodePoint) return characterCount;
- else if (endCodePoint < characterDataBuilders[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
- else if (characterCount == 1) return -1;
+ if (entryCount == 0 || entries[maxIndex = entryCount - 1].CodePointRange.LastCodePoint < startCodePoint) return entryCount;
+ else if (endCodePoint < entries[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
+ else if (entryCount == 1) return -1;
do
{
int index = (minIndex + maxIndex) >> 1;
- int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(startCodePoint);
+ int Δ = entries[index].CodePointRange.CompareCodePoint(startCodePoint);
if (Δ == 0) return -1;
else if (Δ < 0) maxIndex = index;
else minIndex = index;
} while (maxIndex - minIndex > 1);
- if (characterDataBuilders[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
+ if (entries[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
else return maxIndex;
}
@@ -68,25 +69,25 @@ namespace System.Unicode.Builder
if (insertionPoint < 0) throw new InvalidOperationException("The specified range overlaps with pre-existing ranges.");
- if (insertionPoint >= characterDataBuilders.Length)
+ if (insertionPoint >= entries.Length)
{
- Array.Resize(ref characterDataBuilders, characterDataBuilders.Length << 1);
+ Array.Resize(ref entries, entries.Length << 1);
}
- if (insertionPoint < characterCount)
+ if (insertionPoint < entryCount)
{
- Array.Copy(characterDataBuilders, insertionPoint, characterDataBuilders, insertionPoint + 1, characterCount - insertionPoint);
+ Array.Copy(entries, insertionPoint, entries, insertionPoint + 1, entryCount - insertionPoint);
}
- characterDataBuilders[insertionPoint] = data;
- ++characterCount;
+ entries[insertionPoint] = data;
+ ++entryCount;
}
public UnicodeCharacterDataBuilder Get(int codePoint)
{
int index = FindCodePoint(codePoint);
- return index >= 0 ? characterDataBuilders[index] : null;
+ return index >= 0 ? entries[index] : null;
}
public void SetProperties(ContributoryProperties property, UnicodeCharacterRange codePointRange)
@@ -102,8 +103,8 @@ namespace System.Unicode.Builder
if (firstIndex < 0
|| lastIndex < 0
- || characterDataBuilders[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
- || characterDataBuilders[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
+ || entries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
+ || entries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException();
}
@@ -112,7 +113,7 @@ namespace System.Unicode.Builder
while (true)
{
- characterDataBuilders[i].ContributoryProperties |= property;
+ entries[i].ContributoryProperties |= property;
if (i == lastIndex) break;
@@ -122,12 +123,35 @@ namespace System.Unicode.Builder
public UnicodeData ToUnicodeData()
{
- var finalData = new UnicodeCharacterData[characterCount];
+ var finalData = new UnicodeCharacterData[entryCount];
for (int i = 0; i < finalData.Length; ++i)
- finalData[i] = characterDataBuilders[i].ToCharacterData();
+ finalData[i] = entries[i].ToCharacterData();
return new UnicodeData(unicodeVersion, finalData);
}
+
+ public void WriteToStream(Stream stream)
+ {
+ using (var writer = new BinaryWriter(stream, Encoding.UTF8, true))
+ {
+ writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 0 });
+ writer.Write((ushort)7);
+ writer.Write((byte)0);
+ writer.WriteCodePoint(entryCount);
+ for (int i = 0; i < entryCount; ++i)
+ {
+ entries[i].WriteToFile(writer);
+ }
+ }
+ }
+
+ public void WriteToFile(string fileName)
+ {
+ using (var stream = File.Create(fileName))
+ {
+ WriteToStream(stream);
+ }
+ }
}
}
diff --git a/UnicodeInformation.Builder/UnicodeDataManager.cs b/UnicodeInformation.Builder/UnicodeDataManager.cs
index 4efb6f5..1bb0110 100644
--- a/UnicodeInformation.Builder/UnicodeDataManager.cs
+++ b/UnicodeInformation.Builder/UnicodeDataManager.cs
@@ -25,7 +25,7 @@ namespace System.Unicode.Builder
return string.IsNullOrEmpty(s) ? null : s;
}
- public static async Task DownloadAndBuildDataAsync(IUcdSource ucdSource)
+ public static async Task BuildDataAsync(IUcdSource ucdSource)
{
var builder = new UnicodeDataBuilder(new Version(7, 0));
@@ -153,7 +153,7 @@ namespace System.Unicode.Builder
}
}
- return builder.ToUnicodeData();
+ return builder;
}
}
}
diff --git a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
index c918bc8..65cb29b 100644
--- a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
+++ b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
@@ -48,6 +48,7 @@
+
diff --git a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
index 0b234e9..c96bd17 100644
--- a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
+++ b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
@@ -33,13 +33,33 @@ namespace System.Unicode.Tests
}
[TestMethod]
- public async Task DownloadAndBuildDataAsync()
+ public async Task BuildDataAsync()
{
var source = new FileUcdSource(UcdDirectoryName);
- var data = await UnicodeDataManager.DownloadAndBuildDataAsync(source);
+ var data = (await UnicodeDataManager.BuildDataAsync(source)).ToUnicodeData();
Assert.AreEqual((int)'\t', data.GetUnicodeData('\t').CodePointRange.FirstCodePoint);
}
+
+ [TestMethod]
+ public async Task BuildAndWriteDataAsync()
+ {
+ var source = new FileUcdSource(UcdDirectoryName);
+
+ var data = await UnicodeDataManager.BuildDataAsync(source);
+
+ using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
+ {
+ data.WriteToStream(stream);
+ }
+
+ using (var stream = new DeflateStream(File.OpenRead("ucd.dat"), CompressionMode.Decompress, false))
+ {
+ var readData = UnicodeData.FromStream(stream);
+
+ Assert.AreEqual((int)'\t', data.Get('\t').CodePointRange.FirstCodePoint);
+ }
+ }
}
}
diff --git a/UnicodeInformation/UcdFields.cs b/UnicodeInformation/UcdFields.cs
new file mode 100644
index 0000000..cbe07af
--- /dev/null
+++ b/UnicodeInformation/UcdFields.cs
@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Unicode
+{
+ /// Represents the fields available for an UCD entry.
+ /// Not all the enumeration member directly map to a field.
+ [Flags]
+ internal enum UcdFields : ushort
+ {
+ // Not really a field, just here to indicate that the entry is a range
+ CodePointRange = 1,
+
+ Name = 2,
+ Category = 4,
+ CanonicalCombiningClass = 8,
+ BidirectionalClass = 16,
+ DecompositionType = 32,
+
+ // NumericType / NumericValue : Not exactly a bit mask here… More like [0…3] << 6
+ NumericDecimal = 64,
+ NumericDigit = 128,
+ NumericNumeric = 192,
+
+ // This is a yes/no field, so obviously, no extra storage is required for this one…
+ BidirectionalMirrored = 256,
+
+ OldName = 512,
+ SimpleUpperCaseMapping = 1024,
+ SimpleLowerCaseMapping = 2048,
+ SimpleTitleCaseMapping = 4096,
+ ContributoryProperties = 8192,
+ }
+}
diff --git a/UnicodeInformation/UnicodeCharacterData.cs b/UnicodeInformation/UnicodeCharacterData.cs
index d749af3..c6e0c9f 100644
--- a/UnicodeInformation/UnicodeCharacterData.cs
+++ b/UnicodeInformation/UnicodeCharacterData.cs
@@ -10,13 +10,13 @@ namespace System.Unicode
public sealed class UnicodeCharacterData
{
public readonly UnicodeCharacterRange CodePointRange;
- private readonly string name;
+ public readonly string Name;
public readonly UnicodeCategory Category;
public readonly CanonicalCombiningClass CanonicalCombiningClass;
public readonly BidirectionalClass BidirectionalClass;
public readonly string DecompositionType;
public readonly UnicodeNumericType NumericType;
- public readonly UnicodeRationalNumber NumericValue;
+ private readonly UnicodeRationalNumber numericValue;
public readonly bool BidirectionalMirrored;
public readonly string OldName;
public readonly string SimpleUpperCaseMapping;
@@ -46,13 +46,13 @@ namespace System.Unicode
)
{
this.CodePointRange = codePointRange;
- this.name = name;
+ this.Name = name;
this.Category = category;
this.CanonicalCombiningClass = canonicalCombiningClass;
this.BidirectionalClass = bidirectionalClass;
this.DecompositionType = decompositionType;
this.NumericType = numericType;
- this.NumericValue = numericValue;
+ this.numericValue = numericValue;
this.BidirectionalMirrored = bidirectionalMirrored;
this.OldName = oldName;
this.SimpleUpperCaseMapping = simpleUpperCaseMapping;
@@ -62,8 +62,6 @@ namespace System.Unicode
this.RelatedCodePoints = relatedCodePoints;
}
- public bool IsRange { get { return CodePointRange.FirstCodePoint != CodePointRange.LastCodePoint; } }
-
- public string Name { get { return !IsRange ? name : null; } }
+ public UnicodeRationalNumber? NumericValue { get { return NumericType != UnicodeNumericType.None ? numericValue : null as UnicodeRationalNumber?; } }
}
}
diff --git a/UnicodeInformation/UnicodeData.cs b/UnicodeInformation/UnicodeData.cs
index 79d6867..7402c2a 100644
--- a/UnicodeInformation/UnicodeData.cs
+++ b/UnicodeInformation/UnicodeData.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
@@ -12,25 +13,93 @@ namespace System.Unicode
private readonly Version unicodeVersion;
private readonly UnicodeCharacterData[] characterData;
- public static async Task FromStreamAsync(Stream stream)
+ public static UnicodeData FromStream(Stream stream)
{
- var buffer = new byte[4096];
+ using (var reader = new BinaryReader(stream, Encoding.UTF8))
+ {
+ if (reader.ReadByte() != 'U'
+ | reader.ReadByte() != 'C'
+ | reader.ReadByte() != 'D')
+ throw new InvalidDataException();
- if (await stream.ReadAsync(buffer, 0, 6).ConfigureAwait(false) != 6)
- throw new EndOfStreamException();
+ byte formatVersion = reader.ReadByte();
- if (buffer[0] != (byte)'U'
- || buffer[1] != (byte)'C'
- || buffer[2] != (byte)'D')
- throw new InvalidDataException();
+#if !DEBUG
+ if (formatVersion != 1) throw new InvalidDataException();
+#endif
- int formatVersion = buffer[3];
+ var unicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
- if (formatVersion != 1) throw new InvalidDataException();
+ var entries = new UnicodeCharacterData[ReadCodePoint(reader)];
- var unicodeVersion = new Version(buffer[4], buffer[6]);
+ for (int i = 0; i < entries.Length; ++i)
+ {
+ entries[i] = ReadEntry(reader);
+ }
- return new UnicodeData(unicodeVersion, null);
+ return new UnicodeData(unicodeVersion, entries);
+ }
+ }
+
+ private static UnicodeCharacterData ReadEntry(BinaryReader reader)
+ {
+ var fields = (UcdFields)reader.ReadUInt16();
+
+ var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCharacterRange(ReadCodePoint(reader));
+
+ string name = (fields & UcdFields.Name) != 0 ? reader.ReadString() : null;
+ var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned;
+ var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered;
+ var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0;
+ string decompositionType = (fields & UcdFields.DecompositionType) != 0 ? reader.ReadString() : null;
+ var numericType = (UnicodeNumericType)((int)(fields & UcdFields.NumericNumeric) >> 6);
+ UnicodeRationalNumber numericValue = numericType != UnicodeNumericType.None ?
+ new UnicodeRationalNumber(reader.ReadInt64(), reader.ReadByte()) :
+ default(UnicodeRationalNumber);
+ string oldName = (fields & UcdFields.OldName) != 0 ? reader.ReadString() : null;
+ string simpleUpperCaseMapping = (fields & UcdFields.SimpleUpperCaseMapping) != 0 ? reader.ReadString() : null;
+ string simpleLowerCaseMapping = (fields & UcdFields.SimpleLowerCaseMapping) != 0 ? reader.ReadString() : null;
+ string simpleTitleCaseMapping = (fields & UcdFields.SimpleTitleCaseMapping) != 0 ? reader.ReadString() : null;
+ ContributoryProperties contributoryProperties = (fields & UcdFields.ContributoryProperties) != 0 ? (ContributoryProperties)reader.ReadInt32() : 0;
+
+ return new UnicodeCharacterData
+ (
+ codePointRange,
+ name,
+ category,
+ canonicalCombiningClass,
+ bidirectionalClass,
+ decompositionType,
+ numericType,
+ numericValue,
+ (fields & UcdFields.BidirectionalMirrored) != 0,
+ oldName,
+ simpleUpperCaseMapping,
+ simpleLowerCaseMapping,
+ simpleTitleCaseMapping,
+ contributoryProperties,
+ null
+ );
+ }
+
+ //private static int ReadInt24(BinaryReader reader)
+ //{
+ // return reader.ReadByte() | ((reader.ReadByte() | (reader.ReadByte() << 8)) << 8);
+ //}
+
+ private static int ReadCodePoint(BinaryReader reader)
+ {
+ byte b = reader.ReadByte();
+
+ if (b < 128) return b;
+ else if ((b & 0x40) == 0)
+ {
+ return 128 + (((b & 0x3F) << 8) | reader.ReadByte());
+ }
+ else
+ {
+ return 0x4080 + (((((b & 0x3F) << 8) | reader.ReadByte()) << 8) | reader.ReadByte());
+ }
}
internal UnicodeData(Version unicodeVersion, UnicodeCharacterData[] characterData)
@@ -64,9 +133,5 @@ namespace System.Unicode
{
return FindCodePoint(codePoint);
}
-
- private void LoadData(Stream stream)
- {
- }
}
}
diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj
index b5dda56..0edcfe8 100644
--- a/UnicodeInformation/UnicodeInformation.csproj
+++ b/UnicodeInformation/UnicodeInformation.csproj
@@ -59,6 +59,7 @@
+