diff --git a/UnicodeInformation.Builder/BinaryWriterExtensions.cs b/UnicodeInformation.Builder/BinaryWriterExtensions.cs index 2e4bdf1..3f97206 100644 --- a/UnicodeInformation.Builder/BinaryWriterExtensions.cs +++ b/UnicodeInformation.Builder/BinaryWriterExtensions.cs @@ -50,5 +50,41 @@ namespace System.Unicode.Builder writer.Write((byte)value); } } + + /// Writes a character name alias. + /// We assume that character names will not exceed 64 bytes in length. + /// The writer to use. + /// The name alias value to write. + public static void WriteNameAliasToFile(this BinaryWriter writer, UnicodeNameAlias nameAlias) + { + WriteNameToFile(writer, (byte)(nameAlias.Kind - 1), nameAlias.Name); + } + + /// Writes a character name, packing two information bits along with the length. + /// We assume that character names will not exceed 64 bytes in length. + /// The writer to use. + /// Extra bits to pack with the 6 bit length. + /// The name to write. + public static void WriteNameToFile(this BinaryWriter writer, byte extraBits, string name) + { + // This method will stuff two extra bits together with the byte count, provided that this one doesn't exceed 64. + var bytes = Encoding.UTF8.GetBytes(name); + if (bytes.Length > 64) throw new InvalidOperationException("Did not expect UTF-8 encoded names to be longer than 64 bytes."); + writer.WritePackedLength(extraBits, name.Length); + writer.Write(bytes); + } + + /// Writes a 6 bits length packed with two extra bits. + /// The parameters have a restricted range, which must be respected. + /// The writer used to perform the operation. + /// The value of the two extra bits. + /// The length to write. + public static void WritePackedLength(this BinaryWriter writer, byte extraBits, int length) + { + if (extraBits > 3) throw new ArgumentOutOfRangeException("extraBits"); + if (length < 1 || length > 64) throw new ArgumentOutOfRangeException("length"); + + writer.Write((byte)((extraBits << 6) | (length - 1))); + } } } diff --git a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs index 5f5b58a..e0c17de 100644 --- a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs +++ b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs @@ -26,7 +26,8 @@ namespace System.Unicode.Builder private ContributoryProperties contributoryProperties; private CoreProperties coreProperties; - private List relatedCodePoints = new List(); + private readonly List nameAliases = new List(); + private readonly List relatedCodePoints = new List(); public UnicodeCharacterRange CodePointRange { get { return codePointRange; } } @@ -36,6 +37,8 @@ namespace System.Unicode.Builder set { name = value; } } + public ICollection NameAliases { get { return nameAliases; } } + public UnicodeCategory Category { get { return category; } @@ -138,33 +141,36 @@ namespace System.Unicode.Builder { return new UnicodeCharacterData ( - codePointRange, + CodePointRange, Name, - category, - canonicalCombiningClass, - bidirectionalClass, - characterDecompositionMapping.DecompositionType, - characterDecompositionMapping.DecompositionMapping, - numericType, - numericValue, - bidirectionalMirrored, - oldName, - simpleUpperCaseMapping, - simpleLowerCaseMapping, - simpleTitleCaseMapping, - contributoryProperties, - coreProperties, - relatedCodePoints.Count > 0 ? relatedCodePoints.ToArray() : null + NameAliases.Count > 0 ? NameAliases.ToArray() : UnicodeNameAlias.EmptyArray, + Category, + CanonicalCombiningClass, + BidirectionalClass, + CharacterDecompositionMapping.DecompositionType, + CharacterDecompositionMapping.DecompositionMapping, + NumericType, + NumericValue, + BidirectionalMirrored, + OldName, + SimpleUpperCaseMapping, + SimpleLowerCaseMapping, + SimpleTitleCaseMapping, + ContributoryProperties, + CoreProperties, + RelatedCodePoints.Count > 0 ? RelatedCodePoints.ToArray() : null ); } internal void WriteToFile(BinaryWriter writer) { + if (nameAliases.Count > 64) throw new InvalidDataException("Cannot handle more than 64 name aliases."); + UcdFields fields = default(UcdFields); if (!codePointRange.IsSingleCodePoint) fields = UcdFields.CodePointRange; - if (name != null) fields |= UcdFields.Name; + if (name != null || nameAliases.Count > 0) fields |= UcdFields.Name; // This field combines name and alias. if (category != UnicodeCategory.OtherNotAssigned) fields |= UcdFields.Category; if (canonicalCombiningClass != CanonicalCombiningClass.NotReordered) fields |= UcdFields.CanonicalCombiningClass; /*if (bidirectionalClass != 0)*/ @@ -184,7 +190,33 @@ namespace System.Unicode.Builder writer.WriteCodePoint(codePointRange.FirstCodePoint); if ((fields & UcdFields.CodePointRange) != 0) writer.WriteCodePoint(CodePointRange.LastCodePoint); - if ((fields & UcdFields.Name) != 0) writer.Write(name); + if ((fields & UcdFields.Name) != 0) + { + // We write the names by optimizing for the common case. + // i.e. Most characters have one name, and most of those characters have only one name. + // In the first byte, we will encode a 6bit length, along with two extra bits describing the structure: + // 00: The common case, this byte is the length of the name, and no other name follows. + // 01: This byte is the length of the name, and one alias follows. + // 10: This byte is the length of the name, and the number of aliases follows. + // 11: This byte is the number of aliases. + + if (name == null) + { + writer.WritePackedLength(3, nameAliases.Count); + } + else + { + writer.WriteNameToFile((byte)(nameAliases.Count > 0 ? nameAliases.Count > 1 ? 2 : 1 : 0), name); + } + + if (nameAliases.Count > 0) + { + if (name != null) writer.Write((byte)nameAliases.Count); + + foreach (var nameAlias in nameAliases) + writer.WriteNameAliasToFile(nameAlias); + } + } if ((fields & UcdFields.Category) != 0) writer.Write((byte)category); if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass); if ((fields & UcdFields.BidirectionalClass) != 0) writer.Write((byte)bidirectionalClass); diff --git a/UnicodeInformation/UcdFields.cs b/UnicodeInformation/UcdFields.cs index 4f6916e..0590f3e 100644 --- a/UnicodeInformation/UcdFields.cs +++ b/UnicodeInformation/UcdFields.cs @@ -14,7 +14,7 @@ namespace System.Unicode // Not really a field, just here to indicate that the entry is a range CodePointRange = 1, - Name = 2, + Name = 2, // Will stand in for official name as well as related names. Category = 4, CanonicalCombiningClass = 8, BidirectionalClass = 16, @@ -35,5 +35,7 @@ namespace System.Unicode ContributoryProperties = 8192, CoreProperties = 16384, + + RelatedCodePoints = 32768, } } diff --git a/UnicodeInformation/UnicodeCharacterData.cs b/UnicodeInformation/UnicodeCharacterData.cs index de7619b..73c66a2 100644 --- a/UnicodeInformation/UnicodeCharacterData.cs +++ b/UnicodeInformation/UnicodeCharacterData.cs @@ -11,6 +11,7 @@ namespace System.Unicode { public readonly UnicodeCharacterRange CodePointRange; public readonly string Name; + public readonly UnicodeNameAlias[] NameAliases; public readonly UnicodeCategory Category; public readonly CanonicalCombiningClass CanonicalCombiningClass; public readonly BidirectionalClass BidirectionalClass; @@ -32,6 +33,7 @@ namespace System.Unicode ( UnicodeCharacterRange codePointRange, string name, + UnicodeNameAlias[] nameAliases, UnicodeCategory category, CanonicalCombiningClass canonicalCombiningClass, BidirectionalClass bidirectionalClass, @@ -51,6 +53,7 @@ namespace System.Unicode { this.CodePointRange = codePointRange; this.Name = name; + this.NameAliases = nameAliases; this.Category = category; this.CanonicalCombiningClass = canonicalCombiningClass; this.BidirectionalClass = bidirectionalClass; diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs index 00850b1..e409f4a 100644 --- a/UnicodeInformation/UnicodeInfo.cs +++ b/UnicodeInformation/UnicodeInfo.cs @@ -31,7 +31,7 @@ namespace System.Unicode { using (var reader = new BinaryReader(stream, Encoding.UTF8)) { - if (reader.ReadByte() != 'U' + if (reader.ReadByte() != 'U' | reader.ReadByte() != 'C' | reader.ReadByte() != 'D') throw new InvalidDataException(); @@ -43,10 +43,11 @@ namespace System.Unicode var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte()); var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)]; + byte[] nameBuffer = new byte[64]; for (int i = 0; i < unicodeCharacterDataEntries.Length; ++i) { - unicodeCharacterDataEntries[i] = ReadUnicodeCharacterDataEntry(reader); + unicodeCharacterDataEntries[i] = ReadUnicodeCharacterDataEntry(reader, nameBuffer); } var blockEntries = new UnicodeBlock[reader.ReadByte()]; @@ -70,13 +71,50 @@ namespace System.Unicode } } - private static UnicodeCharacterData ReadUnicodeCharacterDataEntry(BinaryReader reader) + private static UnicodeCharacterData ReadUnicodeCharacterDataEntry(BinaryReader reader, byte[] nameBuffer) { var fields = (UcdFields)reader.ReadUInt16(); var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCharacterRange(ReadCodePoint(reader)); - string name = (fields & UcdFields.Name) != 0 ? reader.ReadString() : null; + string name = null; + UnicodeNameAlias[] nameAliases = UnicodeNameAlias.EmptyArray; + + // Read all the official names of the character. + if ((fields & UcdFields.Name) != 0) + { + int length = reader.ReadByte(); + byte @case = (byte)(length & 0xC0); + length = (length & 0x3F) + 1; + + if (@case < 0xC0) // These cases have an official name. + { + if (@case != 0) + { + } + if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException(); + + name = Encoding.UTF8.GetString(nameBuffer, 0, length); + if (@case == 2) length = (reader.ReadByte() & 0x3F) + 1; + else length = @case; + } + + if (length > 0) // These cases have official name aliases. + { + nameAliases = new UnicodeNameAlias[length]; + + for (int i = 0; i < nameAliases.Length; ++i) + { + length = reader.ReadByte(); + UnicodeNameAliasKind aliasKind = (UnicodeNameAliasKind)((length >> 6) + 1); + length = (length & 0x3F) + 1; + + if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException(); + nameAliases[i] = new UnicodeNameAlias(Encoding.UTF8.GetString(nameBuffer, 0, length), aliasKind); + } + } + } + var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned; var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered; var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0; @@ -97,6 +135,7 @@ namespace System.Unicode ( codePointRange, name, + nameAliases, category, canonicalCombiningClass, bidirectionalClass, diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index 344722d..7b422b8 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -77,6 +77,8 @@ + + diff --git a/UnicodeInformation/UnicodeNameAlias.cs b/UnicodeInformation/UnicodeNameAlias.cs new file mode 100644 index 0000000..7b3d3a3 --- /dev/null +++ b/UnicodeInformation/UnicodeNameAlias.cs @@ -0,0 +1,16 @@ +namespace System.Unicode +{ + public struct UnicodeNameAlias + { + internal static readonly UnicodeNameAlias[] EmptyArray = new UnicodeNameAlias[0]; + + public string Name { get; } + public UnicodeNameAliasKind Kind { get; } + + internal UnicodeNameAlias(string name, UnicodeNameAliasKind kind) + { + Name = name; + Kind = kind; + } + } +} diff --git a/UnicodeInformation/UnicodeNameAliasKind.cs b/UnicodeInformation/UnicodeNameAliasKind.cs new file mode 100644 index 0000000..6a0c9e9 --- /dev/null +++ b/UnicodeInformation/UnicodeNameAliasKind.cs @@ -0,0 +1,11 @@ +namespace System.Unicode +{ + public enum UnicodeNameAliasKind : byte + { + Correction = 1, + Control = 2, + Alternate = 3, + Figment = 4, + Abbreviation = 5 + } +}