swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First tentative of name alias encoding.

Browse Source
This commit is contained in:
GoldenCrystal
2014-11-21 15:41:27 +01:00
parent 83fb677dc2
commit 4119824ffc
8 changed files with 165 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeInformation.Builder/BinaryWriterExtensions.cs
+36
View File
@@ -50,5 +50,41 @@ namespace System.Unicode.Builder
writer.Write((byte)value);
}
}
/// <summary>Writes a character name alias.</summary>
/// <remarks>We assume that character names will not exceed 64 bytes in length.</remarks>
/// <param name="writer">The writer to use.</param>
/// <param name="nameAlias">The name alias value to write.</param>
public static void WriteNameAliasToFile(this BinaryWriter writer, UnicodeNameAlias nameAlias)
{
WriteNameToFile(writer, (byte)(nameAlias.Kind - 1), nameAlias.Name);
}
/// <summary>Writes a character name, packing two information bits along with the length.</summary>
/// <remarks>We assume that character names will not exceed 64 bytes in length.</remarks>
/// <param name="writer">The writer to use.</param>
/// <param name="extraBits">Extra bits to pack with the 6 bit length.</param>
/// <param name="name">The name to write.</param>
public static void WriteNameToFile(this BinaryWriter writer, byte extraBits, string name)
{
// This method will stuff two extra bits together with the byte count, provided that this one doesn't exceed 64.
var bytes = Encoding.UTF8.GetBytes(name);
if (bytes.Length > 64) throw new InvalidOperationException("Did not expect UTF-8 encoded names to be longer than 64 bytes.");
writer.WritePackedLength(extraBits, name.Length);
writer.Write(bytes);
}
/// <summary>Writes a 6 bits length packed with two extra bits.</summary>
/// <remarks>The parameters have a restricted range, which must be respected.</remarks>
/// <param name="writer">The writer used to perform the operation.</param>
/// <param name="extraBits">The value of the two extra bits.</param>
/// <param name="length">The length to write.</param>
public static void WritePackedLength(this BinaryWriter writer, byte extraBits, int length)
{
if (extraBits > 3) throw new ArgumentOutOfRangeException("extraBits");
if (length < 1 || length > 64) throw new ArgumentOutOfRangeException("length");
writer.Write((byte)((extraBits << 6) | (length - 1)));
}
}
}
UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs
+51 -19
View File
@@ -26,7 +26,8 @@ namespace System.Unicode.Builder
private ContributoryProperties contributoryProperties;
private CoreProperties coreProperties;
private List<int> relatedCodePoints = new List<int>();
private readonly List<UnicodeNameAlias> nameAliases = new List<UnicodeNameAlias>();
private readonly List<int> relatedCodePoints = new List<int>();
public UnicodeCharacterRange CodePointRange { get { return codePointRange; } }
@@ -36,6 +37,8 @@ namespace System.Unicode.Builder
set { name = value; }
}
public ICollection<UnicodeNameAlias> NameAliases { get { return nameAliases; } }
public UnicodeCategory Category
{
get { return category; }
@@ -138,33 +141,36 @@ namespace System.Unicode.Builder
{
return new UnicodeCharacterData
(
codePointRange,
CodePointRange,
Name,
category,
canonicalCombiningClass,
bidirectionalClass,
characterDecompositionMapping.DecompositionType,
characterDecompositionMapping.DecompositionMapping,
numericType,
numericValue,
bidirectionalMirrored,
oldName,
simpleUpperCaseMapping,
simpleLowerCaseMapping,
simpleTitleCaseMapping,
contributoryProperties,
coreProperties,
relatedCodePoints.Count > 0 ? relatedCodePoints.ToArray() : null
NameAliases.Count > 0 ? NameAliases.ToArray() : UnicodeNameAlias.EmptyArray,
Category,
CanonicalCombiningClass,
BidirectionalClass,
CharacterDecompositionMapping.DecompositionType,
CharacterDecompositionMapping.DecompositionMapping,
NumericType,
NumericValue,
BidirectionalMirrored,
OldName,
SimpleUpperCaseMapping,
SimpleLowerCaseMapping,
SimpleTitleCaseMapping,
ContributoryProperties,
CoreProperties,
RelatedCodePoints.Count > 0 ? RelatedCodePoints.ToArray() : null
);
}
internal void WriteToFile(BinaryWriter writer)
{
if (nameAliases.Count > 64) throw new InvalidDataException("Cannot handle more than 64 name aliases.");
UcdFields fields = default(UcdFields);
if (!codePointRange.IsSingleCodePoint) fields = UcdFields.CodePointRange;
if (name != null) fields |= UcdFields.Name;
if (name != null || nameAliases.Count > 0) fields |= UcdFields.Name; // This field combines name and alias.
if (category != UnicodeCategory.OtherNotAssigned) fields |= UcdFields.Category;
if (canonicalCombiningClass != CanonicalCombiningClass.NotReordered) fields |= UcdFields.CanonicalCombiningClass;
/*if (bidirectionalClass != 0)*/
@@ -184,7 +190,33 @@ namespace System.Unicode.Builder
writer.WriteCodePoint(codePointRange.FirstCodePoint);
if ((fields & UcdFields.CodePointRange) != 0) writer.WriteCodePoint(CodePointRange.LastCodePoint);
if ((fields & UcdFields.Name) != 0) writer.Write(name);
if ((fields & UcdFields.Name) != 0)
{
// We write the names by optimizing for the common case.
// i.e. Most characters have one name, and most of those characters have only one name.
// In the first byte, we will encode a 6bit length, along with two extra bits describing the structure:
// 00: The common case, this byte is the length of the name, and no other name follows.
// 01: This byte is the length of the name, and one alias follows.
// 10: This byte is the length of the name, and the number of aliases follows.
// 11: This byte is the number of aliases.
if (name == null)
{
writer.WritePackedLength(3, nameAliases.Count);
}
else
{
writer.WriteNameToFile((byte)(nameAliases.Count > 0 ? nameAliases.Count > 1 ? 2 : 1 : 0), name);
}
if (nameAliases.Count > 0)
{
if (name != null) writer.Write((byte)nameAliases.Count);
foreach (var nameAlias in nameAliases)
writer.WriteNameAliasToFile(nameAlias);
}
}
if ((fields & UcdFields.Category) != 0) writer.Write((byte)category);
if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass);
if ((fields & UcdFields.BidirectionalClass) != 0) writer.Write((byte)bidirectionalClass);
UnicodeInformation/UcdFields.cs
+3 -1
View File
@@ -14,7 +14,7 @@ namespace System.Unicode
// Not really a field, just here to indicate that the entry is a range
CodePointRange = 1,
Name = 2,
Name = 2, // Will stand in for official name as well as related names.
Category = 4,
CanonicalCombiningClass = 8,
BidirectionalClass = 16,
@@ -35,5 +35,7 @@ namespace System.Unicode
ContributoryProperties = 8192,
CoreProperties = 16384,
RelatedCodePoints = 32768,
}
}
UnicodeInformation/UnicodeCharacterData.cs
+3
View File
@@ -11,6 +11,7 @@ namespace System.Unicode
{
public readonly UnicodeCharacterRange CodePointRange;
public readonly string Name;
public readonly UnicodeNameAlias[] NameAliases;
public readonly UnicodeCategory Category;
public readonly CanonicalCombiningClass CanonicalCombiningClass;
public readonly BidirectionalClass BidirectionalClass;
@@ -32,6 +33,7 @@ namespace System.Unicode
(
UnicodeCharacterRange codePointRange,
string name,
UnicodeNameAlias[] nameAliases,
UnicodeCategory category,
CanonicalCombiningClass canonicalCombiningClass,
BidirectionalClass bidirectionalClass,
@@ -51,6 +53,7 @@ namespace System.Unicode
{
this.CodePointRange = codePointRange;
this.Name = name;
this.NameAliases = nameAliases;
this.Category = category;
this.CanonicalCombiningClass = canonicalCombiningClass;
this.BidirectionalClass = bidirectionalClass;
UnicodeInformation/UnicodeInfo.cs
+43 -4
View File
@@ -31,7 +31,7 @@ namespace System.Unicode
{
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.ReadByte() != 'U'
if (reader.ReadByte() != 'U'
| reader.ReadByte() != 'C'
| reader.ReadByte() != 'D')
throw new InvalidDataException();
@@ -43,10 +43,11 @@ namespace System.Unicode
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
byte[] nameBuffer = new byte[64];
for (int i = 0; i < unicodeCharacterDataEntries.Length; ++i)
{
unicodeCharacterDataEntries[i] = ReadUnicodeCharacterDataEntry(reader);
unicodeCharacterDataEntries[i] = ReadUnicodeCharacterDataEntry(reader, nameBuffer);
}
var blockEntries = new UnicodeBlock[reader.ReadByte()];
@@ -70,13 +71,50 @@ namespace System.Unicode
}
}
private static UnicodeCharacterData ReadUnicodeCharacterDataEntry(BinaryReader reader)
private static UnicodeCharacterData ReadUnicodeCharacterDataEntry(BinaryReader reader, byte[] nameBuffer)
{
var fields = (UcdFields)reader.ReadUInt16();
var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCharacterRange(ReadCodePoint(reader));
string name = (fields & UcdFields.Name) != 0 ? reader.ReadString() : null;
string name = null;
UnicodeNameAlias[] nameAliases = UnicodeNameAlias.EmptyArray;
// Read all the official names of the character.
if ((fields & UcdFields.Name) != 0)
{
int length = reader.ReadByte();
byte @case = (byte)(length & 0xC0);
length = (length & 0x3F) + 1;
if (@case < 0xC0) // These cases have an official name.
{
if (@case != 0)
{
}
if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException();
name = Encoding.UTF8.GetString(nameBuffer, 0, length);
if (@case == 2) length = (reader.ReadByte() & 0x3F) + 1;
else length = @case;
}
if (length > 0) // These cases have official name aliases.
{
nameAliases = new UnicodeNameAlias[length];
for (int i = 0; i < nameAliases.Length; ++i)
{
length = reader.ReadByte();
UnicodeNameAliasKind aliasKind = (UnicodeNameAliasKind)((length >> 6) + 1);
length = (length & 0x3F) + 1;
if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException();
nameAliases[i] = new UnicodeNameAlias(Encoding.UTF8.GetString(nameBuffer, 0, length), aliasKind);
}
}
}
var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned;
var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered;
var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0;
@@ -97,6 +135,7 @@ namespace System.Unicode
(
codePointRange,
name,
nameAliases,
category,
canonicalCombiningClass,
bidirectionalClass,
UnicodeInformation/UnicodeInformation.csproj
+2
View File
@@ -77,6 +77,8 @@
<Compile Include="UnicodeCharacterRange.cs" />
<Compile Include="UnicodeCharInfo.cs" />
<Compile Include="UnicodeInfo.cs" />
<Compile Include="UnicodeNameAlias.cs" />
<Compile Include="UnicodeNameAliasKind.cs" />
<Compile Include="UnihanNumericType.cs" />
<Compile Include="UnicodeNumericType.cs" />
<Compile Include="UnihanCharacterData.cs" />
UnicodeInformation/UnicodeNameAlias.cs
+16
View File
@@ -0,0 +1,16 @@
namespace System.Unicode
{
public struct UnicodeNameAlias
{
internal static readonly UnicodeNameAlias[] EmptyArray = new UnicodeNameAlias[0];
public string Name { get; }
public UnicodeNameAliasKind Kind { get; }
internal UnicodeNameAlias(string name, UnicodeNameAliasKind kind)
{
Name = name;
Kind = kind;
}
}
}
UnicodeInformation/UnicodeNameAliasKind.cs
+11
View File
@@ -0,0 +1,11 @@
namespace System.Unicode
{
public enum UnicodeNameAliasKind : byte
{
Correction = 1,
Control = 2,
Alternate = 3,
Figment = 4,
Abbreviation = 5
}
}