Included name aliases in the data file.

This commit is contained in:
GoldenCrystal
2014-11-21 16:02:19 +01:00
parent 4119824ffc
commit 343dc823cf
4 changed files with 34 additions and 35 deletions
@@ -57,20 +57,22 @@ namespace System.Unicode.Builder
/// <param name="nameAlias">The name alias value to write.</param>
public static void WriteNameAliasToFile(this BinaryWriter writer, UnicodeNameAlias nameAlias)
{
WriteNameToFile(writer, (byte)(nameAlias.Kind - 1), nameAlias.Name);
// This method will stuff two extra bits together with the byte count, provided that this one doesn't exceed 64.
var bytes = Encoding.UTF8.GetBytes(nameAlias.Name);
if (bytes.Length > 64) throw new InvalidOperationException("Did not expect UTF-8 encoded name aliases to be longer than 64 bytes.");
writer.WritePackedLength((byte)(nameAlias.Kind - 1), nameAlias.Name.Length);
writer.Write(bytes);
}
/// <summary>Writes a character name, packing two information bits along with the length.</summary>
/// <remarks>We assume that character names will not exceed 64 bytes in length.</remarks>
/// <remarks>We assume that character names will not exceed 128 bytes in length.</remarks>
/// <param name="writer">The writer to use.</param>
/// <param name="extraBits">Extra bits to pack with the 6 bit length.</param>
/// <param name="name">The name to write.</param>
public static void WriteNameToFile(this BinaryWriter writer, byte extraBits, string name)
public static void WriteNamePropertyToFile(this BinaryWriter writer, string name)
{
// This method will stuff two extra bits together with the byte count, provided that this one doesn't exceed 64.
var bytes = Encoding.UTF8.GetBytes(name);
if (bytes.Length > 64) throw new InvalidOperationException("Did not expect UTF-8 encoded names to be longer than 64 bytes.");
writer.WritePackedLength(extraBits, name.Length);
if (bytes.Length > 128) throw new InvalidOperationException("Did not expect UTF-8 encoded name to be longer than 128 bytes.");
writer.Write((byte)(name.Length - 1)); // The most significant bit will always be cleared, because it will be used for other cases.
writer.Write(bytes);
}
@@ -193,29 +193,24 @@ namespace System.Unicode.Builder
if ((fields & UcdFields.Name) != 0)
{
// We write the names by optimizing for the common case.
// i.e. Most characters have one name, and most of those characters have only one name.
// In the first byte, we will encode a 6bit length, along with two extra bits describing the structure:
// 00: The common case, this byte is the length of the name, and no other name follows.
// 01: This byte is the length of the name, and one alias follows.
// 10: This byte is the length of the name, and the number of aliases follows.
// 11: This byte is the number of aliases.
if (name == null)
{
writer.WritePackedLength(3, nameAliases.Count);
}
else
{
writer.WriteNameToFile((byte)(nameAliases.Count > 0 ? nameAliases.Count > 1 ? 2 : 1 : 0), name);
}
// i.e. Most characters have only one name.
// The first 8 bit sequence will encore either the length of the name property alone,
// or the number of aliases and a bit indicating the presence of the name property.
if (nameAliases.Count > 0)
{
if (name != null) writer.Write((byte)nameAliases.Count);
writer.WritePackedLength((byte)(name != null ? 3 : 2), nameAliases.Count);
if (name != null)
writer.WriteNamePropertyToFile(name);
foreach (var nameAlias in nameAliases)
writer.WriteNameAliasToFile(nameAlias);
}
else
{
writer.WriteNamePropertyToFile(name);
}
}
if ((fields & UcdFields.Category) != 0) writer.Write((byte)category);
if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass);
+14 -12
View File
@@ -43,7 +43,7 @@ namespace System.Unicode
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
byte[] nameBuffer = new byte[64];
byte[] nameBuffer = new byte[128];
for (int i = 0; i < unicodeCharacterDataEntries.Length; ++i)
{
@@ -85,23 +85,25 @@ namespace System.Unicode
{
int length = reader.ReadByte();
byte @case = (byte)(length & 0xC0);
length = (length & 0x3F) + 1;
if (@case < 0xC0) // These cases have an official name.
if (@case < 0x80) // Handles the case where only the name is present.
{
if (@case != 0)
{
}
length = (length & 0x7F) + 1;
if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException();
name = Encoding.UTF8.GetString(nameBuffer, 0, length);
if (@case == 2) length = (reader.ReadByte() & 0x3F) + 1;
else length = @case;
}
if (length > 0) // These cases have official name aliases.
else
{
nameAliases = new UnicodeNameAlias[length];
nameAliases = new UnicodeNameAlias[(length & 0x3F) + 1];
if ((@case & 0x40) != 0)
{
length = reader.ReadByte() + 1;
if (length > 128) throw new InvalidDataException("Did not expect names longer than 128 bytes.");
if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException();
name = Encoding.UTF8.GetString(nameBuffer, 0, length);
}
for (int i = 0; i < nameAliases.Length; ++i)
{
@@ -111,7 +113,7 @@ namespace System.Unicode
if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException();
nameAliases[i] = new UnicodeNameAlias(Encoding.UTF8.GetString(nameBuffer, 0, length), aliasKind);
}
}
}
}
Binary file not shown.