Try to determine the Unicode version dynamically from the data files, in the hope to ease upgrades to future Unicode versions.

This commit is contained in:
GoldenCrystal
2015-07-19 02:28:53 +02:00
parent 07b4086423
commit e3cada4ca7
5 changed files with 33 additions and 12 deletions
+1 -1
View File
@@ -56,7 +56,7 @@ This example shows a few usages of the library. It gets information on a specifi
### Details
In its current state, the project is written in C# 6, compilable by [Roslyn](http://roslyn.codeplex.com/), and targets the .NET 4.5 framework.
The core of the project, UnicodeInformation.dll, is a portable class library usable for either regular .NET or Windows 8 applications.
This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 7.0 at the time of writing) stored in a custom file format.
This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 8.0 at the time of writing) stored in a custom file format.
### Included Properties
#### From UCD
@@ -4,12 +4,14 @@ using System.Globalization;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace System.Unicode.Builder
{
internal class UnicodeDataProcessor
{
public const string ReadMeFileName = "ReadMe.txt";
public const string UnicodeDataFileName = "UnicodeData.txt";
public const string PropListFileName = "PropList.txt";
public const string DerivedCorePropertiesFileName = "DerivedCoreProperties.txt";
@@ -36,7 +38,7 @@ namespace System.Unicode.Builder
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource)
{
var builder = new UnicodeInfoBuilder(new Version(7, 0));
var builder = new UnicodeInfoBuilder(await ReadUnicodeVersionAsync(ucdSource).ConfigureAwait(false));
await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false);
await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
@@ -53,6 +55,24 @@ namespace System.Unicode.Builder
return builder;
}
private static async Task<Version> ReadUnicodeVersionAsync(IDataSource ucdSource)
{
using (var reader = new StreamReader(await ucdSource.OpenDataFileAsync(ReadMeFileName).ConfigureAwait(false), Encoding.UTF8, true))
{
string text = await reader.ReadToEndAsync().ConfigureAwait(false);
var match = Regex.Match(text, @"Version\s+(?<Major>[1-9][0-9]*)\.(?<Minor>[0-9]+)\.(?<Update>[0-9]+)\s+of\s+the\s+Unicode\s+Standard\.");
if (!match.Success)
{
match = Regex.Match(text, @"Unicode\s+(?<Major>[1-9][0-9]*)\.(?<Minor>[0-9]+)\.(?<Update>[0-9]+)\.");
}
if (!match.Success) throw new InvalidDataException("Could not determine the version of the Unicode standard defined in the files.");
return new Version(int.Parse(match.Groups["Major"].Value), int.Parse(match.Groups["Minor"].Value), int.Parse(match.Groups["Update"].Value));
}
}
private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false), ';'))
@@ -81,11 +101,11 @@ namespace System.Unicode.Builder
codePoint = new UnicodeCodePointRange(rangeStartCodePoint, codePoint.LastCodePoint);
name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
rangeStartCodePoint = -1;
}
else if (name == "<control>") // Ignore the name of the property for these code points, as it should really be empty by the spec.
else if (name == "<control>") // Ignore the name of the property for these code points, as it should really be empty by the spec.
{
// For control characters, we can derive a character label in of the form <control-NNNN>, which is not the character name.
name = null;
@@ -228,7 +248,7 @@ namespace System.Unicode.Builder
if (isSimplified)
{
builder.SetRadicalInfo(radicalIndex, UpdateRadicalData(builder.GetRadicalInfo(radicalIndex), radicalCodePoint, characterCodePoint));
}
}
else
{
builder.SetRadicalInfo(radicalIndex, new CjkRadicalData(radicalCodePoint, characterCodePoint));
@@ -502,7 +522,7 @@ namespace System.Unicode.Builder
var entry = builder.GetUnihan(reader.CodePoint);
var values = reader.PropertyValue.Split(' ');
foreach (var value in values)
foreach (var value in values)
{
bool isSimplified = false;
int index;
@@ -521,7 +541,7 @@ namespace System.Unicode.Builder
}
throw new InvalidDataException("Failed to decode value for kRSUnicode / Unicode_Radical_Stroke.");
SeparatorFound: ;
SeparatorFound:;
entry.UnicodeRadicalStrokeCounts.Add(new UnicodeRadicalStrokeCount(byte.Parse(value.Substring(0, index), NumberStyles.None), byte.Parse(value.Substring(index + (isSimplified ? 2 : 1)), NumberStyles.None), isSimplified));
}
break;
@@ -9,7 +9,7 @@ namespace System.Unicode.Builder
{
internal class UnicodeInfoBuilder
{
public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future…
public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future…
private readonly Version unicodeVersion;
private UnicodeCharacterDataBuilder[] ucdEntries = new UnicodeCharacterDataBuilder[10000];
@@ -287,8 +287,9 @@ namespace System.Unicode.Builder
using (var writer = new BinaryWriter(stream, Encoding.UTF8, true))
{
writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 2 });
writer.Write((ushort)8); // Hardcode Unicode 8.0
writer.Write((byte)0);
writer.Write(checked((ushort)unicodeVersion.Major));
writer.Write(checked((byte)unicodeVersion.Minor));
writer.Write(checked((byte)unicodeVersion.Build));
writer.WriteCodePoint(ucdEntryCount);
for (int i = 0; i < ucdEntryCount; ++i)
{
+2 -2
View File
@@ -46,7 +46,7 @@ namespace System.Unicode
if (formatVersion != 2) throw new InvalidDataException();
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte(), reader.ReadByte());
var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
byte[] nameBuffer = new byte[128];
@@ -81,7 +81,7 @@ namespace System.Unicode
for (i = 0; i < cjkRadicalEntries.Length; ++i)
{
cjkRadicalEntries[i] = ReadCjkRadicalInfo(reader);
}
}
var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)];
Binary file not shown.