Try to determine the Unicode version dynamically from the data files, in the hope to ease upgrades to future Unicode versions.
This commit is contained in:
@@ -56,7 +56,7 @@ This example shows a few usages of the library. It gets information on a specifi
|
||||
### Details
|
||||
In its current state, the project is written in C# 6, compilable by [Roslyn](http://roslyn.codeplex.com/), and targets the .NET 4.5 framework.
|
||||
The core of the project, UnicodeInformation.dll, is a portable class library usable for either regular .NET or Windows 8 applications.
|
||||
This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 7.0 at the time of writing) stored in a custom file format.
|
||||
This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 8.0 at the time of writing) stored in a custom file format.
|
||||
|
||||
### Included Properties
|
||||
#### From UCD
|
||||
|
||||
@@ -4,12 +4,14 @@ using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
internal class UnicodeDataProcessor
|
||||
{
|
||||
public const string ReadMeFileName = "ReadMe.txt";
|
||||
public const string UnicodeDataFileName = "UnicodeData.txt";
|
||||
public const string PropListFileName = "PropList.txt";
|
||||
public const string DerivedCorePropertiesFileName = "DerivedCoreProperties.txt";
|
||||
@@ -36,7 +38,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource)
|
||||
{
|
||||
var builder = new UnicodeInfoBuilder(new Version(7, 0));
|
||||
var builder = new UnicodeInfoBuilder(await ReadUnicodeVersionAsync(ucdSource).ConfigureAwait(false));
|
||||
|
||||
await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false);
|
||||
await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
|
||||
@@ -53,6 +55,24 @@ namespace System.Unicode.Builder
|
||||
return builder;
|
||||
}
|
||||
|
||||
private static async Task<Version> ReadUnicodeVersionAsync(IDataSource ucdSource)
|
||||
{
|
||||
using (var reader = new StreamReader(await ucdSource.OpenDataFileAsync(ReadMeFileName).ConfigureAwait(false), Encoding.UTF8, true))
|
||||
{
|
||||
string text = await reader.ReadToEndAsync().ConfigureAwait(false);
|
||||
|
||||
var match = Regex.Match(text, @"Version\s+(?<Major>[1-9][0-9]*)\.(?<Minor>[0-9]+)\.(?<Update>[0-9]+)\s+of\s+the\s+Unicode\s+Standard\.");
|
||||
if (!match.Success)
|
||||
{
|
||||
match = Regex.Match(text, @"Unicode\s+(?<Major>[1-9][0-9]*)\.(?<Minor>[0-9]+)\.(?<Update>[0-9]+)\.");
|
||||
}
|
||||
|
||||
if (!match.Success) throw new InvalidDataException("Could not determine the version of the Unicode standard defined in the files.");
|
||||
|
||||
return new Version(int.Parse(match.Groups["Major"].Value), int.Parse(match.Groups["Minor"].Value), int.Parse(match.Groups["Update"].Value));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false), ';'))
|
||||
@@ -81,11 +101,11 @@ namespace System.Unicode.Builder
|
||||
|
||||
codePoint = new UnicodeCodePointRange(rangeStartCodePoint, codePoint.LastCodePoint);
|
||||
|
||||
name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
|
||||
name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
|
||||
|
||||
rangeStartCodePoint = -1;
|
||||
}
|
||||
else if (name == "<control>") // Ignore the name of the property for these code points, as it should really be empty by the spec.
|
||||
else if (name == "<control>") // Ignore the name of the property for these code points, as it should really be empty by the spec.
|
||||
{
|
||||
// For control characters, we can derive a character label in of the form <control-NNNN>, which is not the character name.
|
||||
name = null;
|
||||
@@ -228,7 +248,7 @@ namespace System.Unicode.Builder
|
||||
if (isSimplified)
|
||||
{
|
||||
builder.SetRadicalInfo(radicalIndex, UpdateRadicalData(builder.GetRadicalInfo(radicalIndex), radicalCodePoint, characterCodePoint));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.SetRadicalInfo(radicalIndex, new CjkRadicalData(radicalCodePoint, characterCodePoint));
|
||||
@@ -502,7 +522,7 @@ namespace System.Unicode.Builder
|
||||
var entry = builder.GetUnihan(reader.CodePoint);
|
||||
var values = reader.PropertyValue.Split(' ');
|
||||
|
||||
foreach (var value in values)
|
||||
foreach (var value in values)
|
||||
{
|
||||
bool isSimplified = false;
|
||||
int index;
|
||||
@@ -521,7 +541,7 @@ namespace System.Unicode.Builder
|
||||
}
|
||||
throw new InvalidDataException("Failed to decode value for kRSUnicode / Unicode_Radical_Stroke.");
|
||||
|
||||
SeparatorFound: ;
|
||||
SeparatorFound:;
|
||||
entry.UnicodeRadicalStrokeCounts.Add(new UnicodeRadicalStrokeCount(byte.Parse(value.Substring(0, index), NumberStyles.None), byte.Parse(value.Substring(index + (isSimplified ? 2 : 1)), NumberStyles.None), isSimplified));
|
||||
}
|
||||
break;
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace System.Unicode.Builder
|
||||
{
|
||||
internal class UnicodeInfoBuilder
|
||||
{
|
||||
public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future…
|
||||
public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future…
|
||||
|
||||
private readonly Version unicodeVersion;
|
||||
private UnicodeCharacterDataBuilder[] ucdEntries = new UnicodeCharacterDataBuilder[10000];
|
||||
@@ -287,8 +287,9 @@ namespace System.Unicode.Builder
|
||||
using (var writer = new BinaryWriter(stream, Encoding.UTF8, true))
|
||||
{
|
||||
writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 2 });
|
||||
writer.Write((ushort)8); // Hardcode Unicode 8.0
|
||||
writer.Write((byte)0);
|
||||
writer.Write(checked((ushort)unicodeVersion.Major));
|
||||
writer.Write(checked((byte)unicodeVersion.Minor));
|
||||
writer.Write(checked((byte)unicodeVersion.Build));
|
||||
writer.WriteCodePoint(ucdEntryCount);
|
||||
for (int i = 0; i < ucdEntryCount; ++i)
|
||||
{
|
||||
|
||||
@@ -46,7 +46,7 @@ namespace System.Unicode
|
||||
|
||||
if (formatVersion != 2) throw new InvalidDataException();
|
||||
|
||||
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
|
||||
var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte(), reader.ReadByte());
|
||||
|
||||
var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
|
||||
byte[] nameBuffer = new byte[128];
|
||||
@@ -81,7 +81,7 @@ namespace System.Unicode
|
||||
for (i = 0; i < cjkRadicalEntries.Length; ++i)
|
||||
{
|
||||
cjkRadicalEntries[i] = ReadCjkRadicalInfo(reader);
|
||||
}
|
||||
}
|
||||
|
||||
var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)];
|
||||
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user