diff --git a/README.md b/README.md index 2e73694..596ad32 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ This example shows a few usages of the library. It gets information on a specifi ### Details In its current state, the project is written in C# 6, compilable by [Roslyn](http://roslyn.codeplex.com/), and targets the .NET 4.5 framework. The core of the project, UnicodeInformation.dll, is a portable class library usable for either regular .NET or Windows 8 applications. -This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 7.0 at the time of writing) stored in a custom file format. +This library includes a subset of the official [Unicode Character Database](http://www.unicode.org/Public/UCD/latest/) (Version 8.0 at the time of writing) stored in a custom file format. ### Included Properties #### From UCD diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs index 27693de..cd2074e 100644 --- a/UnicodeInformation.Builder/UnicodeDataProcessor.cs +++ b/UnicodeInformation.Builder/UnicodeDataProcessor.cs @@ -4,12 +4,14 @@ using System.Globalization; using System.IO; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace System.Unicode.Builder { internal class UnicodeDataProcessor { + public const string ReadMeFileName = "ReadMe.txt"; public const string UnicodeDataFileName = "UnicodeData.txt"; public const string PropListFileName = "PropList.txt"; public const string DerivedCorePropertiesFileName = "DerivedCoreProperties.txt"; @@ -36,7 +38,7 @@ namespace System.Unicode.Builder public static async Task BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource) { - var builder = new UnicodeInfoBuilder(new Version(7, 0)); + var builder = new UnicodeInfoBuilder(await ReadUnicodeVersionAsync(ucdSource).ConfigureAwait(false)); await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false); await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false); @@ -53,6 +55,24 @@ namespace System.Unicode.Builder return builder; } + private static async Task ReadUnicodeVersionAsync(IDataSource ucdSource) + { + using (var reader = new StreamReader(await ucdSource.OpenDataFileAsync(ReadMeFileName).ConfigureAwait(false), Encoding.UTF8, true)) + { + string text = await reader.ReadToEndAsync().ConfigureAwait(false); + + var match = Regex.Match(text, @"Version\s+(?[1-9][0-9]*)\.(?[0-9]+)\.(?[0-9]+)\s+of\s+the\s+Unicode\s+Standard\."); + if (!match.Success) + { + match = Regex.Match(text, @"Unicode\s+(?[1-9][0-9]*)\.(?[0-9]+)\.(?[0-9]+)\."); + } + + if (!match.Success) throw new InvalidDataException("Could not determine the version of the Unicode standard defined in the files."); + + return new Version(int.Parse(match.Groups["Major"].Value), int.Parse(match.Groups["Minor"].Value), int.Parse(match.Groups["Update"].Value)); + } + } + private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false), ';')) @@ -81,11 +101,11 @@ namespace System.Unicode.Builder codePoint = new UnicodeCodePointRange(rangeStartCodePoint, codePoint.LastCodePoint); - name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII) + name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII) rangeStartCodePoint = -1; } - else if (name == "") // Ignore the name of the property for these code points, as it should really be empty by the spec. + else if (name == "") // Ignore the name of the property for these code points, as it should really be empty by the spec. { // For control characters, we can derive a character label in of the form , which is not the character name. name = null; @@ -228,7 +248,7 @@ namespace System.Unicode.Builder if (isSimplified) { builder.SetRadicalInfo(radicalIndex, UpdateRadicalData(builder.GetRadicalInfo(radicalIndex), radicalCodePoint, characterCodePoint)); - } + } else { builder.SetRadicalInfo(radicalIndex, new CjkRadicalData(radicalCodePoint, characterCodePoint)); @@ -502,7 +522,7 @@ namespace System.Unicode.Builder var entry = builder.GetUnihan(reader.CodePoint); var values = reader.PropertyValue.Split(' '); - foreach (var value in values) + foreach (var value in values) { bool isSimplified = false; int index; @@ -521,7 +541,7 @@ namespace System.Unicode.Builder } throw new InvalidDataException("Failed to decode value for kRSUnicode / Unicode_Radical_Stroke."); - SeparatorFound: ; + SeparatorFound:; entry.UnicodeRadicalStrokeCounts.Add(new UnicodeRadicalStrokeCount(byte.Parse(value.Substring(0, index), NumberStyles.None), byte.Parse(value.Substring(index + (isSimplified ? 2 : 1)), NumberStyles.None), isSimplified)); } break; diff --git a/UnicodeInformation.Builder/UnicodeInfoBuilder.cs b/UnicodeInformation.Builder/UnicodeInfoBuilder.cs index 6e5bb06..6efd819 100644 --- a/UnicodeInformation.Builder/UnicodeInfoBuilder.cs +++ b/UnicodeInformation.Builder/UnicodeInfoBuilder.cs @@ -9,7 +9,7 @@ namespace System.Unicode.Builder { internal class UnicodeInfoBuilder { - public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future… + public const int CjkRadicalCount = 214; // The number of radicals (214) shouldn't change in the near future… private readonly Version unicodeVersion; private UnicodeCharacterDataBuilder[] ucdEntries = new UnicodeCharacterDataBuilder[10000]; @@ -287,8 +287,9 @@ namespace System.Unicode.Builder using (var writer = new BinaryWriter(stream, Encoding.UTF8, true)) { writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 2 }); - writer.Write((ushort)8); // Hardcode Unicode 8.0 - writer.Write((byte)0); + writer.Write(checked((ushort)unicodeVersion.Major)); + writer.Write(checked((byte)unicodeVersion.Minor)); + writer.Write(checked((byte)unicodeVersion.Build)); writer.WriteCodePoint(ucdEntryCount); for (int i = 0; i < ucdEntryCount; ++i) { diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs index a6cf6c2..5d3af74 100644 --- a/UnicodeInformation/UnicodeInfo.cs +++ b/UnicodeInformation/UnicodeInfo.cs @@ -46,7 +46,7 @@ namespace System.Unicode if (formatVersion != 2) throw new InvalidDataException(); - var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte()); + var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte(), reader.ReadByte()); var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)]; byte[] nameBuffer = new byte[128]; @@ -81,7 +81,7 @@ namespace System.Unicode for (i = 0; i < cjkRadicalEntries.Length; ++i) { cjkRadicalEntries[i] = ReadCjkRadicalInfo(reader); - } + } var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)]; diff --git a/UnicodeInformation/ucd.dat b/UnicodeInformation/ucd.dat index 5dd9c55..794c72a 100644 Binary files a/UnicodeInformation/ucd.dat and b/UnicodeInformation/ucd.dat differ