diff --git a/UnicodeInformation.Builder/CharExtensions.cs b/UnicodeInformation.Builder/CharExtensions.cs new file mode 100644 index 0000000..82c8408 --- /dev/null +++ b/UnicodeInformation.Builder/CharExtensions.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode.Builder +{ + public static class CharExtensions + { + public static bool IsHexDigit(this char c) + { + return c >= '0' && c <= 'f' && (c <= '9' || c <= 'F' && c >= 'A' || c >= 'a'); + } + } +} diff --git a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs index a48f997..863f394 100644 --- a/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs +++ b/UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs @@ -184,6 +184,7 @@ namespace System.Unicode.Builder if (simpleTitleCaseMapping != null) fields |= UcdFields.SimpleTitleCaseMapping; if (contributoryProperties != 0) fields |= UcdFields.ContributoryProperties; if (coreProperties != 0) fields |= UcdFields.CoreProperties; + if (relatedCodePoints.Count > 0) fields |= UcdFields.RelatedCodePoints; writer.Write((ushort)fields); @@ -231,6 +232,12 @@ namespace System.Unicode.Builder if ((fields & UcdFields.SimpleTitleCaseMapping) != 0) writer.Write(simpleTitleCaseMapping); if ((fields & UcdFields.ContributoryProperties) != 0) writer.Write((int)contributoryProperties); if ((fields & UcdFields.CoreProperties) != 0) writer.WriteUInt24((int)coreProperties); + if ((fields & UcdFields.RelatedCodePoints) != 0) + { + writer.Write(checked((byte)(relatedCodePoints.Count - 1))); + foreach (int relatedCodePoint in relatedCodePoints) + writer.WriteCodePoint(relatedCodePoint); + } } } } diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs index 5571c79..dbca7d8 100644 --- a/UnicodeInformation.Builder/UnicodeDataProcessor.cs +++ b/UnicodeInformation.Builder/UnicodeDataProcessor.cs @@ -14,6 +14,7 @@ namespace System.Unicode.Builder public const string PropListFileName = "PropList.txt"; public const string DerivedCorePropertiesFileName = "DerivedCoreProperties.txt"; public const string NameAliasesFileName = "NameAliases.txt"; + public const string NamesListFileName = "NamesList.txt"; public const string BlocksFileName = "Blocks.txt"; public const string UnihanReadingsFileName = "Unihan_Readings.txt"; public const string UnihanVariantsFileName = "Unihan_Variants.txt"; @@ -39,7 +40,8 @@ namespace System.Unicode.Builder await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false); await ProcessDerivedCorePropertiesFile(ucdSource, builder).ConfigureAwait(false); await ProcessNameAliasesFile(ucdSource, builder).ConfigureAwait(false); - await ProcessBlocksFile(ucdSource, builder).ConfigureAwait(false); + await ProcessNamesListFile(ucdSource, builder).ConfigureAwait(false); + await ProcessBlocksFile(ucdSource, builder).ConfigureAwait(false); await ProcessUnihanReadings(unihanSource, builder).ConfigureAwait(false); await ProcessUnihanVariants(unihanSource, builder).ConfigureAwait(false); await ProcessUnihanNumericValues(unihanSource, builder).ConfigureAwait(false); @@ -218,6 +220,93 @@ namespace System.Unicode.Builder } } + private static async Task ProcessNamesListFile(IDataSource ucdSource, UnicodeInfoBuilder builder) + { + using (var reader = new StreamReader(await ucdSource.OpenDataFileAsync(NamesListFileName).ConfigureAwait(false), Encoding.UTF8, false)) + { + string line; + var characterData = null as UnicodeCharacterDataBuilder; + + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + characterData = null; + continue; + } + + if (characterData != null && line.Length > 3 && line[0] == '\t') + { + if (line[1] == 'x') + { + // We should get at least 7 characters for a valid line: "x" [0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z] + if (line.Length < 7) + { + characterData = null; + continue; + } + if (line[2] != ' ') throw new InvalidDataException(); + + int length; + + if (line[3].IsHexDigit()) + { + length = line.IndexOf(' ', 3); + if (length < 0) length = line.Length; + length -= 3; + + characterData.RelatedCodePoints.Add(int.Parse(line.Substring(3, length), NumberStyles.HexNumber)); + } + else if (line[3] == '(') + { + bool hasBrackets = line[4] == '<'; + int codePointOffset = line.IndexOf(hasBrackets ? "> - " : "- ", 4); + + if (codePointOffset < 0) throw new InvalidDataException(); + codePointOffset += hasBrackets ? 4 : 2; + + length = line.IndexOf(')', codePointOffset); + if (length < 0) throw new InvalidDataException(); + length -= codePointOffset; + + characterData.RelatedCodePoints.Add(int.Parse(line.Substring(codePointOffset, length), NumberStyles.HexNumber)); + } + else throw new InvalidDataException(); + } + continue; + } + + if (line[0].IsHexDigit()) + { + int codePoint = int.Parse(line.Substring(0, line.IndexOf('\t')), NumberStyles.HexNumber); + // This may return null, but for now, we will ignore code points that are not defined in UnicodeData.txt. + characterData = builder.GetUcd(codePoint); + // There should be no NamesList.txt entries for code points defined in a range. + if (characterData != null && !characterData.CodePointRange.IsSingleCodePoint) + { + // The only exception to this rule will be when we added the "Noncharacter_Code_Point" property to a few ranges, and we will ignore those. + if ((characterData.ContributoryProperties & ContributoryProperties.NonCharacterCodePoint) != 0) + characterData = null; + else + throw new InvalidDataException("Did not expect an NamesList.txt entry for U+" + codePoint.ToString("X4") + "."); + } + continue; + } + + switch (line[0]) + { + case '@': + case ';': + case '\t': + characterData = null; + break; + default: + throw new InvalidDataException("Unrecognized data in NamesList.txt."); + } + } + } + } + private static async Task ProcessBlocksFile(IDataSource ucdSource, UnicodeInfoBuilder builder) { using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false), ';')) diff --git a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj index 1408e46..43bd7a4 100644 --- a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj +++ b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj @@ -53,6 +53,7 @@ + True True diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs index 3b17510..8135890 100644 --- a/UnicodeInformation/UnicodeInfo.cs +++ b/UnicodeInformation/UnicodeInfo.cs @@ -143,6 +143,13 @@ namespace System.Unicode string simpleTitleCaseMapping = (fields & UcdFields.SimpleTitleCaseMapping) != 0 ? reader.ReadString() : null; ContributoryProperties contributoryProperties = (fields & UcdFields.ContributoryProperties) != 0 ? (ContributoryProperties)reader.ReadInt32() : 0; CoreProperties coreProperties = (fields & UcdFields.CoreProperties) != 0 ? (CoreProperties)ReadInt24(reader) : 0; + int[] relatedCodePoints = (fields & UcdFields.RelatedCodePoints) != 0 ? new int[reader.ReadByte() + 1] : null; + + if (relatedCodePoints != null) + { + for (int i = 0; i < relatedCodePoints.Length; ++i) + relatedCodePoints[i] = ReadCodePoint(reader); + } return new UnicodeCharacterData ( @@ -163,7 +170,7 @@ namespace System.Unicode simpleTitleCaseMapping, contributoryProperties, coreProperties, - null + relatedCodePoints ); } diff --git a/UnicodeInformation/ucd.dat b/UnicodeInformation/ucd.dat index 46f4f8d..429368b 100644 Binary files a/UnicodeInformation/ucd.dat and b/UnicodeInformation/ucd.dat differ