diff --git a/README.md b/README.md index 006fa6d..157d1dd 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ This library includes a subset of the official [Unicode Character Database](http * ID_Continue * XID_Start * XID_Continue +* Unicode_Radical_Stroke (This is actually kRSUnicode from the Unihan database) * Code point cross references extracted from NamesList.txt NB: The UCD property ISO_Comment will never be included since this one is empty in all new Unicode versions. @@ -131,6 +132,7 @@ NB: The UCD property ISO_Comment will never be included since this one is empty * kAccountingNumeric * kOtherNumeric * kPrimaryNumeric +* kRSUnicode * kDefinition * kMandarin * kCantonese diff --git a/UnicodeInformation.Builder/Program.cs b/UnicodeInformation.Builder/Program.cs index 6d1d8ab..68af15e 100644 --- a/UnicodeInformation.Builder/Program.cs +++ b/UnicodeInformation.Builder/Program.cs @@ -32,6 +32,7 @@ namespace System.Unicode.Builder "Unihan_NumericValues.txt", "Unihan_Readings.txt", "Unihan_Variants.txt", + "Unihan_IRGSources.txt", }; private static HttpMessageHandler httpMessageHandler; diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs index e35693d..bd1abc3 100644 --- a/UnicodeInformation.Builder/UnicodeDataProcessor.cs +++ b/UnicodeInformation.Builder/UnicodeDataProcessor.cs @@ -19,6 +19,7 @@ namespace System.Unicode.Builder public const string UnihanReadingsFileName = "Unihan_Readings.txt"; public const string UnihanVariantsFileName = "Unihan_Variants.txt"; public const string UnihanNumericValuesFileName = "Unihan_NumericValues.txt"; + public const string UnihanIrgSourcesFileName = "Unihan_IRGSources.txt"; private static string ParseSimpleCaseMapping(string mapping) { @@ -45,6 +46,7 @@ namespace System.Unicode.Builder await ProcessUnihanReadings(unihanSource, builder).ConfigureAwait(false); await ProcessUnihanVariants(unihanSource, builder).ConfigureAwait(false); await ProcessUnihanNumericValues(unihanSource, builder).ConfigureAwait(false); + await ProcessUnihanIrgSources(unihanSource, builder).ConfigureAwait(false); return builder; } @@ -438,5 +440,48 @@ namespace System.Unicode.Builder } } } + + private static async Task ProcessUnihanIrgSources(IDataSource unihanDataSource, UnicodeInfoBuilder builder) + { + using (var reader = new UnihanDataFileReader(await unihanDataSource.OpenDataFileAsync(UnihanIrgSourcesFileName).ConfigureAwait(false))) + { + while (reader.Read()) + { + switch (reader.PropertyName) + { + case UnihanProperty.kRSUnicode: + var entry = builder.GetUnihan(reader.CodePoint); + var values = reader.PropertyValue.Split(' '); + + foreach (var value in values) + { + bool isSimplified = false; + int index; + + for (int i = 0; i < value.Length; ++i) + { + switch (value[i]) + { + case '\'': + isSimplified = true; + goto case '.'; + case '.': + index = i; + goto SeparatorFound; + } + } + throw new InvalidDataException("Failed to decode value for kRSUnicode / Unicode_Radical_Stroke."); + + SeparatorFound: ; + entry.UnicodeRadicalStrokeCounts.Add(new UnicodeRadicalStrokeCount(byte.Parse(value.Substring(0, index), NumberStyles.None), byte.Parse(value.Substring(index + (isSimplified ? 2 : 1)), NumberStyles.None), isSimplified)); + } + break; + default: + // Ignore unhandled properties for now. + break; + } + } + } + } } } diff --git a/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs index 2203a9c..4e805f8 100644 --- a/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs +++ b/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs @@ -23,6 +23,8 @@ namespace System.Unicode.Builder private string simplifiedVariant; private string traditionalVariant; + private readonly List unicodeRadicalStrokeCounts = new List(); + public int CodePoint { get { return codePoint; } } public UnihanNumericType NumericType { get { return numericType; } set { numericType = value; } } public long NumericValue { get { return numericValue; } set { numericValue = value; } } @@ -36,6 +38,7 @@ namespace System.Unicode.Builder public string VietnameseReading { get { return vietnameseReading; } set { vietnameseReading = value; } } public string SimplifiedVariant { get { return simplifiedVariant; } set { simplifiedVariant = value; } } public string TraditionalVariant { get { return traditionalVariant; } set { traditionalVariant = value; } } + public ICollection UnicodeRadicalStrokeCounts { get { return unicodeRadicalStrokeCounts; } } internal UnihanCharacterDataBuilder(int codePoint) { @@ -49,6 +52,7 @@ namespace System.Unicode.Builder codePoint, numericType, numericValue, + unicodeRadicalStrokeCounts.ToArray(), definition, mandarinReading, cantoneseReading, @@ -67,6 +71,14 @@ namespace System.Unicode.Builder UnihanFields fields = default(UnihanFields); fields |= (UnihanFields)NumericType; + // For now, we have enough bits to encode the length of the array in the field specifier, so we'll do that. + // (NB: A quick analysis of the files revealed thare there are almost always exactly one Radical/Stroke count, and occasionally two, yet never more.) + if (unicodeRadicalStrokeCounts.Count > 0) + { + if (unicodeRadicalStrokeCounts.Count == 1) fields |= UnihanFields.UnicodeRadicalStrokeCount; + else if (unicodeRadicalStrokeCounts.Count == 2) fields |= UnihanFields.UnicodeRadicalStrokeCountTwice; + else fields |= UnihanFields.UnicodeRadicalStrokeCount | UnihanFields.UnicodeRadicalStrokeCountTwice; + } if (Definition != null) fields |= UnihanFields.Definition; if (MandarinReading != null) fields |= UnihanFields.MandarinReading; if (CantoneseReading != null) fields |= UnihanFields.CantoneseReading; @@ -81,6 +93,17 @@ namespace System.Unicode.Builder writer.Write((ushort)fields); writer.WriteCodePoint(UnihanCharacterData.PackCodePoint(codePoint)); + if ((fields & UnihanFields.UnicodeRadicalStrokeCountMore) != 0) + { + if ((fields & (UnihanFields.UnicodeRadicalStrokeCountMore)) == UnihanFields.UnicodeRadicalStrokeCountMore) + writer.Write(checked((byte)(unicodeRadicalStrokeCounts.Count - 3))); + + foreach (var radicalStrokeCount in unicodeRadicalStrokeCounts) + { + writer.Write(radicalStrokeCount.Radical); + writer.Write((byte)(radicalStrokeCount.StrokeCount | (radicalStrokeCount.IsSimplified ? 0x80 : 0))); + } + } if ((fields & UnihanFields.OtherNumeric) != 0) writer.Write(numericValue); if ((fields & UnihanFields.Definition) != 0) writer.Write(Definition); if ((fields & UnihanFields.MandarinReading) != 0) writer.Write(MandarinReading); diff --git a/UnicodeInformation/UnicodeCharInfo.cs b/UnicodeInformation/UnicodeCharInfo.cs index 169a6d2..7bd84e8 100644 --- a/UnicodeInformation/UnicodeCharInfo.cs +++ b/UnicodeInformation/UnicodeCharInfo.cs @@ -52,6 +52,8 @@ namespace System.Unicode public ContributoryProperties ContributoryProperties { get { return unicodeCharacterData?.ContributoryProperties ?? 0; } } public CoreProperties CoreProperties { get { return unicodeCharacterData?.CoreProperties ?? 0; } } public UnicodeCrossReferenceCollection CrossRerefences { get { return new UnicodeCrossReferenceCollection(unicodeCharacterData?.CrossRerefences); } } + [ValueName("kRSUnicode"), ValueName("cjkRSUnicode"), ValueName("Unicode_Radical_Stroke"), ValueName("URS")] + public UnicodeRadicalStrokeCountCollection UnicodeRadicalStrokeCounts { get { return new UnicodeRadicalStrokeCountCollection(unihanCharacterData?.UnicodeRadicalStrokeCounts); } } [ValueName("kDefinition")] public string Definition { get { return unihanCharacterData?.Definition; } } diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs index 9afcd5f..b72d6f4 100644 --- a/UnicodeInformation/UnicodeInfo.cs +++ b/UnicodeInformation/UnicodeInfo.cs @@ -185,6 +185,18 @@ namespace System.Unicode reader.ReadInt64() : 0; + UnicodeRadicalStrokeCount[] unicodeRadicalStrokeCounts = (fields & UnihanFields.UnicodeRadicalStrokeCountMore) != 0 ? + new UnicodeRadicalStrokeCount + [ + (fields & UnihanFields.UnicodeRadicalStrokeCountMore) == UnihanFields.UnicodeRadicalStrokeCountMore ? + reader.ReadByte() + 3 : + ((byte)(fields & UnihanFields.UnicodeRadicalStrokeCountMore) >> 2) + ] : + UnicodeRadicalStrokeCount.EmptyArray; + + for (int i = 0; i < unicodeRadicalStrokeCounts.Length; ++i) + unicodeRadicalStrokeCounts[i] = new UnicodeRadicalStrokeCount(reader.ReadByte(), reader.ReadByte()); + string definition = (fields & UnihanFields.Definition) != 0 ? reader.ReadString() : null; string mandarinReading = (fields & UnihanFields.MandarinReading) != 0 ? reader.ReadString() : null; string cantoneseReading = (fields & UnihanFields.CantoneseReading) != 0 ? reader.ReadString() : null; @@ -201,6 +213,7 @@ namespace System.Unicode codePoint, numericType, numericValue, + unicodeRadicalStrokeCounts, definition, mandarinReading, cantoneseReading, diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index 2cf390d..6d2ccf7 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -67,6 +67,7 @@ + @@ -81,6 +82,7 @@ + diff --git a/UnicodeInformation/UnicodeRadicalStrokeCount.cs b/UnicodeInformation/UnicodeRadicalStrokeCount.cs new file mode 100644 index 0000000..9dcc55b --- /dev/null +++ b/UnicodeInformation/UnicodeRadicalStrokeCount.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + public struct UnicodeRadicalStrokeCount + { + internal static readonly UnicodeRadicalStrokeCount[] EmptyArray = new UnicodeRadicalStrokeCount[0]; + + private readonly byte radical; + private readonly byte strokeCount; + + internal UnicodeRadicalStrokeCount(byte rawRadical, byte rawStrokeCount) + { + radical = rawRadical; + strokeCount = rawStrokeCount; + } + + internal UnicodeRadicalStrokeCount(byte radical, byte strokeCount, bool isSimplified) + { + this.radical = radical; + this.strokeCount = strokeCount; + + if (isSimplified) this.strokeCount |= 0x80; + } + + public byte Radical { get { return radical; } } + public byte StrokeCount { get { return strokeCount; } } + public bool IsSimplified { get { return (strokeCount & 0x80) != 0; } } + } +} diff --git a/UnicodeInformation/UnicodeRadicalStrokeCountCollection.cs b/UnicodeInformation/UnicodeRadicalStrokeCountCollection.cs new file mode 100644 index 0000000..94915f2 --- /dev/null +++ b/UnicodeInformation/UnicodeRadicalStrokeCountCollection.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + public struct UnicodeRadicalStrokeCountCollection : IList + { + public struct Enumerator : IEnumerator + { + private readonly UnicodeRadicalStrokeCount[] items; + private int index; + + internal Enumerator(UnicodeRadicalStrokeCount[] items) + { + this.items = items; + this.index = -1; + } + + public void Dispose() { } + + public UnicodeRadicalStrokeCount Current { get { return items[index]; } } + object IEnumerator.Current { get { return Current; } } + + public bool MoveNext() { return index < items.Length && ++index < items.Length; } + + void IEnumerator.Reset() { this.index = -1; } + } + + private readonly UnicodeRadicalStrokeCount[] items; + + public UnicodeRadicalStrokeCountCollection() { items = UnicodeRadicalStrokeCount.EmptyArray; } + internal UnicodeRadicalStrokeCountCollection(UnicodeRadicalStrokeCount[] items) { this.items = items ?? UnicodeRadicalStrokeCount.EmptyArray; } + + public UnicodeRadicalStrokeCount this[int index] { get { return items[index]; } } + + UnicodeRadicalStrokeCount IList.this[int index] + { + get { return items[index]; } + set { throw new NotSupportedException(); } + } + + public int Count { get { return items.Length; } } + + bool ICollection.IsReadOnly { get { return true; } } + + public void Add(UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); } + public void Insert(int index, UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); } + + public bool Remove(UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); } + public void RemoveAt(int index) { throw new NotSupportedException(); } + + public void Clear() { throw new NotSupportedException(); } + + public int IndexOf(UnicodeRadicalStrokeCount item) { return Array.IndexOf(items, item); } + public bool Contains(UnicodeRadicalStrokeCount item) { return IndexOf(item) >= 0; } + + public void CopyTo(UnicodeRadicalStrokeCount[] array, int arrayIndex) { items.CopyTo(array, arrayIndex); } + + public Enumerator GetEnumerator() { return new Enumerator(items); } + + IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } + } +} diff --git a/UnicodeInformation/UnihanCharacterData.cs b/UnicodeInformation/UnihanCharacterData.cs index 6fb0acd..5ae538e 100644 --- a/UnicodeInformation/UnihanCharacterData.cs +++ b/UnicodeInformation/UnihanCharacterData.cs @@ -11,6 +11,7 @@ namespace System.Unicode public readonly int CodePoint; public readonly UnihanNumericType NumericType; public readonly long NumericValue; + public readonly UnicodeRadicalStrokeCount[] UnicodeRadicalStrokeCounts; public readonly string Definition; public readonly string MandarinReading; public readonly string CantoneseReading; @@ -27,6 +28,7 @@ namespace System.Unicode int codePoint, UnihanNumericType numericType, long numericValue, + UnicodeRadicalStrokeCount[] unicodeRadicalStrokeCounts, string definition, string mandarinReading, string cantoneseReading, @@ -42,6 +44,7 @@ namespace System.Unicode CodePoint = codePoint; NumericType = numericType; NumericValue = numericValue; + UnicodeRadicalStrokeCounts = unicodeRadicalStrokeCounts; Definition = definition; MandarinReading = mandarinReading; CantoneseReading = cantoneseReading; diff --git a/UnicodeInformation/UnihanFields.cs b/UnicodeInformation/UnihanFields.cs index be362dc..29edc7e 100644 --- a/UnicodeInformation/UnihanFields.cs +++ b/UnicodeInformation/UnihanFields.cs @@ -14,16 +14,21 @@ namespace System.Unicode AccountingNumeric = 2, OtherNumeric = 3, - Definition = 4, - MandarinReading = 8, - CantoneseReading = 16, - JapaneseKunReading = 32, - JapaneseOnReading = 64, - KoreanReading = 128, - HangulReading = 256, - VietnameseReading = 512, + // UnicodeRadicalStroke : Not exactly a bit mask… + UnicodeRadicalStrokeCount = 4, // Will indicate exactly one value for Unicode_Radical_Stroke. + UnicodeRadicalStrokeCountTwice = 8, // Will indicate exactly two values for Unicode_Radical_Stroke. + UnicodeRadicalStrokeCountMore = 12, // Will indicate three or more values for Unicode_Radical_Stroke. This combination should never happen in the current files. - SimplifiedVariant = 1024, - TraditionalVariant = 2048, + Definition = 16, + MandarinReading = 32, + CantoneseReading = 64, + JapaneseKunReading = 128, + JapaneseOnReading = 256, + KoreanReading = 512, + HangulReading = 1024, + VietnameseReading = 2048, + + SimplifiedVariant = 4096, + TraditionalVariant = 8192, } } diff --git a/UnicodeInformation/ucd.dat b/UnicodeInformation/ucd.dat index 429368b..6117a71 100644 Binary files a/UnicodeInformation/ucd.dat and b/UnicodeInformation/ucd.dat differ