using System.Globalization; using System.IO; using System.IO.Compression; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; namespace System.Unicode { /// Provides access to unicode information. public static class UnicodeInfo { // NB: These fields will be used as a default value for UnicodeCharacterData and UnihanCharacterData, and passed by reference. // They should be considered as being redonly, even if they aren't explicitely. private static /*readonly*/ UnicodeCharacterData DefaultUnicodeCharacterData = new UnicodeCharacterData(default(UnicodeCodePointRange), null, null, UnicodeCategory.OtherNotAssigned, 0, 0, 0, null, 0, default(UnicodeRationalNumber), false, null, null, null, null, 0, 0, null); private static /*readonly*/ UnihanCharacterData DefaultUnihanCharacterData = default(UnihanCharacterData); /// The block name returned when no block is assigned to a specific code point. public const string DefaultBlock = "No_Block"; private static readonly Version unicodeVersion; private static readonly UnicodeCharacterData[] unicodeCharacterData; private static readonly UnihanCharacterData[] unihanCharacterData; private static readonly UnicodeBlock[] blocks; private static readonly CjkRadicalData[] radicals; private static readonly int maxContiguousIndex; private static readonly string[] emojiSequences; static UnicodeInfo() { using (var stream = new DeflateStream(typeof(UnicodeInfo).GetTypeInfo().Assembly.GetManifestResourceStream("ucd.dat"), CompressionMode.Decompress, false)) { ReadFromStream(stream, out unicodeVersion, out unicodeCharacterData, out unihanCharacterData, out radicals, out blocks, out maxContiguousIndex); } } internal static void ReadFromStream(Stream stream, out Version unicodeVersion, out UnicodeCharacterData[] unicodeCharacterData, out UnihanCharacterData[] unihanCharacterData, out CjkRadicalData[] radicals, out UnicodeBlock[] blocks, out int maxContiguousIndex) { using (var reader = new BinaryReader(stream, Encoding.UTF8)) { int i; if (reader.ReadByte() != 'U' | reader.ReadByte() != 'C' | reader.ReadByte() != 'D') throw new InvalidDataException(); byte formatVersion = reader.ReadByte(); if (formatVersion != 2) throw new InvalidDataException(); var fileUnicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte(), reader.ReadByte()); var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)]; // Allocate one extra entry to act as a dummy entry. byte[] nameBuffer = new byte[128]; int mci = 0; for (i = 0; i < unicodeCharacterDataEntries.Length; ++i) { ReadUnicodeCharacterDataEntry(reader, nameBuffer, out unicodeCharacterDataEntries[i]); if (unicodeCharacterDataEntries[i].CodePointRange.Contains(i)) { mci = i; } else { ++i; break; } } maxContiguousIndex = mci; for (; i < unicodeCharacterDataEntries.Length; ++i) { ReadUnicodeCharacterDataEntry(reader, nameBuffer, out unicodeCharacterDataEntries[i]); } var blockEntries = new UnicodeBlock[reader.ReadUInt16()]; for (i = 0; i < blockEntries.Length; ++i) { ReadBlockEntry(reader, out blockEntries[i]); } var cjkRadicalEntries = new CjkRadicalData[reader.ReadByte()]; for (i = 0; i < cjkRadicalEntries.Length; ++i) { ReadCjkRadicalInfo(reader, out cjkRadicalEntries[i]); } var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)]; for (i = 0; i < unihanCharacterDataEntries.Length; ++i) { ReadUnihanCharacterDataEntry(reader, out unihanCharacterDataEntries[i]); } unicodeVersion = fileUnicodeVersion; unicodeCharacterData = unicodeCharacterDataEntries; unihanCharacterData = unihanCharacterDataEntries; radicals = cjkRadicalEntries; blocks = blockEntries; } } private static void ReadUnicodeCharacterDataEntry(BinaryReader reader, byte[] nameBuffer, out UnicodeCharacterData value) { var fields = (UcdFields)reader.ReadUInt16(); var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCodePointRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCodePointRange(ReadCodePoint(reader)); string name = null; UnicodeNameAlias[] nameAliases = UnicodeNameAlias.EmptyArray; // Read all the official names of the character. if ((fields & UcdFields.Name) != 0) { int length = reader.ReadByte(); byte @case = (byte)(length & 0xC0); if (@case < 0x80) // Handles the case where only the name is present. { length = (length & 0x7F) + 1; if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException(); name = Encoding.UTF8.GetString(nameBuffer, 0, length); } else { nameAliases = new UnicodeNameAlias[(length & 0x3F) + 1]; if ((@case & 0x40) != 0) { length = reader.ReadByte() + 1; if (length > 128) throw new InvalidDataException("Did not expect names longer than 128 bytes."); if (reader.Read(nameBuffer, 0, length) != length) throw new EndOfStreamException(); name = Encoding.UTF8.GetString(nameBuffer, 0, length); } for (int i = 0; i < nameAliases.Length; ++i) { nameAliases[i] = new UnicodeNameAlias(reader.ReadString(), (UnicodeNameAliasKind)(reader.ReadByte())); } } } var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned; var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered; var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0; CompatibilityFormattingTag decompositionType = (fields & UcdFields.DecompositionMapping) != 0 ? (CompatibilityFormattingTag)reader.ReadByte() : CompatibilityFormattingTag.Canonical; string decompositionMapping = (fields & UcdFields.DecompositionMapping) != 0 ? reader.ReadString() : null; var numericType = (UnicodeNumericType)((int)(fields & UcdFields.NumericNumeric) >> 6); UnicodeRationalNumber numericValue = numericType != UnicodeNumericType.None ? new UnicodeRationalNumber(reader.ReadInt64(), reader.ReadByte()) : default(UnicodeRationalNumber); string oldName = (fields & UcdFields.OldName) != 0 ? reader.ReadString() : null; string simpleUpperCaseMapping = (fields & UcdFields.SimpleUpperCaseMapping) != 0 ? reader.ReadString() : null; string simpleLowerCaseMapping = (fields & UcdFields.SimpleLowerCaseMapping) != 0 ? reader.ReadString() : null; string simpleTitleCaseMapping = (fields & UcdFields.SimpleTitleCaseMapping) != 0 ? reader.ReadString() : null; ContributoryProperties contributoryProperties = (fields & UcdFields.ContributoryProperties) != 0 ? (ContributoryProperties)reader.ReadInt32() : 0; int corePropertiesAndEmojiProperties = (fields & UcdFields.CorePropertiesAndEmojiProperties) != 0 ? ReadInt24(reader) : 0; int[] crossReferences = (fields & UcdFields.CrossRerefences) != 0 ? new int[reader.ReadByte() + 1] : null; if (crossReferences != null) { for (int i = 0; i < crossReferences.Length; ++i) crossReferences[i] = ReadCodePoint(reader); } value = new UnicodeCharacterData ( codePointRange, name, nameAliases, category, canonicalCombiningClass, bidirectionalClass, decompositionType, decompositionMapping, numericType, numericValue, (fields & UcdFields.BidirectionalMirrored) != 0, oldName, simpleUpperCaseMapping, simpleLowerCaseMapping, simpleTitleCaseMapping, contributoryProperties, corePropertiesAndEmojiProperties, crossReferences ); } private static void ReadUnihanCharacterDataEntry(BinaryReader reader, out UnihanCharacterData value) { var fields = (UnihanFields)reader.ReadUInt16(); int codePoint = UnihanCharacterData.UnpackCodePoint(ReadCodePoint(reader)); var numericType = (UnihanNumericType)((int)(fields & UnihanFields.OtherNumeric)); long numericValue = numericType != UnihanNumericType.None ? reader.ReadInt64() : 0; UnicodeRadicalStrokeCount[] unicodeRadicalStrokeCounts = (fields & UnihanFields.UnicodeRadicalStrokeCountMore) != 0 ? new UnicodeRadicalStrokeCount [ (fields & UnihanFields.UnicodeRadicalStrokeCountMore) == UnihanFields.UnicodeRadicalStrokeCountMore ? reader.ReadByte() + 3 : ((byte)(fields & UnihanFields.UnicodeRadicalStrokeCountMore) >> 2) ] : UnicodeRadicalStrokeCount.EmptyArray; for (int i = 0; i < unicodeRadicalStrokeCounts.Length; ++i) unicodeRadicalStrokeCounts[i] = new UnicodeRadicalStrokeCount(reader.ReadByte(), reader.ReadByte()); string definition = (fields & UnihanFields.Definition) != 0 ? reader.ReadString() : null; string mandarinReading = (fields & UnihanFields.MandarinReading) != 0 ? reader.ReadString() : null; string cantoneseReading = (fields & UnihanFields.CantoneseReading) != 0 ? reader.ReadString() : null; string japaneseKunReading = (fields & UnihanFields.JapaneseKunReading) != 0 ? reader.ReadString() : null; string japaneseOnReading = (fields & UnihanFields.JapaneseOnReading) != 0 ? reader.ReadString() : null; string koreanReading = (fields & UnihanFields.KoreanReading) != 0 ? reader.ReadString() : null; string hangulReading = (fields & UnihanFields.HangulReading) != 0 ? reader.ReadString() : null; string vietnameseReading = (fields & UnihanFields.VietnameseReading) != 0 ? reader.ReadString() : null; string simplifiedVariant = (fields & UnihanFields.SimplifiedVariant) != 0 ? reader.ReadString() : null; string traditionalVariant = (fields & UnihanFields.TraditionalVariant) != 0 ? reader.ReadString() : null; value = new UnihanCharacterData ( codePoint, numericType, numericValue, unicodeRadicalStrokeCounts, definition, mandarinReading, cantoneseReading, japaneseKunReading, japaneseOnReading, koreanReading, hangulReading, vietnameseReading, simplifiedVariant, traditionalVariant ); } private static void ReadCjkRadicalInfo(BinaryReader reader, out CjkRadicalData value) { char tr; char tc; tr = (char)reader.ReadUInt16(); tc = (char)reader.ReadUInt16(); value = (tr & 0x8000) == 0 ? new CjkRadicalData(tr, tc) : new CjkRadicalData((char)(tr & 0x7FFF), tc, (char)reader.ReadUInt16(), (char)reader.ReadUInt16()); } private static void ReadBlockEntry(BinaryReader reader, out UnicodeBlock value) => value = new UnicodeBlock(new UnicodeCodePointRange(ReadCodePoint(reader), ReadCodePoint(reader)), reader.ReadString()); private static int ReadInt24(BinaryReader reader) => reader.ReadByte() | ((reader.ReadByte() | (reader.ReadByte() << 8)) << 8); #if DEBUG internal static int ReadCodePoint(BinaryReader reader) #else private static int ReadCodePoint(BinaryReader reader) #endif { byte b = reader.ReadByte(); if (b < 0xA0) return b; else if (b < 0xC0) { return 0xA0 + (((b & 0x1F) << 8) | reader.ReadByte()); } else if (b < 0xE0) { return 0x20A0 + (((b & 0x1F) << 8) | reader.ReadByte()); } else { return 0x40A0 + (((((b & 0x1F) << 8) | reader.ReadByte()) << 8) | reader.ReadByte()); } } /// Gets the version of the Unicode standard supported by the class. public static Version UnicodeVersion { get { return unicodeVersion; } } private static int FindUnicodeCodePointIndex(int codePoint) => codePoint <= maxContiguousIndex ? codePoint : // For the first code points (this includes all of ASCII, and quite a bit more), the index in the table will be the code point itself. BinarySearchUnicodeCodePointIndex(codePoint); // For other code points, we will use a classic binary search with adjusted search indexes. private static int BinarySearchUnicodeCodePointIndex(int codePoint) { // NB: Due to the strictly ordered nature of the table, we know that a code point can never happen after the index which is the code point itself. // This will greatly reduce the range to scan for characters close to maxContiguousIndex, and will have a lesser impact on other characters. int minIndex = maxContiguousIndex + 1; int maxIndex = codePoint < unicodeCharacterData.Length ? codePoint - 1 : unicodeCharacterData.Length - 1; do { int index = (minIndex + maxIndex) >> 1; int Δ = unicodeCharacterData[index].CodePointRange.CompareCodePoint(codePoint); if (Δ == 0) return index; else if (Δ < 0) maxIndex = index - 1; else minIndex = index + 1; } while (minIndex <= maxIndex); return -1; } private static int FindUnihanCodePointIndex(int codePoint) { int minIndex; int maxIndex; if (unihanCharacterData.Length == 0 || codePoint < unihanCharacterData[minIndex = 0].CodePoint || codePoint > unihanCharacterData[maxIndex = unihanCharacterData.Length - 1].CodePoint) { return -1; } do { int index = (minIndex + maxIndex) >> 1; int Δ = codePoint - unihanCharacterData[index].CodePoint; if (Δ == 0) return index; else if (Δ < 0) maxIndex = index - 1; else minIndex = index + 1; } while (minIndex <= maxIndex); return -1; } private static int FindBlockIndex(int codePoint) { int minIndex = 0; int maxIndex = blocks.Length - 1; do { int index = (minIndex + maxIndex) >> 1; int Δ = blocks[index].CodePointRange.CompareCodePoint(codePoint); if (Δ == 0) return index; else if (Δ < 0) maxIndex = index - 1; else minIndex = index + 1; } while (minIndex <= maxIndex); return -1; } internal static ref UnicodeCharacterData GetUnicodeCharacterData(int index) { if (index >= 0) return ref unicodeCharacterData[index]; return ref DefaultUnicodeCharacterData; } internal static ref UnihanCharacterData GetUnihanCharacterData(int index) { if (index >= 0) return ref unihanCharacterData[index]; return ref DefaultUnihanCharacterData; } /// Gets the name of the Unicode block containing the character. /// If the character has not been assigned to a block, the value of will be returned. /// The Unicode code point whose block should be retrieved. /// The name of the block the code point was assigned to. public static string GetBlockName(int codePoint) => FindBlockIndex(codePoint) is int i && i >= 0 ? blocks[i].Name : DefaultBlock; /// Gets Unicode information on the specified code point. /// /// This method will consolidate the data from a few different sources. /// There are more efficient way of retrieving the data for some properties if only one of those is needed at a time: /// /// /// Property /// Method /// /// /// Name /// /// /// /// Category /// /// /// /// Block /// /// /// /// /// The Unicode code point for which the data must be retrieved. /// The name of the code point, if defined; otherwise. public static UnicodeCharInfo GetCharInfo(int codePoint) => new UnicodeCharInfo(codePoint, FindUnicodeCodePointIndex(codePoint), FindUnihanCodePointIndex(codePoint), GetBlockName(codePoint)); /// Gets the category of the specified code point. /// /// The name referred to is the unicode General_Category property. /// If you only need the category of a character, calling this method is faster than calling and retrieving , because there is less information to lookup. /// /// The Unicode code point for which the category must be retrieved. /// The category of the code point. public static UnicodeCategory GetCategory(int codePoint) => FindUnicodeCodePointIndex(codePoint) is int unicodeCharacterDataIndex && unicodeCharacterDataIndex >= 0 ? GetUnicodeCharacterData(unicodeCharacterDataIndex).Category : UnicodeCategory.OtherNotAssigned; /// Gets a display text for the specified code point. /// The information for the code point. /// A display text for the code point, which may be the representation of the code point itself. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static string GetDisplayText(this UnicodeCharInfo charInfo) => GetDisplayText(charInfo.CodePoint, charInfo.unicodeCharacterDataIndex); /// Gets a display text for the specified code point. /// The Unicode code point, for which a display text should be returned. /// A display text for the code point, which may be the representation of the code point itself. public static string GetDisplayText(int codePoint) { if (codePoint <= 0x0020) return ((char)(0x2400 + codePoint)).ToString(); // Provide a display text for control characters, including space. else if (GetCategory(codePoint) == UnicodeCategory.NonSpacingMark) return "\u25CC" + char.ConvertFromUtf32(codePoint); else if (codePoint >= 0xD800 && codePoint <= 0xDFFF) return "\xFFFD"; else if (codePoint >= 0xE0020 && codePoint < 0xE007F) return char.ConvertFromUtf32(codePoint - 0xE0000); // Handle "TAG" ASCII subset by remapping it to regular ASCII else return char.ConvertFromUtf32(codePoint); } private static string GetDisplayText(int codePoint, int unicodeCharacterDataIndex) { if (codePoint <= 0x0020) return ((char)(0x2400 + codePoint)).ToString(); // Provide a display text for control characters, including space. else if (GetUnicodeCharacterData(unicodeCharacterDataIndex).Category == UnicodeCategory.NonSpacingMark) return "\u25CC" + char.ConvertFromUtf32(codePoint); else if (codePoint >= 0xD800 && codePoint <= 0xDFFF) return "\xFFFD"; else if (codePoint >= 0xE0020 && codePoint < 0xE007F) return char.ConvertFromUtf32(codePoint - 0xE0000); // Handle "TAG" ASCII subset by remapping it to regular ASCII else return char.ConvertFromUtf32(codePoint); } /// Gets the name of the specified code point. /// /// The name referred to is the unicode Name property. /// If you only need the name of a character, calling this method is faster than calling and retrieving , because there is less information to lookup. /// /// The Unicode code point for which the name must be retrieved. /// The name of the code point, if defined; otherwise. public static string GetName(int codePoint) => HangulInfo.IsHangul(codePoint) ? HangulInfo.GetHangulName((char)codePoint) : GetNameInternal(codePoint); private static string GetNameInternal(int codePoint) => FindUnicodeCodePointIndex(codePoint) is int codePointInfoIndex && codePointInfoIndex >= 0 ? GetName(codePoint, ref GetUnicodeCharacterData(codePointInfoIndex)) : null; internal static string GetName(int codePoint, ref UnicodeCharacterData characterData) { if (characterData.CodePointRange.IsSingleCodePoint) return characterData.Name; else if (HangulInfo.IsHangul(codePoint)) return HangulInfo.GetHangulName((char)codePoint); else if (characterData.Name != null) return characterData.Name + "-" + codePoint.ToString("X4"); else return null; } /// Returns information for a CJK radical. /// The index of the radical. Must be between 1 and . /// Information on the specified radical. /// The parameter is out of range. public static CjkRadicalInfo GetCjkRadicalInfo(int radicalIndex) => new CjkRadicalInfo(checked((byte)radicalIndex), radicals[radicalIndex - 1]); /// Returns the number of CJK radicals in the Unicode data. /// This value will be 214 for the foreseeable future. public static int CjkRadicalCount => radicals.Length; /// Gets all the blocks defined in the Unicode data. /// is not the name of a block, but only a value indicating the abscence of block information for a given code point. /// An array containing an entry for every block defined in the Unicode data. public static UnicodeBlock[] GetBlocks() => (UnicodeBlock[])blocks.Clone(); } }