From f8742d9dada11308a7664121b56f93eace2f87c5 Mon Sep 17 00:00:00 2001 From: GoldenCrystal Date: Sat, 1 Nov 2014 02:26:07 +0100 Subject: [PATCH] Added a code point enumerator and new reader for unicode official data files. --- UnicodeInformation/CodePointEnumerable.cs | 36 +++ UnicodeInformation/CodePointEnumerator.cs | 51 ++++ UnicodeInformation/StringExtensions.cs | 16 ++ UnicodeInformation/UnicodeCharacterRange.cs | 26 ++ UnicodeInformation/UnicodeDataFileReader.cs | 246 +++++++++++++++++++ UnicodeInformation/UnicodeDataManager.cs | 52 ++-- UnicodeInformation/UnicodeInformation.csproj | 5 + 7 files changed, 406 insertions(+), 26 deletions(-) create mode 100644 UnicodeInformation/CodePointEnumerable.cs create mode 100644 UnicodeInformation/CodePointEnumerator.cs create mode 100644 UnicodeInformation/StringExtensions.cs create mode 100644 UnicodeInformation/UnicodeCharacterRange.cs create mode 100644 UnicodeInformation/UnicodeDataFileReader.cs diff --git a/UnicodeInformation/CodePointEnumerable.cs b/UnicodeInformation/CodePointEnumerable.cs new file mode 100644 index 0000000..c0a84ba --- /dev/null +++ b/UnicodeInformation/CodePointEnumerable.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace UnicodeInformation +{ + public struct CodePointEnumerable : IEnumerable + { + private readonly string text; + + public CodePointEnumerable(string text) + { + this.text = text; + } + + public string Text { get { return text; } } + + public CodePointEnumerator GetEnumerator() + { + return new CodePointEnumerator(text); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/UnicodeInformation/CodePointEnumerator.cs b/UnicodeInformation/CodePointEnumerator.cs new file mode 100644 index 0000000..3f06604 --- /dev/null +++ b/UnicodeInformation/CodePointEnumerator.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace UnicodeInformation +{ + public struct CodePointEnumerator : IEnumerator + { + private readonly string text; + private int current; + private int index; + + public CodePointEnumerator(string text) + { + if (text == null) throw new ArgumentNullException("text"); + + this.text = text; + this.current = 0; + this.index = -1; + } + + public int Current { get { return current; } } + + object IEnumerator.Current { get { return current; } } + + void IDisposable.Dispose() { } + + public bool MoveNext() + { + if (index < text.Length && (index += current > 0xFFFF ? 2 : 1) < text.Length) + { + current = char.ConvertToUtf32(text, index); + return true; + } + else + { + current = 0; + return false; + } + } + + void IEnumerator.Reset() + { + current = 0; + index = -1; + } + } +} diff --git a/UnicodeInformation/StringExtensions.cs b/UnicodeInformation/StringExtensions.cs new file mode 100644 index 0000000..5421185 --- /dev/null +++ b/UnicodeInformation/StringExtensions.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace UnicodeInformation +{ + public static class StringExtensions + { + public static CodePointEnumerable AsCodePointEnumerable(this string s) + { + return new CodePointEnumerable(s); + } + } +} diff --git a/UnicodeInformation/UnicodeCharacterRange.cs b/UnicodeInformation/UnicodeCharacterRange.cs new file mode 100644 index 0000000..715e0a4 --- /dev/null +++ b/UnicodeInformation/UnicodeCharacterRange.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace UnicodeInformation +{ + public struct UnicodeCharacterRange + { + public readonly int FirstCodePoint; + public readonly int LastCodePoint; + + public UnicodeCharacterRange(int codePoint) + { + FirstCodePoint = codePoint; + LastCodePoint = codePoint; + } + + public UnicodeCharacterRange(int firstCodePoint, int lastCodePoint) + { + FirstCodePoint = firstCodePoint; + LastCodePoint = lastCodePoint; + } + } +} diff --git a/UnicodeInformation/UnicodeDataFileReader.cs b/UnicodeInformation/UnicodeDataFileReader.cs new file mode 100644 index 0000000..bd8cea3 --- /dev/null +++ b/UnicodeInformation/UnicodeDataFileReader.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace UnicodeInformation +{ + public class UnicodeDataFileReader : IDisposable + { + private struct AsciiCharBuffer + { + private readonly UnicodeDataFileReader reader; + private int length; + + public AsciiCharBuffer(UnicodeDataFileReader reader) + { + this.reader = reader; + this.length = 0; + } + + public int Length { get { return length; } } + + private void EnsureExtraCapacity(int count) + { + if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity"); + if (reader.charBuffer.Length < checked(length + count)) + Array.Resize(ref reader.charBuffer, Math.Max(length + count, reader.charBuffer.Length << 1)); + } + + public void Append(byte[] value, int startIndex, int count) + { + if (value == null) throw new ArgumentNullException("value"); + if (startIndex >= value.Length) throw new ArgumentOutOfRangeException("startIndex"); + if (checked(count += startIndex) > value.Length) throw new ArgumentOutOfRangeException("count"); + + EnsureExtraCapacity(value.Length); + + var buffer = reader.charBuffer; + + for (int i = startIndex; i < count; ++i) + { + byte b = value[i]; + + if (b > 127) throw new InvalidDataException(string.Format("Unexpected character value: {0}.", b)); + + buffer[length++] = (char)b; + } + } + + public override string ToString() + { + return length > 0 ? new string(reader.charBuffer, 0, length) : string.Empty; + } + } + + private readonly Stream stream; + private readonly byte[] byteBuffer; + private char[] charBuffer; + private int index; + private int length; + private bool hasField = false; + private readonly bool leaveOpen; + + public UnicodeDataFileReader(Stream stream) + : this(stream, false) + { + } + + public UnicodeDataFileReader(Stream stream, bool leaveOpen) + { + this.stream = stream; + this.byteBuffer = new byte[8192]; + this.charBuffer = new char[100]; + this.leaveOpen = leaveOpen; + } + + public void Dispose() + { + if (!leaveOpen) stream.Dispose(); + } + + private bool RefillBuffer() + { + // Evilish line of code. 😈 + return (length = stream.Read(byteBuffer, 0, byteBuffer.Length)) != (index = 0); + } + + private static bool IsNewLineOrComment(byte b) + { + return b == '\n' || b == '#'; + } + + /// Moves the stream to the next valid data row. + /// if data is available; otherwise. + public bool MoveToNextLine() + { + if (length == 0) + { + if (RefillBuffer()) + { + if (!IsNewLineOrComment(byteBuffer[index])) + { + hasField = true; + goto Completed; + } + } + else + { + return false; + } + } + + do + { + while (index < length) + { + if (byteBuffer[index++] == '\n') + { + if (index < length && !IsNewLineOrComment(byteBuffer[index])) + { + hasField = true; + goto Completed; + } + } + } + } while (RefillBuffer()); + + hasField = false; + Completed: ; + return hasField; + } + + /// Reads the next data field. + /// This method will return on end of line. + /// The text value of the read field, if available; otherwise. + public string ReadField() + { + if (length == 0) throw new InvalidOperationException(); + + if (!hasField) return null; + else if (index >= length) RefillBuffer(); + + // If the current character is a new line or a comment, we are at the end of a line. + if (IsNewLineOrComment(byteBuffer[index])) + { + if (hasField) + { + hasField = false; + return string.Empty; + } + else + { + return null; + } + } + + var buffer = new AsciiCharBuffer(this); + int startOffset; + int endOffset; + + do + { + startOffset = index; + endOffset = -1; + + while (index < length) + { + byte b = byteBuffer[index]; + + if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached. + { + endOffset = index; + hasField = false; + break; + } + else if(b == ';') + { + endOffset = index++; + break; + } + else + { + ++index; + } + } + + if (endOffset >= 0) + { + buffer.Append(byteBuffer, startOffset, endOffset - startOffset); + break; + } + else if (index > startOffset) + { + buffer.Append(byteBuffer, startOffset, index - startOffset); + } + } while (RefillBuffer()); + + return buffer.ToString(); + } + + /// Skips the next data field. + /// This method will return on end of line. + /// if a field was skipped; otherwise. + public bool SkipField() + { + if (length == 0) throw new InvalidOperationException(); + + if (!hasField) return false; + else if (index >= length) RefillBuffer(); + + // If the current character is a new line or a comment, we are at the end of a line. + if (IsNewLineOrComment(byteBuffer[index])) + { + hasField = false; + return false; + } + + do + { + while (index < length) + { + byte b = byteBuffer[index]; + + if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached. + { + hasField = false; + return true; + } + else + { + ++index; + + if (b == ';') + { + return true; + } + } + } + } while (RefillBuffer()); + + return true; + } + } +} diff --git a/UnicodeInformation/UnicodeDataManager.cs b/UnicodeInformation/UnicodeDataManager.cs index a443aaa..e09a21f 100644 --- a/UnicodeInformation/UnicodeDataManager.cs +++ b/UnicodeInformation/UnicodeDataManager.cs @@ -20,49 +20,49 @@ namespace UnicodeInformation using (var httpClient = new HttpClient()) { - using (var reader = new StreamReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + UnicodeDataFileName).ConfigureAwait(false), Encoding.UTF8, false)) + using (var reader = new UnicodeDataFileReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + UnicodeDataFileName).ConfigureAwait(false))) { - string line; - - while (!string.IsNullOrEmpty((line = await reader.ReadLineAsync().ConfigureAwait(false)))) + while (reader.MoveToNextLine()) { - var fields = line.Split(';'); - - if (fields.Length < 15) throw new InvalidDataException(); - // NB: Fields 10 and 11 are deemed obsolete. Field 11 should always be empty, and will be ignored here. - var characterData = new UnicodeCharacterDataBuilder(int.Parse(fields[0], NumberStyles.HexNumber)) + var characterData = new UnicodeCharacterDataBuilder(int.Parse(reader.ReadField(), NumberStyles.HexNumber)) { - Name = fields[1], - Category = UnicodeCategoryInfo.FromShortName(fields[2]).Category, - CanonicalCombiningClass = (CanonicalCombiningClass)byte.Parse(fields[3]), - BidirectionalClass = fields[4], - DecompositionType = fields[5], - BidirectionalMirrored = fields[9] == "Y", - OldName = fields[10], - SimpleUpperCaseMapping = fields[12], - SimpleLowerCaseMapping = fields[13], - SimpleTitleCaseMapping = fields[14] + Name = reader.ReadField(), + Category = UnicodeCategoryInfo.FromShortName(reader.ReadField()).Category, + CanonicalCombiningClass = (CanonicalCombiningClass)byte.Parse(reader.ReadField()), + BidirectionalClass = reader.ReadField(), + DecompositionType = reader.ReadField() }; + string numericDecimalField = reader.ReadField(); + string numericDigitField = reader.ReadField(); + string numericNumericField = reader.ReadField(); + + characterData.BidirectionalMirrored = reader.ReadField() == "Y"; + characterData.OldName = reader.ReadField(); + reader.SkipField(); + characterData.SimpleUpperCaseMapping = reader.ReadField(); + characterData.SimpleLowerCaseMapping = reader.ReadField(); + characterData.SimpleTitleCaseMapping = reader.ReadField(); + // Handle Numeric_Type & Numeric_Value: // If field 6 is set, fields 7 and 8 should have the same value, and Numeric_Type is Decimal. // If field 6 is not set but field 7 is set, field 8 should be set and have the same value. Then, the type is Digit. // If field 6 and 7 are not set, but field 8 is set, then Numeric_Type is Numeric. - if (!string.IsNullOrEmpty(fields[8])) + if (!string.IsNullOrEmpty(numericNumericField)) { - characterData.NumericValue = UnicodeRationalNumber.Parse(fields[8]); + characterData.NumericValue = UnicodeRationalNumber.Parse(numericNumericField); - if (!string.IsNullOrEmpty(fields[7])) + if (!string.IsNullOrEmpty(numericDigitField)) { - if (fields[7] != fields[8]) + if (numericDigitField != numericNumericField) { throw new InvalidDataException("Invalid value for field 7 of character U+" + characterData.CodePoint.ToString("X4") + "."); } - if (!string.IsNullOrEmpty(fields[6])) + if (!string.IsNullOrEmpty(numericDecimalField)) { - if (fields[6] != fields[7]) + if (numericDecimalField != numericDigitField) { throw new InvalidDataException("Invalid value for field 6 of character U+" + characterData.CodePoint.ToString("X4") + "."); } @@ -82,7 +82,7 @@ namespace UnicodeInformation } } - //using (var reader = new StreamReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + PropListFileName))) + //using (var reader = new UnicodeDataFileReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + PropListFileName).ConfigureAwait(false))) //{ //} } diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index 25075d6..a6025ea 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -48,12 +48,17 @@ + + + + +