swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a code point enumerator and new reader for unicode official data files.

Browse Source
This commit is contained in:
GoldenCrystal
2014-11-01 02:26:07 +01:00
parent e9cc4e16cb
commit f8742d9dad
7 changed files with 406 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeInformation/CodePointEnumerable.cs
+36
View File
@@ -0,0 +1,36 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UnicodeInformation
{
public struct CodePointEnumerable : IEnumerable<int>
{
private readonly string text;
public CodePointEnumerable(string text)
{
this.text = text;
}
public string Text { get { return text; } }
public CodePointEnumerator GetEnumerator()
{
return new CodePointEnumerator(text);
}
IEnumerator<int> IEnumerable<int>.GetEnumerator()
{
return GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
UnicodeInformation/CodePointEnumerator.cs
+51
View File
@@ -0,0 +1,51 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UnicodeInformation
{
public struct CodePointEnumerator : IEnumerator<int>
{
private readonly string text;
private int current;
private int index;
public CodePointEnumerator(string text)
{
if (text == null) throw new ArgumentNullException("text");
this.text = text;
this.current = 0;
this.index = -1;
}
public int Current { get { return current; } }
object IEnumerator.Current { get { return current; } }
void IDisposable.Dispose() { }
public bool MoveNext()
{
if (index < text.Length && (index += current > 0xFFFF ? 2 : 1) < text.Length)
{
current = char.ConvertToUtf32(text, index);
return true;
}
else
{
current = 0;
return false;
}
}
void IEnumerator.Reset()
{
current = 0;
index = -1;
}
}
}
UnicodeInformation/StringExtensions.cs
+16
View File
@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UnicodeInformation
{
public static class StringExtensions
{
public static CodePointEnumerable AsCodePointEnumerable(this string s)
{
return new CodePointEnumerable(s);
}
}
}
UnicodeInformation/UnicodeCharacterRange.cs
+26
View File
@@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UnicodeInformation
{
public struct UnicodeCharacterRange
{
public readonly int FirstCodePoint;
public readonly int LastCodePoint;
public UnicodeCharacterRange(int codePoint)
{
FirstCodePoint = codePoint;
LastCodePoint = codePoint;
}
public UnicodeCharacterRange(int firstCodePoint, int lastCodePoint)
{
FirstCodePoint = firstCodePoint;
LastCodePoint = lastCodePoint;
}
}
}
UnicodeInformation/UnicodeDataFileReader.cs
+246
View File
@@ -0,0 +1,246 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UnicodeInformation
{
public class UnicodeDataFileReader : IDisposable
{
private struct AsciiCharBuffer
{
private readonly UnicodeDataFileReader reader;
private int length;
public AsciiCharBuffer(UnicodeDataFileReader reader)
{
this.reader = reader;
this.length = 0;
}
public int Length { get { return length; } }
private void EnsureExtraCapacity(int count)
{
if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity");
if (reader.charBuffer.Length < checked(length + count))
Array.Resize(ref reader.charBuffer, Math.Max(length + count, reader.charBuffer.Length << 1));
}
public void Append(byte[] value, int startIndex, int count)
{
if (value == null) throw new ArgumentNullException("value");
if (startIndex >= value.Length) throw new ArgumentOutOfRangeException("startIndex");
if (checked(count += startIndex) > value.Length) throw new ArgumentOutOfRangeException("count");
EnsureExtraCapacity(value.Length);
var buffer = reader.charBuffer;
for (int i = startIndex; i < count; ++i)
{
byte b = value[i];
if (b > 127) throw new InvalidDataException(string.Format("Unexpected character value: {0}.", b));
buffer[length++] = (char)b;
}
}
public override string ToString()
{
return length > 0 ? new string(reader.charBuffer, 0, length) : string.Empty;
}
}
private readonly Stream stream;
private readonly byte[] byteBuffer;
private char[] charBuffer;
private int index;
private int length;
private bool hasField = false;
private readonly bool leaveOpen;
public UnicodeDataFileReader(Stream stream)
: this(stream, false)
{
}
public UnicodeDataFileReader(Stream stream, bool leaveOpen)
{
this.stream = stream;
this.byteBuffer = new byte[8192];
this.charBuffer = new char[100];
this.leaveOpen = leaveOpen;
}
public void Dispose()
{
if (!leaveOpen) stream.Dispose();
}
private bool RefillBuffer()
{
// Evilish line of code. 😈
return (length = stream.Read(byteBuffer, 0, byteBuffer.Length)) != (index = 0);
}
private static bool IsNewLineOrComment(byte b)
{
return b == '\n' || b == '#';
}
/// <summary>Moves the stream to the next valid data row.</summary>
/// <returns><see langword="true"/> if data is available; <see langword="false"/> otherwise.</returns>
public bool MoveToNextLine()
{
if (length == 0)
{
if (RefillBuffer())
{
if (!IsNewLineOrComment(byteBuffer[index]))
{
hasField = true;
goto Completed;
}
}
else
{
return false;
}
}
do
{
while (index < length)
{
if (byteBuffer[index++] == '\n')
{
if (index < length && !IsNewLineOrComment(byteBuffer[index]))
{
hasField = true;
goto Completed;
}
}
}
} while (RefillBuffer());
hasField = false;
Completed: ;
return hasField;
}
/// <summary>Reads the next data field.</summary>
/// <remarks>This method will return <see langword="null"/> on end of line.</remarks>
/// <returns>The text value of the read field, if available; <see langword="null"/> otherwise.</returns>
public string ReadField()
{
if (length == 0) throw new InvalidOperationException();
if (!hasField) return null;
else if (index >= length) RefillBuffer();
// If the current character is a new line or a comment, we are at the end of a line.
if (IsNewLineOrComment(byteBuffer[index]))
{
if (hasField)
{
hasField = false;
return string.Empty;
}
else
{
return null;
}
}
var buffer = new AsciiCharBuffer(this);
int startOffset;
int endOffset;
do
{
startOffset = index;
endOffset = -1;
while (index < length)
{
byte b = byteBuffer[index];
if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached.
{
endOffset = index;
hasField = false;
break;
}
else if(b == ';')
{
endOffset = index++;
break;
}
else
{
++index;
}
}
if (endOffset >= 0)
{
buffer.Append(byteBuffer, startOffset, endOffset - startOffset);
break;
}
else if (index > startOffset)
{
buffer.Append(byteBuffer, startOffset, index - startOffset);
}
} while (RefillBuffer());
return buffer.ToString();
}
/// <summary>Skips the next data field.</summary>
/// <remarks>This method will return <see langword="false"/> on end of line.</remarks>
/// <returns><see langword="true"/> if a field was skipped; <see langword="false"/> otherwise.</returns>
public bool SkipField()
{
if (length == 0) throw new InvalidOperationException();
if (!hasField) return false;
else if (index >= length) RefillBuffer();
// If the current character is a new line or a comment, we are at the end of a line.
if (IsNewLineOrComment(byteBuffer[index]))
{
hasField = false;
return false;
}
do
{
while (index < length)
{
byte b = byteBuffer[index];
if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached.
{
hasField = false;
return true;
}
else
{
++index;
if (b == ';')
{
return true;
}
}
}
} while (RefillBuffer());
return true;
}
}
}
UnicodeInformation/UnicodeDataManager.cs
+26 -26
View File
@@ -20,49 +20,49 @@ namespace UnicodeInformation
using (var httpClient = new HttpClient())
{
using (var reader = new StreamReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + UnicodeDataFileName).ConfigureAwait(false), Encoding.UTF8, false))
using (var reader = new UnicodeDataFileReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + UnicodeDataFileName).ConfigureAwait(false)))
{
string line;
while (!string.IsNullOrEmpty((line = await reader.ReadLineAsync().ConfigureAwait(false))))
while (reader.MoveToNextLine())
{
var fields = line.Split(';');
if (fields.Length < 15) throw new InvalidDataException();
// NB: Fields 10 and 11 are deemed obsolete. Field 11 should always be empty, and will be ignored here.
var characterData = new UnicodeCharacterDataBuilder(int.Parse(fields[0], NumberStyles.HexNumber))
var characterData = new UnicodeCharacterDataBuilder(int.Parse(reader.ReadField(), NumberStyles.HexNumber))
{
Name = fields[1],
Category = UnicodeCategoryInfo.FromShortName(fields[2]).Category,
CanonicalCombiningClass = (CanonicalCombiningClass)byte.Parse(fields[3]),
BidirectionalClass = fields[4],
DecompositionType = fields[5],
BidirectionalMirrored = fields[9] == "Y",
OldName = fields[10],
SimpleUpperCaseMapping = fields[12],
SimpleLowerCaseMapping = fields[13],
SimpleTitleCaseMapping = fields[14]
Name = reader.ReadField(),
Category = UnicodeCategoryInfo.FromShortName(reader.ReadField()).Category,
CanonicalCombiningClass = (CanonicalCombiningClass)byte.Parse(reader.ReadField()),
BidirectionalClass = reader.ReadField(),
DecompositionType = reader.ReadField()
};
string numericDecimalField = reader.ReadField();
string numericDigitField = reader.ReadField();
string numericNumericField = reader.ReadField();
characterData.BidirectionalMirrored = reader.ReadField() == "Y";
characterData.OldName = reader.ReadField();
reader.SkipField();
characterData.SimpleUpperCaseMapping = reader.ReadField();
characterData.SimpleLowerCaseMapping = reader.ReadField();
characterData.SimpleTitleCaseMapping = reader.ReadField();
// Handle Numeric_Type & Numeric_Value:
// If field 6 is set, fields 7 and 8 should have the same value, and Numeric_Type is Decimal.
// If field 6 is not set but field 7 is set, field 8 should be set and have the same value. Then, the type is Digit.
// If field 6 and 7 are not set, but field 8 is set, then Numeric_Type is Numeric.
if (!string.IsNullOrEmpty(fields[8]))
if (!string.IsNullOrEmpty(numericNumericField))
{
characterData.NumericValue = UnicodeRationalNumber.Parse(fields[8]);
characterData.NumericValue = UnicodeRationalNumber.Parse(numericNumericField);
if (!string.IsNullOrEmpty(fields[7]))
if (!string.IsNullOrEmpty(numericDigitField))
{
if (fields[7] != fields[8])
if (numericDigitField != numericNumericField)
{
throw new InvalidDataException("Invalid value for field 7 of character U+" + characterData.CodePoint.ToString("X4") + ".");
}
if (!string.IsNullOrEmpty(fields[6]))
if (!string.IsNullOrEmpty(numericDecimalField))
{
if (fields[6] != fields[7])
if (numericDecimalField != numericDigitField)
{
throw new InvalidDataException("Invalid value for field 6 of character U+" + characterData.CodePoint.ToString("X4") + ".");
}
@@ -82,7 +82,7 @@ namespace UnicodeInformation
}
}
//using (var reader = new StreamReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + PropListFileName)))
//using (var reader = new UnicodeDataFileReader(await httpClient.GetStreamAsync(UnicodeCharacterDataUri + PropListFileName).ConfigureAwait(false)))
//{
//}
}
UnicodeInformation/UnicodeInformation.csproj
+5
View File
@@ -48,12 +48,17 @@
<ItemGroup>
<Compile Include="BidirectionalClass.cs" />
<Compile Include="CanonicalCombiningClass.cs" />
<Compile Include="CodePointEnumerable.cs" />
<Compile Include="CodePointEnumerator.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="StringExtensions.cs" />
<Compile Include="UnicodeCategoryExtensions.cs" />
<Compile Include="UnicodeCategoryInfo.cs" />
<Compile Include="UnicodeCharacterData.cs" />
<Compile Include="UnicodeCharacterDataBuilder.cs" />
<Compile Include="UnicodeCharacterRange.cs" />
<Compile Include="UnicodeData.cs" />
<Compile Include="UnicodeDataFileReader.cs" />
<Compile Include="UnicodeDataManager.cs" />
<Compile Include="UnicodeNumericType.cs" />
<Compile Include="UnicodeRationalNumber.cs" />