Added Unihan data.
This commit is contained in:
GoldenCrystal
@@ -6,5 +6,6 @@
|
||||
<Application.Resources>
|
||||
<BooleanToVisibilityConverter x:Key="BooleanToVisibilityConverter" />
|
||||
<local:NullToVisibilityConverter x:Key="NullToVisibilityConverter" />
|
||||
<local:ZeroToVisibilityConverter x:Key="ZeroToVisibilityConverter" />
|
||||
</Application.Resources>
|
||||
</Application>
|
||||
|
||||
@@ -49,19 +49,30 @@ namespace UnicodeCharacterInspector
|
||||
}
|
||||
|
||||
NotifyPropertyChanged();
|
||||
NotifyPropertyChanged("CodePoint");
|
||||
NotifyPropertyChanged("Name");
|
||||
NotifyPropertyChanged("OldName");
|
||||
NotifyPropertyChanged("Category");
|
||||
NotifyPropertyChanged("Block");
|
||||
NotifyPropertyChanged("CanonicalCombiningClass");
|
||||
NotifyPropertyChanged("BidirectionalClass");
|
||||
NotifyPropertyChanged("DecompositionType");
|
||||
NotifyPropertyChanged("DecompositionMapping");
|
||||
NotifyPropertyChanged("NumericType");
|
||||
NotifyPropertyChanged("NumericValue");
|
||||
NotifyPropertyChanged("ContributoryProperties");
|
||||
NotifyPropertyChanged("CoreProperties");
|
||||
NotifyPropertyChanged(nameof(CodePoint));
|
||||
NotifyPropertyChanged(nameof(Name));
|
||||
NotifyPropertyChanged(nameof(OldName));
|
||||
NotifyPropertyChanged(nameof(Definition));
|
||||
NotifyPropertyChanged(nameof(Category));
|
||||
NotifyPropertyChanged(nameof(Block));
|
||||
NotifyPropertyChanged(nameof(CanonicalCombiningClass));
|
||||
NotifyPropertyChanged(nameof(BidirectionalClass));
|
||||
NotifyPropertyChanged(nameof(DecompositionType));
|
||||
NotifyPropertyChanged(nameof(DecompositionMapping));
|
||||
NotifyPropertyChanged(nameof(NumericType));
|
||||
NotifyPropertyChanged(nameof(UnihanNumericType));
|
||||
NotifyPropertyChanged(nameof(NumericValue));
|
||||
NotifyPropertyChanged(nameof(ContributoryProperties));
|
||||
NotifyPropertyChanged(nameof(CoreProperties));
|
||||
NotifyPropertyChanged(nameof(MandarinReading));
|
||||
NotifyPropertyChanged(nameof(CantoneseReading));
|
||||
NotifyPropertyChanged(nameof(JapaneseKunReading));
|
||||
NotifyPropertyChanged(nameof(JapaneseOnReading));
|
||||
NotifyPropertyChanged(nameof(KoreanReading));
|
||||
NotifyPropertyChanged(nameof(HangulReading));
|
||||
NotifyPropertyChanged(nameof(VietnameseReading));
|
||||
NotifyPropertyChanged(nameof(SimplifiedVariant));
|
||||
NotifyPropertyChanged(nameof(TraditionalVariant));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,6 +92,11 @@ namespace UnicodeCharacterInspector
|
||||
get { return character != null ? characterInfo.OldName : null; }
|
||||
}
|
||||
|
||||
public string Definition
|
||||
{
|
||||
get { return character != null ? characterInfo.Definition : null; }
|
||||
}
|
||||
|
||||
public UnicodeCategory? Category
|
||||
{
|
||||
get { return character != null ? characterInfo.Category : null as UnicodeCategory?; }
|
||||
@@ -116,6 +132,11 @@ namespace UnicodeCharacterInspector
|
||||
get { return character != null ? characterInfo.NumericType : null as UnicodeNumericType?; }
|
||||
}
|
||||
|
||||
public UnihanNumericType? UnihanNumericType
|
||||
{
|
||||
get { return character != null ? characterInfo.UnihanNumericType : null as UnihanNumericType?; }
|
||||
}
|
||||
|
||||
public UnicodeRationalNumber? NumericValue
|
||||
{
|
||||
get { return character != null && characterInfo.NumericType != UnicodeNumericType.None ? characterInfo.NumericValue : null as UnicodeRationalNumber?; }
|
||||
@@ -130,5 +151,50 @@ namespace UnicodeCharacterInspector
|
||||
{
|
||||
get { return character != null ? characterInfo.CoreProperties : null as CoreProperties?; }
|
||||
}
|
||||
|
||||
public string MandarinReading
|
||||
{
|
||||
get { return character != null ? characterInfo.MandarinReading : null; }
|
||||
}
|
||||
|
||||
public string CantoneseReading
|
||||
{
|
||||
get { return character != null ? characterInfo.CantoneseReading : null; }
|
||||
}
|
||||
|
||||
public string JapaneseKunReading
|
||||
{
|
||||
get { return character != null ? characterInfo.JapaneseKunReading : null; }
|
||||
}
|
||||
|
||||
public string JapaneseOnReading
|
||||
{
|
||||
get { return character != null ? characterInfo.JapaneseOnReading : null; }
|
||||
}
|
||||
|
||||
public string KoreanReading
|
||||
{
|
||||
get { return character != null ? characterInfo.KoreanReading : null; }
|
||||
}
|
||||
|
||||
public string HangulReading
|
||||
{
|
||||
get { return character != null ? characterInfo.HangulReading : null; }
|
||||
}
|
||||
|
||||
public string VietnameseReading
|
||||
{
|
||||
get { return character != null ? characterInfo.VietnameseReading : null; }
|
||||
}
|
||||
|
||||
public string SimplifiedVariant
|
||||
{
|
||||
get { return character != null ? characterInfo.SimplifiedVariant : null; }
|
||||
}
|
||||
|
||||
public string TraditionalVariant
|
||||
{
|
||||
get { return character != null ? characterInfo.TraditionalVariant : null; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,10 @@
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="*" />
|
||||
</Grid.RowDefinitions>
|
||||
<TextBlock Grid.Row="0" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.CodePoint, Converter={StaticResource NullToVisibilityConverter}}" Text="Code Point" />
|
||||
@@ -83,26 +87,34 @@
|
||||
<TextBlock Grid.Row="1" Grid.Column="1" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.Name, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Name}" />
|
||||
<TextBlock Grid.Row="2" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.OldName, Converter={StaticResource NullToVisibilityConverter}}" Text="Old Name" />
|
||||
<TextBlock Grid.Row="2" Grid.Column="1" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.OldName, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.OldName}" />
|
||||
<TextBlock Grid.Row="3" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.Category, Converter={StaticResource NullToVisibilityConverter}}" Text="Category" />
|
||||
<TextBlock Grid.Row="3" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.Category, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Category}" />
|
||||
<TextBlock Grid.Row="4" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.Block, Converter={StaticResource NullToVisibilityConverter}}" Text="Block" />
|
||||
<TextBlock Grid.Row="4" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.Block, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Block}" />
|
||||
<TextBlock Grid.Row="5" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.CanonicalCombiningClass, Converter={StaticResource NullToVisibilityConverter}}" Text="Canonical Combining Class" />
|
||||
<TextBlock Grid.Row="5" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.CanonicalCombiningClass, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.CanonicalCombiningClass}" />
|
||||
<TextBlock Grid.Row="6" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.BidirectionalClass, Converter={StaticResource NullToVisibilityConverter}}" Text="Bidirectional Class" />
|
||||
<TextBlock Grid.Row="6" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.BidirectionalClass, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.BidirectionalClass}" />
|
||||
<TextBlock Grid.Row="7" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.DecompositionType, Converter={StaticResource NullToVisibilityConverter}}" Text="Decomposition Type" />
|
||||
<TextBlock Grid.Row="7" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.DecompositionType, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.DecompositionType}" />
|
||||
<TextBlock Grid.Row="8" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.DecompositionMapping, Converter={StaticResource NullToVisibilityConverter}}" Text="Decomposition Mapping" />
|
||||
<TextBlock Grid.Row="8" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.DecompositionMapping, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.DecompositionMapping}" />
|
||||
<TextBlock Grid.Row="9" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.NumericType, Converter={StaticResource NullToVisibilityConverter}}" Text="Numeric Type" />
|
||||
<TextBlock Grid.Row="9" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.NumericType, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.NumericType}" />
|
||||
<TextBlock Grid.Row="10" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.NumericValue, Converter={StaticResource NullToVisibilityConverter}}" Text="Numeric Value" />
|
||||
<TextBlock Grid.Row="10" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.NumericValue, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.NumericValue}" />
|
||||
<TextBlock Grid.Row="11" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.ContributoryProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="Contributory Properties" />
|
||||
<TextBlock Grid.Row="11" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.ContributoryProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.ContributoryProperties}" />
|
||||
<TextBlock Grid.Row="12" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="Core Properties" />
|
||||
<TextBlock Grid.Row="12" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.CoreProperties}" />
|
||||
<TextBlock Grid.Row="3" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.SimplifiedVariant, Converter={StaticResource NullToVisibilityConverter}}" Text="Simplified Variant" />
|
||||
<TextBlock Grid.Row="3" Grid.Column="1" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.SimplifiedVariant, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.SimplifiedVariant}" />
|
||||
<TextBlock Grid.Row="4" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.TraditionalVariant, Converter={StaticResource NullToVisibilityConverter}}" Text="Traditional Variant" />
|
||||
<TextBlock Grid.Row="4" Grid.Column="1" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.TraditionalVariant, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.TraditionalVariant}" />
|
||||
<TextBlock Grid.Row="5" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.Definition, Converter={StaticResource NullToVisibilityConverter}}" Text="Unihan Definition" />
|
||||
<TextBlock Grid.Row="5" Grid.Column="1" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.Definition, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Definition}" />
|
||||
<TextBlock Grid.Row="6" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.Category, Converter={StaticResource NullToVisibilityConverter}}" Text="Category" />
|
||||
<TextBlock Grid.Row="6" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.Category, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Category}" />
|
||||
<TextBlock Grid.Row="7" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.Block, Converter={StaticResource NullToVisibilityConverter}}" Text="Block" />
|
||||
<TextBlock Grid.Row="7" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.Block, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.Block}" />
|
||||
<TextBlock Grid.Row="8" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.CanonicalCombiningClass, Converter={StaticResource NullToVisibilityConverter}}" Text="Canonical Combining Class" />
|
||||
<TextBlock Grid.Row="8" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.CanonicalCombiningClass, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.CanonicalCombiningClass}" />
|
||||
<TextBlock Grid.Row="9" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.BidirectionalClass, Converter={StaticResource NullToVisibilityConverter}}" Text="Bidirectional Class" />
|
||||
<TextBlock Grid.Row="9" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.BidirectionalClass, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.BidirectionalClass}" />
|
||||
<TextBlock Grid.Row="10" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.DecompositionType, Converter={StaticResource NullToVisibilityConverter}}" Text="Decomposition Type" />
|
||||
<TextBlock Grid.Row="10" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.DecompositionType, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.DecompositionType}" />
|
||||
<TextBlock Grid.Row="11" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.DecompositionMapping, Converter={StaticResource NullToVisibilityConverter}}" Text="Decomposition Mapping" />
|
||||
<TextBlock Grid.Row="11" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.DecompositionMapping, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.DecompositionMapping}" />
|
||||
<TextBlock Grid.Row="12" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.NumericType, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Numeric Type" />
|
||||
<TextBlock Grid.Row="12" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.NumericType, Converter={StaticResource ZeroToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.NumericType}" />
|
||||
<TextBlock Grid.Row="13" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.UnihanNumericType, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Numeric Type (Unihan)" />
|
||||
<TextBlock Grid.Row="13" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.UnihanNumericType, Converter={StaticResource ZeroToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.UnihanNumericType}" />
|
||||
<TextBlock Grid.Row="14" Grid.Column="0" Visibility="{Binding SelectedCharacterInfo.NumericValue, Converter={StaticResource NullToVisibilityConverter}}" Text="Numeric Value" />
|
||||
<TextBlock Grid.Row="14" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.NumericValue, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.NumericValue}" />
|
||||
<TextBlock Grid.Row="15" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.ContributoryProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="Contributory Properties" />
|
||||
<TextBlock Grid.Row="15" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.ContributoryProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.ContributoryProperties}" />
|
||||
<TextBlock Grid.Row="16" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="Core Properties" />
|
||||
<TextBlock Grid.Row="16" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource NullToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.CoreProperties}" />
|
||||
</Grid>
|
||||
</ScrollViewer>
|
||||
</Grid>
|
||||
|
||||
@@ -69,6 +69,7 @@
|
||||
<Compile Include="BindableObject.cs" />
|
||||
<Compile Include="CharacterInfoViewModel.cs" />
|
||||
<Compile Include="CharacterInspectorViewModel.cs" />
|
||||
<Compile Include="ZeroToVisibilityConverter.cs" />
|
||||
<Compile Include="StringToUtf32Converter.cs" />
|
||||
<Compile Include="MainWindow.xaml.cs">
|
||||
<DependentUpon>MainWindow.xaml</DependentUpon>
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Windows;
|
||||
using System.Windows.Data;
|
||||
|
||||
namespace UnicodeCharacterInspector
|
||||
{
|
||||
internal sealed class ZeroToVisibilityConverter : IValueConverter
|
||||
{
|
||||
public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
return value != null && System.Convert.ToInt32(value) != 0 ? Visibility.Visible : Visibility.Collapsed;
|
||||
}
|
||||
|
||||
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@ namespace System.Unicode.Builder
|
||||
if (c == ' ') ++index;
|
||||
else
|
||||
{
|
||||
int codePoint = ParseCodePoint(s, ref index);
|
||||
int codePoint = HexCodePoint.Parse(s, ref index);
|
||||
|
||||
if (codePoint < 0x10000)
|
||||
buffer[charIndex++] = (char)codePoint;
|
||||
@@ -65,30 +65,5 @@ namespace System.Unicode.Builder
|
||||
|
||||
return new CharacterDecompositionMapping(tag, new string(buffer, 0, charIndex));
|
||||
}
|
||||
|
||||
private static int ParseCodePoint(string s, ref int index)
|
||||
{
|
||||
int i = index;
|
||||
int accum = 0;
|
||||
|
||||
while (i < s.Length)
|
||||
{
|
||||
char c = s[i];
|
||||
|
||||
if (c == ' ') break;
|
||||
|
||||
accum <<= 4;
|
||||
|
||||
if (c >= '0' && c <= '9') accum |= c - '0';
|
||||
else if (c >= 'A' && c <= 'F') accum |= c - 'A' + 0xA;
|
||||
else if (c >= 'a' && c <= 'f') accum |= c - 'a' + 0xA;
|
||||
else throw new FormatException();
|
||||
|
||||
++i;
|
||||
}
|
||||
|
||||
index = i;
|
||||
return accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
public static class HexCodePoint
|
||||
{
|
||||
public static int ParsePrefixed(string s)
|
||||
{
|
||||
if (!s.StartsWith("U+"))
|
||||
{
|
||||
throw new FormatException("Expected a code point in the form U+nnnn.");
|
||||
}
|
||||
return Parse(s, 2);
|
||||
}
|
||||
|
||||
public static int Parse(string s, int index)
|
||||
{
|
||||
return Parse(s, ref index);
|
||||
}
|
||||
|
||||
public static int Parse(string s, ref int index)
|
||||
{
|
||||
int i = index;
|
||||
int accum = 0;
|
||||
|
||||
while (i < s.Length)
|
||||
{
|
||||
char c = s[i];
|
||||
|
||||
if (c == ' ') break;
|
||||
|
||||
accum <<= 4;
|
||||
|
||||
if (c >= '0' && c <= '9') accum |= c - '0';
|
||||
else if (c >= 'A' && c <= 'F') accum |= c - 'A' + 0xA;
|
||||
else if (c >= 'a' && c <= 'f') accum |= c - 'a' + 0xA;
|
||||
else throw new FormatException();
|
||||
|
||||
++i;
|
||||
}
|
||||
|
||||
index = i;
|
||||
return accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,80 +9,24 @@ namespace System.Unicode.Builder
|
||||
{
|
||||
public class UnicodeDataFileReader : IDisposable
|
||||
{
|
||||
private struct Utf8Buffer
|
||||
{
|
||||
private readonly UnicodeDataFileReader reader;
|
||||
private int length;
|
||||
|
||||
public Utf8Buffer(UnicodeDataFileReader reader)
|
||||
{
|
||||
this.reader = reader;
|
||||
this.length = 0;
|
||||
}
|
||||
|
||||
public int Length { get { return length; } }
|
||||
|
||||
private void EnsureExtraCapacity(int count)
|
||||
{
|
||||
if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity");
|
||||
if (reader.utf8StringBuffer.Length < checked(length + count))
|
||||
Array.Resize(ref reader.utf8StringBuffer, Math.Max(length + count, reader.utf8StringBuffer.Length << 1));
|
||||
}
|
||||
|
||||
public void Append(byte[] value, int startIndex, int count)
|
||||
{
|
||||
if (value == null) throw new ArgumentNullException("value");
|
||||
if (startIndex >= value.Length) throw new ArgumentOutOfRangeException("startIndex");
|
||||
if (checked(count += startIndex) > value.Length) throw new ArgumentOutOfRangeException("count");
|
||||
|
||||
EnsureExtraCapacity(value.Length);
|
||||
|
||||
var buffer = reader.utf8StringBuffer;
|
||||
|
||||
for (int i = startIndex; i < count; ++i)
|
||||
{
|
||||
buffer[length++] = value[i];
|
||||
}
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return length > 0 ? Encoding.UTF8.GetString(reader.utf8StringBuffer, 0, length) : string.Empty;
|
||||
}
|
||||
|
||||
public string ToTrimmedString()
|
||||
{
|
||||
if (length == 0) return string.Empty;
|
||||
|
||||
var buffer = reader.utf8StringBuffer;
|
||||
int start = 0;
|
||||
int end = length;
|
||||
|
||||
while (buffer[start] == ' ') if (++start == length) return string.Empty;
|
||||
while (buffer[--end] == ' ') ;
|
||||
|
||||
return length > 0 ? Encoding.UTF8.GetString(reader.utf8StringBuffer, start, end - start + 1) : string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private readonly Stream stream;
|
||||
private readonly byte[] byteBuffer;
|
||||
private byte[] utf8StringBuffer;
|
||||
private int index;
|
||||
private int length;
|
||||
private readonly char fieldSeparator;
|
||||
private bool hasField = false;
|
||||
private readonly bool leaveOpen;
|
||||
|
||||
public UnicodeDataFileReader(Stream stream)
|
||||
: this(stream, false)
|
||||
public UnicodeDataFileReader(Stream stream, char fieldSeparator)
|
||||
: this(stream, fieldSeparator, false)
|
||||
{
|
||||
}
|
||||
|
||||
public UnicodeDataFileReader(Stream stream, bool leaveOpen)
|
||||
public UnicodeDataFileReader(Stream stream, char fieldSeparator, bool leaveOpen)
|
||||
{
|
||||
this.stream = stream;
|
||||
this.fieldSeparator = fieldSeparator;
|
||||
this.byteBuffer = new byte[8192];
|
||||
this.utf8StringBuffer = new byte[100];
|
||||
this.leaveOpen = leaveOpen;
|
||||
}
|
||||
|
||||
@@ -163,49 +107,51 @@ namespace System.Unicode.Builder
|
||||
}
|
||||
}
|
||||
|
||||
var buffer = new Utf8Buffer(this);
|
||||
int startOffset;
|
||||
int endOffset;
|
||||
|
||||
do
|
||||
using (var buffer = Utf8Buffer.Get())
|
||||
{
|
||||
startOffset = index;
|
||||
endOffset = -1;
|
||||
int startOffset;
|
||||
int endOffset;
|
||||
|
||||
while (index < length)
|
||||
do
|
||||
{
|
||||
byte b = byteBuffer[index];
|
||||
startOffset = index;
|
||||
endOffset = -1;
|
||||
|
||||
if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached.
|
||||
while (index < length)
|
||||
{
|
||||
endOffset = index;
|
||||
hasField = false;
|
||||
byte b = byteBuffer[index];
|
||||
|
||||
if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached.
|
||||
{
|
||||
endOffset = index;
|
||||
hasField = false;
|
||||
break;
|
||||
}
|
||||
else if (b == fieldSeparator)
|
||||
{
|
||||
endOffset = index++;
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
++index;
|
||||
}
|
||||
}
|
||||
|
||||
if (endOffset >= 0)
|
||||
{
|
||||
buffer.Append(byteBuffer, startOffset, endOffset - startOffset);
|
||||
break;
|
||||
}
|
||||
else if(b == ';')
|
||||
else if (index > startOffset)
|
||||
{
|
||||
endOffset = index++;
|
||||
break;
|
||||
buffer.Append(byteBuffer, startOffset, index - startOffset);
|
||||
}
|
||||
else
|
||||
{
|
||||
++index;
|
||||
}
|
||||
}
|
||||
} while (RefillBuffer());
|
||||
|
||||
if (endOffset >= 0)
|
||||
{
|
||||
buffer.Append(byteBuffer, startOffset, endOffset - startOffset);
|
||||
break;
|
||||
}
|
||||
else if (index > startOffset)
|
||||
{
|
||||
buffer.Append(byteBuffer, startOffset, index - startOffset);
|
||||
}
|
||||
} while (RefillBuffer());
|
||||
|
||||
return trim ? buffer.ToTrimmedString() : buffer.ToString();
|
||||
}
|
||||
return trim ? buffer.ToTrimmedString() : buffer.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reads the next data field.</summary>
|
||||
/// <remarks>This method will return <see langword="null"/> on end of line.</remarks>
|
||||
@@ -255,7 +201,7 @@ namespace System.Unicode.Builder
|
||||
{
|
||||
++index;
|
||||
|
||||
if (b == ';')
|
||||
if (b == fieldSeparator)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -14,6 +14,9 @@ namespace System.Unicode.Builder
|
||||
public const string PropListFileName = "PropList.txt";
|
||||
public const string DerivedCorePropertiesFileName = "DerivedCoreProperties.txt";
|
||||
public const string BlocksFileName = "Blocks.txt";
|
||||
public const string UnihanReadingsFileName = "Unihan_Readings.txt";
|
||||
public const string UnihanVariantsFileName = "Unihan_Variants.txt";
|
||||
public const string UnihanNumericValuesFileName = "Unihan_NumericValues.txt";
|
||||
|
||||
private static string ParseSimpleCaseMapping(string mapping)
|
||||
{
|
||||
@@ -35,13 +38,16 @@ namespace System.Unicode.Builder
|
||||
await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
|
||||
await ProcessDerivedCorePropertiesFile(ucdSource, builder).ConfigureAwait(false);
|
||||
await ProcessBlocksFile(ucdSource, builder).ConfigureAwait(false);
|
||||
await ProcessUnihanReadings(unihanSource, builder).ConfigureAwait(false);
|
||||
await ProcessUnihanVariants(unihanSource, builder).ConfigureAwait(false);
|
||||
await ProcessUnihanNumericValues(unihanSource, builder).ConfigureAwait(false);
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false)))
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false), ';'))
|
||||
{
|
||||
int rangeStartCodePoint = -1;
|
||||
|
||||
@@ -158,7 +164,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
private static async Task ProcessPropListFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(PropListFileName).ConfigureAwait(false)))
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(PropListFileName).ConfigureAwait(false), ';'))
|
||||
{
|
||||
while (reader.MoveToNextLine())
|
||||
{
|
||||
@@ -175,7 +181,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
private static async Task ProcessDerivedCorePropertiesFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(DerivedCorePropertiesFileName).ConfigureAwait(false)))
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(DerivedCorePropertiesFileName).ConfigureAwait(false), ';'))
|
||||
{
|
||||
while (reader.MoveToNextLine())
|
||||
{
|
||||
@@ -192,7 +198,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
private static async Task ProcessBlocksFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false)))
|
||||
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false), ';'))
|
||||
{
|
||||
while (reader.MoveToNextLine())
|
||||
{
|
||||
@@ -200,5 +206,126 @@ namespace System.Unicode.Builder
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessUnihanReadings(IDataSource unihanDataSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnihanDataFileReader(await unihanDataSource.OpenDataFileAsync(UnihanReadingsFileName).ConfigureAwait(false)))
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
// This statement is used to skip unhandled properties entirely.
|
||||
switch (reader.PropertyName)
|
||||
{
|
||||
case UnihanProperty.kDefinition:
|
||||
case UnihanProperty.kMandarin:
|
||||
case UnihanProperty.kCantonese:
|
||||
case UnihanProperty.kJapaneseKun:
|
||||
case UnihanProperty.kJapaneseOn:
|
||||
case UnihanProperty.kKorean:
|
||||
case UnihanProperty.kHangul:
|
||||
case UnihanProperty.kVietnamese:
|
||||
break;
|
||||
default:
|
||||
// Ignore unhandled properties for now.
|
||||
continue;
|
||||
}
|
||||
|
||||
// This entry will only be created if there is meaningful data.
|
||||
var entry = builder.GetUnihan(reader.CodePoint);
|
||||
|
||||
switch (reader.PropertyName)
|
||||
{
|
||||
case UnihanProperty.kDefinition:
|
||||
entry.Definition = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kMandarin:
|
||||
entry.MandarinReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kCantonese:
|
||||
entry.CantoneseReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kJapaneseKun:
|
||||
entry.JapaneseKunReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kJapaneseOn:
|
||||
entry.JapaneseOnReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kKorean:
|
||||
entry.KoreanReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kHangul:
|
||||
entry.HangulReading = reader.PropertyValue;
|
||||
break;
|
||||
case UnihanProperty.kVietnamese:
|
||||
entry.VietnameseReading = reader.PropertyValue;
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessUnihanVariants(IDataSource unihanDataSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnihanDataFileReader(await unihanDataSource.OpenDataFileAsync(UnihanVariantsFileName).ConfigureAwait(false)))
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
// This statement is used to skip unhandled properties entirely.
|
||||
switch (reader.PropertyName)
|
||||
{
|
||||
case UnihanProperty.kSimplifiedVariant:
|
||||
case UnihanProperty.kTraditionalVariant:
|
||||
break;
|
||||
default:
|
||||
// Ignore unhandled properties for now.
|
||||
continue;
|
||||
}
|
||||
|
||||
var entry = builder.GetUnihan(reader.CodePoint);
|
||||
|
||||
switch (reader.PropertyName)
|
||||
{
|
||||
case UnihanProperty.kSimplifiedVariant:
|
||||
entry.SimplifiedVariant = char.ConvertFromUtf32(HexCodePoint.ParsePrefixed(reader.PropertyValue));
|
||||
break;
|
||||
case UnihanProperty.kTraditionalVariant:
|
||||
entry.TraditionalVariant = char.ConvertFromUtf32(HexCodePoint.ParsePrefixed(reader.PropertyValue));
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ProcessUnihanNumericValues(IDataSource unihanDataSource, UnicodeInfoBuilder builder)
|
||||
{
|
||||
using (var reader = new UnihanDataFileReader(await unihanDataSource.OpenDataFileAsync(UnihanNumericValuesFileName).ConfigureAwait(false)))
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
var entry = builder.GetUnihan(reader.CodePoint);
|
||||
|
||||
switch (reader.PropertyName)
|
||||
{
|
||||
case UnihanProperty.kAccountingNumeric:
|
||||
entry.NumericType = UnihanNumericType.Accounting;
|
||||
break;
|
||||
case UnihanProperty.kOtherNumeric:
|
||||
entry.NumericType = UnihanNumericType.Other;
|
||||
break;
|
||||
case UnihanProperty.kPrimaryNumeric:
|
||||
entry.NumericType = UnihanNumericType.Primary;
|
||||
break;
|
||||
default:
|
||||
throw new InvalidDataException("Unrecognized property name: " + reader.PropertyName + ".");
|
||||
}
|
||||
|
||||
entry.NumericValue = long.Parse(reader.PropertyValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,10 @@ namespace System.Unicode.Builder
|
||||
internal class UnicodeInfoBuilder
|
||||
{
|
||||
private readonly Version unicodeVersion;
|
||||
private UnicodeCharacterDataBuilder[] entries = new UnicodeCharacterDataBuilder[10000];
|
||||
private int entryCount;
|
||||
private UnicodeCharacterDataBuilder[] ucdEntries = new UnicodeCharacterDataBuilder[10000];
|
||||
private int ucdEntryCount;
|
||||
private UnihanCharacterDataBuilder[] unihanEntries = new UnihanCharacterDataBuilder[10000];
|
||||
private int unihanEntryCount;
|
||||
private List<UnicodeBlock> blockEntries = new List<UnicodeBlock>(100);
|
||||
|
||||
public UnicodeInfoBuilder(Version unicodeVersion)
|
||||
@@ -19,18 +21,18 @@ namespace System.Unicode.Builder
|
||||
this.unicodeVersion = unicodeVersion;
|
||||
}
|
||||
|
||||
private int FindCodePoint(int codePoint)
|
||||
private int FindUcdCodePoint(int codePoint)
|
||||
{
|
||||
if (entryCount == 0) return -1;
|
||||
if (ucdEntryCount == 0) return -1;
|
||||
|
||||
int minIndex = 0;
|
||||
int maxIndex = entryCount - 1;
|
||||
int maxIndex = ucdEntryCount - 1;
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = entries[index].CodePointRange.CompareCodePoint(codePoint);
|
||||
int Δ = ucdEntries[index].CodePointRange.CompareCodePoint(codePoint);
|
||||
|
||||
if (Δ == 0) return index;
|
||||
else if (Δ < 0) maxIndex = index - 1;
|
||||
@@ -40,61 +42,144 @@ namespace System.Unicode.Builder
|
||||
return -1;
|
||||
}
|
||||
|
||||
private int FindInsertionPoint(int startCodePoint, int endCodePoint)
|
||||
private int FindUcdInsertionPoint(int startCodePoint, int endCodePoint)
|
||||
{
|
||||
int minIndex;
|
||||
int maxIndex;
|
||||
|
||||
if (entryCount == 0 || entries[maxIndex = entryCount - 1].CodePointRange.LastCodePoint < startCodePoint) return entryCount;
|
||||
else if (endCodePoint < entries[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
|
||||
else if (entryCount == 1) return -1;
|
||||
if (ucdEntryCount == 0 || ucdEntries[maxIndex = ucdEntryCount - 1].CodePointRange.LastCodePoint < startCodePoint) return ucdEntryCount;
|
||||
else if (endCodePoint < ucdEntries[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
|
||||
else if (ucdEntryCount == 1) return -1;
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = entries[index].CodePointRange.CompareCodePoint(startCodePoint);
|
||||
int Δ = ucdEntries[index].CodePointRange.CompareCodePoint(startCodePoint);
|
||||
|
||||
if (Δ == 0) return -1;
|
||||
else if (Δ < 0) maxIndex = index;
|
||||
else minIndex = index;
|
||||
} while (maxIndex - minIndex > 1);
|
||||
|
||||
if (entries[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
|
||||
if (ucdEntries[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
|
||||
else return maxIndex;
|
||||
}
|
||||
|
||||
private int FindUnihanCodePoint(int codePoint)
|
||||
{
|
||||
if (unihanEntryCount == 0) return -1;
|
||||
|
||||
int minIndex = 0;
|
||||
int maxIndex = unihanEntryCount - 1;
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = codePoint - unihanEntries[index].CodePoint;
|
||||
|
||||
if (Δ == 0) return index;
|
||||
else if (Δ < 0) maxIndex = index - 1;
|
||||
else minIndex = index + 1;
|
||||
} while (minIndex <= maxIndex);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
private int FindUnihanInsertionPoint(int codePoint)
|
||||
{
|
||||
int minIndex;
|
||||
int maxIndex;
|
||||
|
||||
if (unihanEntryCount == 0 || unihanEntries[maxIndex = unihanEntryCount - 1].CodePoint < codePoint) return unihanEntryCount;
|
||||
else if (codePoint < unihanEntries[minIndex = 0].CodePoint) return 0;
|
||||
else if (unihanEntryCount == 1) return -1;
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = codePoint - unihanEntries[index].CodePoint;
|
||||
|
||||
if (Δ == 0) return -1;
|
||||
else if (Δ < 0) maxIndex = index;
|
||||
else minIndex = index;
|
||||
} while (maxIndex - minIndex > 1);
|
||||
|
||||
if (unihanEntries[maxIndex].CodePoint < codePoint) return -1;
|
||||
else return maxIndex;
|
||||
}
|
||||
|
||||
public void Insert(UnicodeCharacterDataBuilder data)
|
||||
{
|
||||
int insertionPoint = FindInsertionPoint(data.CodePointRange.FirstCodePoint, data.CodePointRange.LastCodePoint);
|
||||
int insertionPoint = FindUcdInsertionPoint(data.CodePointRange.FirstCodePoint, data.CodePointRange.LastCodePoint);
|
||||
|
||||
if (insertionPoint < 0) throw new InvalidOperationException("The specified range overlaps with pre-existing ranges.");
|
||||
|
||||
if (insertionPoint >= entries.Length)
|
||||
if (insertionPoint >= ucdEntries.Length)
|
||||
{
|
||||
Array.Resize(ref entries, entries.Length << 1);
|
||||
Array.Resize(ref ucdEntries, ucdEntries.Length << 1);
|
||||
}
|
||||
|
||||
if (insertionPoint < entryCount)
|
||||
if (insertionPoint < ucdEntryCount)
|
||||
{
|
||||
Array.Copy(entries, insertionPoint, entries, insertionPoint + 1, entryCount - insertionPoint);
|
||||
Array.Copy(ucdEntries, insertionPoint, ucdEntries, insertionPoint + 1, ucdEntryCount - insertionPoint);
|
||||
}
|
||||
|
||||
entries[insertionPoint] = data;
|
||||
++entryCount;
|
||||
ucdEntries[insertionPoint] = data;
|
||||
++ucdEntryCount;
|
||||
}
|
||||
|
||||
public UnicodeCharacterDataBuilder Get(int codePoint)
|
||||
private void Insert(UnihanCharacterDataBuilder data)
|
||||
{
|
||||
int index = FindCodePoint(codePoint);
|
||||
int insertionPoint = FindUnihanInsertionPoint(data.CodePoint);
|
||||
|
||||
return index >= 0 ? entries[index] : null;
|
||||
if (insertionPoint < 0) throw new InvalidOperationException("The specified range overlaps with pre-existing ranges.");
|
||||
|
||||
if (insertionPoint >= unihanEntries.Length)
|
||||
{
|
||||
Array.Resize(ref unihanEntries, unihanEntries.Length << 1);
|
||||
}
|
||||
|
||||
if (insertionPoint < unihanEntryCount)
|
||||
{
|
||||
Array.Copy(unihanEntries, insertionPoint, unihanEntries, insertionPoint + 1, unihanEntryCount - insertionPoint);
|
||||
}
|
||||
|
||||
unihanEntries[insertionPoint] = data;
|
||||
++unihanEntryCount;
|
||||
}
|
||||
|
||||
public UnicodeCharacterDataBuilder GetUcd(int codePoint)
|
||||
{
|
||||
int index = FindUcdCodePoint(codePoint);
|
||||
|
||||
return index >= 0 ? ucdEntries[index] : null;
|
||||
}
|
||||
|
||||
public UnihanCharacterDataBuilder GetUnihan(int codePoint)
|
||||
{
|
||||
int index = FindUnihanCodePoint(codePoint);
|
||||
|
||||
if (index >= 0)
|
||||
{
|
||||
return unihanEntries[index];
|
||||
}
|
||||
else
|
||||
{
|
||||
var data = new UnihanCharacterDataBuilder(codePoint);
|
||||
|
||||
Insert(data);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
public void SetProperties(ContributoryProperties property, UnicodeCharacterRange codePointRange)
|
||||
{
|
||||
int firstIndex = FindCodePoint(codePointRange.FirstCodePoint);
|
||||
int lastIndex = FindCodePoint(codePointRange.LastCodePoint);
|
||||
int firstIndex = FindUcdCodePoint(codePointRange.FirstCodePoint);
|
||||
int lastIndex = FindUcdCodePoint(codePointRange.LastCodePoint);
|
||||
|
||||
if (firstIndex < 0 && lastIndex < 0)
|
||||
{
|
||||
@@ -104,8 +189,8 @@ namespace System.Unicode.Builder
|
||||
|
||||
if (firstIndex < 0
|
||||
|| lastIndex < 0
|
||||
|| entries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|
||||
|| entries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
|
||||
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|
||||
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
|
||||
{
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
@@ -114,7 +199,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
while (true)
|
||||
{
|
||||
entries[i].ContributoryProperties |= property;
|
||||
ucdEntries[i].ContributoryProperties |= property;
|
||||
|
||||
if (i == lastIndex) break;
|
||||
|
||||
@@ -124,8 +209,8 @@ namespace System.Unicode.Builder
|
||||
|
||||
public void SetProperties(CoreProperties property, UnicodeCharacterRange codePointRange)
|
||||
{
|
||||
int firstIndex = FindCodePoint(codePointRange.FirstCodePoint);
|
||||
int lastIndex = FindCodePoint(codePointRange.LastCodePoint);
|
||||
int firstIndex = FindUcdCodePoint(codePointRange.FirstCodePoint);
|
||||
int lastIndex = FindUcdCodePoint(codePointRange.LastCodePoint);
|
||||
|
||||
if (firstIndex < 0 && lastIndex < 0)
|
||||
{
|
||||
@@ -135,8 +220,8 @@ namespace System.Unicode.Builder
|
||||
|
||||
if (firstIndex < 0
|
||||
|| lastIndex < 0
|
||||
|| entries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|
||||
|| entries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
|
||||
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|
||||
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
|
||||
{
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
@@ -145,7 +230,7 @@ namespace System.Unicode.Builder
|
||||
|
||||
while (true)
|
||||
{
|
||||
entries[i].CoreProperties |= property;
|
||||
ucdEntries[i].CoreProperties |= property;
|
||||
|
||||
if (i == lastIndex) break;
|
||||
|
||||
@@ -160,12 +245,17 @@ namespace System.Unicode.Builder
|
||||
|
||||
public UnicodeInfo ToUnicodeData()
|
||||
{
|
||||
var finalData = new UnicodeCharacterData[entryCount];
|
||||
var finalUnicodeData = new UnicodeCharacterData[ucdEntryCount];
|
||||
|
||||
for (int i = 0; i < finalData.Length; ++i)
|
||||
finalData[i] = entries[i].ToCharacterData();
|
||||
for (int i = 0; i < finalUnicodeData.Length; ++i)
|
||||
finalUnicodeData[i] = ucdEntries[i].ToCharacterData();
|
||||
|
||||
return new UnicodeInfo(unicodeVersion, finalData, blockEntries.ToArray());
|
||||
var finalUnihanData = new UnihanCharacterData[unihanEntryCount];
|
||||
|
||||
for (int i = 0; i < finalUnihanData.Length; ++i)
|
||||
finalUnihanData[i] = unihanEntries[i].ToCharacterData();
|
||||
|
||||
return new UnicodeInfo(unicodeVersion, finalUnicodeData, finalUnihanData, blockEntries.ToArray());
|
||||
}
|
||||
|
||||
private void WriteUnicodeBlockToFile(BinaryWriter writer, UnicodeBlock block)
|
||||
@@ -182,15 +272,22 @@ namespace System.Unicode.Builder
|
||||
writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 0 });
|
||||
writer.Write((ushort)7);
|
||||
writer.Write((byte)0);
|
||||
writer.WriteCodePoint(entryCount);
|
||||
for (int i = 0; i < entryCount; ++i)
|
||||
writer.WriteCodePoint(ucdEntryCount);
|
||||
for (int i = 0; i < ucdEntryCount; ++i)
|
||||
{
|
||||
entries[i].WriteToFile(writer);
|
||||
ucdEntries[i].WriteToFile(writer);
|
||||
}
|
||||
if (blockEntries.Count > 255) throw new InvalidOperationException("There are too many block entries. The file format needs to be upgraded.");
|
||||
writer.Write((byte)blockEntries.Count);
|
||||
for (int i = 0; i < blockEntries.Count; ++i)
|
||||
{
|
||||
WriteUnicodeBlockToFile(writer, blockEntries[i]);
|
||||
}
|
||||
writer.WriteCodePoint(unihanEntryCount);
|
||||
for (int i = 0; i < unihanEntryCount; ++i)
|
||||
{
|
||||
unihanEntries[i].WriteToFile(writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,6 +53,15 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="BinaryWriterExtensions.cs" />
|
||||
<Compile Include="UnihanProperty.cs">
|
||||
<AutoGen>True</AutoGen>
|
||||
<DesignTime>True</DesignTime>
|
||||
<DependentUpon>UnihanProperty.tt</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="HexCodePoint.cs" />
|
||||
<Compile Include="UnihanCharacterDataBuilder.cs" />
|
||||
<Compile Include="UnihanDataFileReader.cs" />
|
||||
<Compile Include="Utf8Buffer.cs" />
|
||||
<Compile Include="CharacterDecompositionMapping.cs" />
|
||||
<Compile Include="FileDataSource.cs" />
|
||||
<Compile Include="HttpDataSource.cs" />
|
||||
@@ -77,6 +86,15 @@
|
||||
<Name>UnicodeInformation</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Content Include="UnihanProperty.tt">
|
||||
<Generator>TextTemplatingFileGenerator</Generator>
|
||||
<LastGenOutput>UnihanProperty.cs</LastGenOutput>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
internal sealed class UnihanCharacterDataBuilder
|
||||
{
|
||||
private readonly int codePoint;
|
||||
private UnihanNumericType numericType;
|
||||
private long numericValue;
|
||||
private string definition;
|
||||
private string mandarinReading;
|
||||
private string cantoneseReading;
|
||||
private string japaneseKunReading;
|
||||
private string japaneseOnReading;
|
||||
private string koreanReading;
|
||||
private string hangulReading;
|
||||
private string vietnameseReading;
|
||||
private string simplifiedVariant;
|
||||
private string traditionalVariant;
|
||||
|
||||
public int CodePoint { get { return codePoint; } }
|
||||
public UnihanNumericType NumericType { get { return numericType; } set { numericType = value; } }
|
||||
public long NumericValue { get { return numericValue; } set { numericValue = value; } }
|
||||
public string Definition { get { return definition; } set { definition = value; } }
|
||||
public string MandarinReading { get { return mandarinReading; } set { mandarinReading = value; } }
|
||||
public string CantoneseReading { get { return cantoneseReading; } set { cantoneseReading = value; } }
|
||||
public string JapaneseKunReading { get { return japaneseKunReading; } set { japaneseKunReading = value; } }
|
||||
public string JapaneseOnReading { get { return japaneseOnReading; } set { japaneseOnReading = value; } }
|
||||
public string KoreanReading { get { return koreanReading; } set { koreanReading = value; } }
|
||||
public string HangulReading { get { return hangulReading; } set { hangulReading = value; } }
|
||||
public string VietnameseReading { get { return vietnameseReading; } set { vietnameseReading = value; } }
|
||||
public string SimplifiedVariant { get { return simplifiedVariant; } set { simplifiedVariant = value; } }
|
||||
public string TraditionalVariant { get { return traditionalVariant; } set { traditionalVariant = value; } }
|
||||
|
||||
internal UnihanCharacterDataBuilder(int codePoint)
|
||||
{
|
||||
this.codePoint = codePoint;
|
||||
}
|
||||
|
||||
internal UnihanCharacterData ToCharacterData()
|
||||
{
|
||||
return new UnihanCharacterData
|
||||
(
|
||||
codePoint,
|
||||
numericType,
|
||||
numericValue,
|
||||
definition,
|
||||
mandarinReading,
|
||||
cantoneseReading,
|
||||
japaneseKunReading,
|
||||
japaneseOnReading,
|
||||
koreanReading,
|
||||
hangulReading,
|
||||
vietnameseReading,
|
||||
simplifiedVariant,
|
||||
traditionalVariant
|
||||
);
|
||||
}
|
||||
|
||||
internal void WriteToFile(BinaryWriter writer)
|
||||
{
|
||||
UnihanFields fields = default(UnihanFields);
|
||||
|
||||
fields |= (UnihanFields)NumericType;
|
||||
if (Definition != null) fields |= UnihanFields.Definition;
|
||||
if (MandarinReading != null) fields |= UnihanFields.MandarinReading;
|
||||
if (CantoneseReading != null) fields |= UnihanFields.CantoneseReading;
|
||||
if (JapaneseKunReading != null) fields |= UnihanFields.JapaneseKunReading;
|
||||
if (JapaneseOnReading != null) fields |= UnihanFields.JapaneseOnReading;
|
||||
if (KoreanReading != null) fields |= UnihanFields.KoreanReading;
|
||||
if (HangulReading != null) fields |= UnihanFields.HangulReading;
|
||||
if (VietnameseReading != null) fields |= UnihanFields.VietnameseReading;
|
||||
if (SimplifiedVariant != null) fields |= UnihanFields.SimplifiedVariant;
|
||||
if (TraditionalVariant != null) fields |= UnihanFields.TraditionalVariant;
|
||||
|
||||
writer.Write((ushort)fields);
|
||||
|
||||
writer.WriteCodePoint(UnihanCharacterData.PackCodePoint(codePoint));
|
||||
if ((fields & UnihanFields.OtherNumeric) != 0) writer.Write(numericValue);
|
||||
if ((fields & UnihanFields.Definition) != 0) writer.Write(Definition);
|
||||
if ((fields & UnihanFields.MandarinReading) != 0) writer.Write(MandarinReading);
|
||||
if ((fields & UnihanFields.CantoneseReading) != 0) writer.Write(CantoneseReading);
|
||||
if ((fields & UnihanFields.JapaneseKunReading) != 0) writer.Write(JapaneseKunReading);
|
||||
if ((fields & UnihanFields.JapaneseOnReading) != 0) writer.Write(JapaneseOnReading);
|
||||
if ((fields & UnihanFields.KoreanReading) != 0) writer.Write(KoreanReading);
|
||||
if ((fields & UnihanFields.HangulReading) != 0) writer.Write(HangulReading);
|
||||
if ((fields & UnihanFields.VietnameseReading) != 0) writer.Write(VietnameseReading);
|
||||
if ((fields & UnihanFields.SimplifiedVariant) != 0) writer.Write(SimplifiedVariant);
|
||||
if ((fields & UnihanFields.TraditionalVariant) != 0) writer.Write(TraditionalVariant);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
public sealed class UnihanDataFileReader : IDisposable
|
||||
{
|
||||
private readonly UnicodeDataFileReader reader;
|
||||
private int codePoint;
|
||||
private string propertyName;
|
||||
private string propertyValue;
|
||||
|
||||
public UnihanDataFileReader(Stream stream)
|
||||
: this(stream, false)
|
||||
{
|
||||
}
|
||||
|
||||
public UnihanDataFileReader(Stream stream, bool leaveOpen)
|
||||
{
|
||||
reader = new UnicodeDataFileReader(stream, '\t', leaveOpen);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
reader.Dispose();
|
||||
}
|
||||
|
||||
public bool Read()
|
||||
{
|
||||
bool result;
|
||||
|
||||
if (result = reader.MoveToNextLine())
|
||||
{
|
||||
codePoint = HexCodePoint.ParsePrefixed(reader.ReadField());
|
||||
propertyName = reader.ReadField();
|
||||
propertyValue = reader.ReadField();
|
||||
}
|
||||
else
|
||||
{
|
||||
codePoint = 0;
|
||||
propertyName = null;
|
||||
propertyValue = null;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public int CodePoint { get { return codePoint; } }
|
||||
|
||||
public string PropertyName { get { return propertyName; } }
|
||||
|
||||
public string PropertyValue { get { return propertyValue; } }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
public static class UnihanProperty
|
||||
{
|
||||
public const string kAccountingNumeric = "kAccountingNumeric";
|
||||
public const string kBigFive = "kBigFive";
|
||||
public const string kCangjie = "kCangjie";
|
||||
public const string kCantonese = "kCantonese";
|
||||
public const string kCCCII = "kCCCII";
|
||||
public const string kCheungBauer = "kCheungBauer";
|
||||
public const string kCheungBauerIndex = "kCheungBauerIndex";
|
||||
public const string kCihaiT = "kCihaiT";
|
||||
public const string kCNS1986 = "kCNS1986";
|
||||
public const string kCNS1992 = "kCNS1992";
|
||||
public const string kCompatibilityVariant = "kCompatibilityVariant";
|
||||
public const string kCowles = "kCowles";
|
||||
public const string kDaeJaweon = "kDaeJaweon";
|
||||
public const string kDefinition = "kDefinition";
|
||||
public const string kEACC = "kEACC";
|
||||
public const string kFenn = "kFenn";
|
||||
public const string kFennIndex = "kFennIndex";
|
||||
public const string kFourCornerCode = "kFourCornerCode";
|
||||
public const string kFrequency = "kFrequency";
|
||||
public const string kGB0 = "kGB0";
|
||||
public const string kGB1 = "kGB1";
|
||||
public const string kGB3 = "kGB3";
|
||||
public const string kGB5 = "kGB5";
|
||||
public const string kGB7 = "kGB7";
|
||||
public const string kGB8 = "kGB8";
|
||||
public const string kGradeLevel = "kGradeLevel";
|
||||
public const string kGSR = "kGSR";
|
||||
public const string kHangul = "kHangul";
|
||||
public const string kHanYu = "kHanYu";
|
||||
public const string kHanyuPinlu = "kHanyuPinlu";
|
||||
public const string kHanyuPinyin = "kHanyuPinyin";
|
||||
public const string kHDZRadBreak = "kHDZRadBreak";
|
||||
public const string kHKGlyph = "kHKGlyph";
|
||||
public const string kHKSCS = "kHKSCS";
|
||||
public const string kIBMJapan = "kIBMJapan";
|
||||
public const string kIICore = "kIICore";
|
||||
public const string kIRG_GSource = "kIRG_GSource";
|
||||
public const string kIRG_HSource = "kIRG_HSource";
|
||||
public const string kIRG_JSource = "kIRG_JSource";
|
||||
public const string kIRG_KPSource = "kIRG_KPSource";
|
||||
public const string kIRG_KSource = "kIRG_KSource";
|
||||
public const string kIRG_MSource = "kIRG_MSource";
|
||||
public const string kIRG_TSource = "kIRG_TSource";
|
||||
public const string kIRG_USource = "kIRG_USource";
|
||||
public const string kIRG_VSource = "kIRG_VSource";
|
||||
public const string kIRGDaeJaweon = "kIRGDaeJaweon";
|
||||
public const string kIRGDaiKanwaZiten = "kIRGDaiKanwaZiten";
|
||||
public const string kIRGHanyuDaZidian = "kIRGHanyuDaZidian";
|
||||
public const string kIRGKangXi = "kIRGKangXi";
|
||||
public const string kJapaneseKun = "kJapaneseKun";
|
||||
public const string kJapaneseOn = "kJapaneseOn";
|
||||
public const string kJis0 = "kJis0";
|
||||
public const string kJis1 = "kJis1";
|
||||
public const string kJIS0213 = "kJIS0213";
|
||||
public const string kKangXi = "kKangXi";
|
||||
public const string kKarlgren = "kKarlgren";
|
||||
public const string kKorean = "kKorean";
|
||||
public const string kKPS0 = "kKPS0";
|
||||
public const string kKPS1 = "kKPS1";
|
||||
public const string kKSC0 = "kKSC0";
|
||||
public const string kKSC1 = "kKSC1";
|
||||
public const string kLau = "kLau";
|
||||
public const string kMainlandTelegraph = "kMainlandTelegraph";
|
||||
public const string kMandarin = "kMandarin";
|
||||
public const string kMatthews = "kMatthews";
|
||||
public const string kMeyerWempe = "kMeyerWempe";
|
||||
public const string kMorohashi = "kMorohashi";
|
||||
public const string kNelson = "kNelson";
|
||||
public const string kOtherNumeric = "kOtherNumeric";
|
||||
public const string kPhonetic = "kPhonetic";
|
||||
public const string kPrimaryNumeric = "kPrimaryNumeric";
|
||||
public const string kPseudoGB1 = "kPseudoGB1";
|
||||
public const string kRSAdobe_Japan1_6 = "kRSAdobe_Japan1_6";
|
||||
public const string kRSJapanese = "kRSJapanese";
|
||||
public const string kRSKangXi = "kRSKangXi";
|
||||
public const string kRSKanWa = "kRSKanWa";
|
||||
public const string kRSKorean = "kRSKorean";
|
||||
public const string kRSUnicode = "kRSUnicode";
|
||||
public const string kSBGY = "kSBGY";
|
||||
public const string kSemanticVariant = "kSemanticVariant";
|
||||
public const string kSimplifiedVariant = "kSimplifiedVariant";
|
||||
public const string kSpecializedSemanticVariant = "kSpecializedSemanticVariant";
|
||||
public const string kTaiwanTelegraph = "kTaiwanTelegraph";
|
||||
public const string kTang = "kTang";
|
||||
public const string kTotalStrokes = "kTotalStrokes";
|
||||
public const string kTraditionalVariant = "kTraditionalVariant";
|
||||
public const string kVietnamese = "kVietnamese";
|
||||
public const string kXerox = "kXerox";
|
||||
public const string kXHC1983 = "kXHC1983";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
<#@ template debug="false" hostspecific="false" language="C#" #>
|
||||
<#@ assembly name="System.Core" #>
|
||||
<#@ import namespace="System.Linq" #>
|
||||
<#@ import namespace="System.Text" #>
|
||||
<#@ import namespace="System.Collections.Generic" #>
|
||||
<#@ output extension=".cs" #>
|
||||
<#
|
||||
var propertyNames = new[]
|
||||
{
|
||||
"kAccountingNumeric",
|
||||
"kBigFive",
|
||||
"kCangjie",
|
||||
"kCantonese",
|
||||
"kCCCII",
|
||||
"kCheungBauer",
|
||||
"kCheungBauerIndex",
|
||||
"kCihaiT",
|
||||
"kCNS1986",
|
||||
"kCNS1992",
|
||||
"kCompatibilityVariant",
|
||||
"kCowles",
|
||||
"kDaeJaweon",
|
||||
"kDefinition",
|
||||
"kEACC",
|
||||
"kFenn",
|
||||
"kFennIndex",
|
||||
"kFourCornerCode",
|
||||
"kFrequency",
|
||||
"kGB0",
|
||||
"kGB1",
|
||||
"kGB3",
|
||||
"kGB5",
|
||||
"kGB7",
|
||||
"kGB8",
|
||||
"kGradeLevel",
|
||||
"kGSR",
|
||||
"kHangul",
|
||||
"kHanYu",
|
||||
"kHanyuPinlu",
|
||||
"kHanyuPinyin",
|
||||
"kHDZRadBreak",
|
||||
"kHKGlyph",
|
||||
"kHKSCS",
|
||||
"kIBMJapan",
|
||||
"kIICore",
|
||||
"kIRG_GSource",
|
||||
"kIRG_HSource",
|
||||
"kIRG_JSource",
|
||||
"kIRG_KPSource",
|
||||
"kIRG_KSource",
|
||||
"kIRG_MSource",
|
||||
"kIRG_TSource",
|
||||
"kIRG_USource",
|
||||
"kIRG_VSource",
|
||||
"kIRGDaeJaweon",
|
||||
"kIRGDaiKanwaZiten",
|
||||
"kIRGHanyuDaZidian",
|
||||
"kIRGKangXi",
|
||||
"kJapaneseKun",
|
||||
"kJapaneseOn",
|
||||
"kJis0",
|
||||
"kJis1",
|
||||
"kJIS0213",
|
||||
"kKangXi",
|
||||
"kKarlgren",
|
||||
"kKorean",
|
||||
"kKPS0",
|
||||
"kKPS1",
|
||||
"kKSC0",
|
||||
"kKSC1",
|
||||
"kLau",
|
||||
"kMainlandTelegraph",
|
||||
"kMandarin",
|
||||
"kMatthews",
|
||||
"kMeyerWempe",
|
||||
"kMorohashi",
|
||||
"kNelson",
|
||||
"kOtherNumeric",
|
||||
"kPhonetic",
|
||||
"kPrimaryNumeric",
|
||||
"kPseudoGB1",
|
||||
"kRSAdobe_Japan1_6",
|
||||
"kRSJapanese",
|
||||
"kRSKangXi",
|
||||
"kRSKanWa",
|
||||
"kRSKorean",
|
||||
"kRSUnicode",
|
||||
"kSBGY",
|
||||
"kSemanticVariant",
|
||||
"kSimplifiedVariant",
|
||||
"kSpecializedSemanticVariant",
|
||||
"kTaiwanTelegraph",
|
||||
"kTang",
|
||||
"kTotalStrokes",
|
||||
"kTraditionalVariant",
|
||||
"kVietnamese",
|
||||
"kXerox",
|
||||
"kXHC1983",
|
||||
};
|
||||
#>
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
public static class UnihanProperty
|
||||
{
|
||||
<# foreach (string propertyName in propertyNames) { #>
|
||||
public const string <#=propertyName#> = "<#=propertyName#>";
|
||||
<# } #>
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace System.Unicode.Builder
|
||||
{
|
||||
public struct Utf8Buffer : IDisposable
|
||||
{
|
||||
private static readonly ConcurrentStack<byte[]> bufferStack = new ConcurrentStack<byte[]>();
|
||||
|
||||
public static Utf8Buffer Get()
|
||||
{
|
||||
byte[] buffer;
|
||||
|
||||
return new Utf8Buffer(bufferStack.TryPop(out buffer) ? buffer : new byte[100]);
|
||||
}
|
||||
|
||||
private byte[] buffer;
|
||||
private int length;
|
||||
|
||||
private Utf8Buffer(byte[] buffer)
|
||||
{
|
||||
this.buffer = buffer;
|
||||
this.length = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (buffer != null)
|
||||
{
|
||||
bufferStack.Push(buffer);
|
||||
this = default(Utf8Buffer);
|
||||
}
|
||||
}
|
||||
|
||||
public int Length { get { return length; } }
|
||||
|
||||
private void EnsureExtraCapacity(int count)
|
||||
{
|
||||
if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity");
|
||||
if (buffer.Length < checked(length + count))
|
||||
Array.Resize(ref buffer, Math.Max(length + count, buffer.Length << 1));
|
||||
}
|
||||
|
||||
public void Append(byte[] value, int startIndex, int count)
|
||||
{
|
||||
if (value == null) throw new ArgumentNullException("value");
|
||||
if (startIndex >= value.Length) throw new ArgumentOutOfRangeException("startIndex");
|
||||
if (checked(count += startIndex) > value.Length) throw new ArgumentOutOfRangeException("count");
|
||||
|
||||
EnsureExtraCapacity(value.Length);
|
||||
|
||||
var buffer = this.buffer;
|
||||
|
||||
for (int i = startIndex; i < count; ++i)
|
||||
{
|
||||
buffer[length++] = value[i];
|
||||
}
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return length > 0 ? Encoding.UTF8.GetString(buffer, 0, length) : string.Empty;
|
||||
}
|
||||
|
||||
public string ToTrimmedString()
|
||||
{
|
||||
if (length == 0) return string.Empty;
|
||||
|
||||
var buffer = this.buffer;
|
||||
int start = 0;
|
||||
int end = length;
|
||||
|
||||
while (buffer[start] == ' ') if (++start == length) return string.Empty;
|
||||
while (buffer[--end] == ' ') ;
|
||||
|
||||
return end > start ? Encoding.UTF8.GetString(buffer, start, end - start + 1) : string.Empty;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -66,7 +66,7 @@ namespace System.Unicode.Tests
|
||||
{
|
||||
var readData = UnicodeInfo.FromStream(stream);
|
||||
|
||||
Assert.AreEqual((int)'\t', data.Get('\t').CodePointRange.FirstCodePoint);
|
||||
Assert.AreEqual((int)'\t', data.GetUcd('\t').CodePointRange.FirstCodePoint);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,8 @@ namespace System.Unicode
|
||||
public struct UnicodeCharInfo
|
||||
{
|
||||
private readonly int codePoint;
|
||||
private readonly UnicodeCharacterData characterData;
|
||||
private readonly UnicodeCharacterData unicodeCharacterData;
|
||||
private readonly UnihanCharacterData unihanCharacterData;
|
||||
private readonly string block;
|
||||
|
||||
public int CodePoint { get { return codePoint; } }
|
||||
@@ -19,32 +20,46 @@ namespace System.Unicode
|
||||
{
|
||||
get
|
||||
{
|
||||
return characterData.Name == null || characterData.CodePointRange.IsSingleCodePoint ?
|
||||
characterData.Name :
|
||||
characterData.Name + "-" + codePoint.ToString("X4");
|
||||
return unicodeCharacterData.Name == null || unicodeCharacterData.CodePointRange.IsSingleCodePoint ?
|
||||
unicodeCharacterData.Name :
|
||||
unicodeCharacterData.Name + "-" + codePoint.ToString("X4");
|
||||
}
|
||||
}
|
||||
|
||||
public UnicodeCategory Category { get { return characterData.Category; } }
|
||||
public UnicodeCategory Category { get { return unicodeCharacterData.Category; } }
|
||||
public string Block { get { return block; } }
|
||||
public CanonicalCombiningClass CanonicalCombiningClass { get { return characterData.CanonicalCombiningClass; } }
|
||||
public BidirectionalClass BidirectionalClass { get { return characterData.BidirectionalClass; } }
|
||||
public CompatibilityFormattingTag DecompositionType { get { return characterData.DecompositionType; } }
|
||||
public string DecompositionMapping { get { return characterData.DecompositionMapping; } }
|
||||
public UnicodeNumericType NumericType { get { return characterData.NumericType; } }
|
||||
public UnicodeRationalNumber? NumericValue { get { return characterData.NumericValue; } }
|
||||
public bool BidirectionalMirrored { get { return characterData.BidirectionalMirrored; } }
|
||||
public string OldName { get { return characterData.OldName; } }
|
||||
public string SimpleUpperCaseMapping { get { return characterData.SimpleUpperCaseMapping; } }
|
||||
public string SimpleLowerCaseMapping { get { return characterData.SimpleLowerCaseMapping; } }
|
||||
public string SimpleTitleCaseMapping { get { return characterData.SimpleTitleCaseMapping; } }
|
||||
public ContributoryProperties ContributoryProperties { get { return characterData.ContributoryProperties; } }
|
||||
public CoreProperties CoreProperties { get { return characterData.CoreProperties; } }
|
||||
public CanonicalCombiningClass CanonicalCombiningClass { get { return unicodeCharacterData.CanonicalCombiningClass; } }
|
||||
public BidirectionalClass BidirectionalClass { get { return unicodeCharacterData.BidirectionalClass; } }
|
||||
public CompatibilityFormattingTag DecompositionType { get { return unicodeCharacterData.DecompositionType; } }
|
||||
public string DecompositionMapping { get { return unicodeCharacterData.DecompositionMapping; } }
|
||||
public UnicodeNumericType NumericType { get { return unihanCharacterData != null ? unihanCharacterData.NumericType != UnihanNumericType.None ? UnicodeNumericType.Numeric : UnicodeNumericType.None : unicodeCharacterData.NumericType; } }
|
||||
public UnihanNumericType UnihanNumericType { get { return unihanCharacterData != null ? unihanCharacterData.NumericType : UnihanNumericType.None; } }
|
||||
public UnicodeRationalNumber? NumericValue { get { return unihanCharacterData != null && unihanCharacterData.NumericType != UnihanNumericType.None ? new UnicodeRationalNumber(unihanCharacterData.NumericValue, 1) : unicodeCharacterData.NumericValue; } }
|
||||
public bool BidirectionalMirrored { get { return unicodeCharacterData.BidirectionalMirrored; } }
|
||||
public string OldName { get { return unicodeCharacterData.OldName; } }
|
||||
public string SimpleUpperCaseMapping { get { return unicodeCharacterData.SimpleUpperCaseMapping; } }
|
||||
public string SimpleLowerCaseMapping { get { return unicodeCharacterData.SimpleLowerCaseMapping; } }
|
||||
public string SimpleTitleCaseMapping { get { return unicodeCharacterData.SimpleTitleCaseMapping; } }
|
||||
public ContributoryProperties ContributoryProperties { get { return unicodeCharacterData.ContributoryProperties; } }
|
||||
public CoreProperties CoreProperties { get { return unicodeCharacterData.CoreProperties; } }
|
||||
|
||||
internal UnicodeCharInfo(int codePoint, UnicodeCharacterData characterData, string block)
|
||||
public string Definition { get { return unihanCharacterData?.Definition; } }
|
||||
public string MandarinReading { get { return unihanCharacterData?.MandarinReading; } }
|
||||
public string CantoneseReading { get { return unihanCharacterData?.CantoneseReading; } }
|
||||
public string JapaneseKunReading { get { return unihanCharacterData?.JapaneseKunReading; } }
|
||||
public string JapaneseOnReading { get { return unihanCharacterData?.JapaneseOnReading; } }
|
||||
public string KoreanReading { get { return unihanCharacterData?.KoreanReading; } }
|
||||
public string HangulReading { get { return unihanCharacterData?.HangulReading; } }
|
||||
public string VietnameseReading { get { return unihanCharacterData?.VietnameseReading; } }
|
||||
|
||||
public string SimplifiedVariant { get { return unihanCharacterData?.SimplifiedVariant; } }
|
||||
public string TraditionalVariant { get { return unihanCharacterData?.TraditionalVariant; } }
|
||||
|
||||
internal UnicodeCharInfo(int codePoint, UnicodeCharacterData unicodeCharacterData, UnihanCharacterData unihanCharacterData, string block)
|
||||
{
|
||||
this.codePoint = codePoint;
|
||||
this.characterData = characterData;
|
||||
this.unicodeCharacterData = unicodeCharacterData;
|
||||
this.unihanCharacterData = unihanCharacterData;
|
||||
this.block = block;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,10 +23,6 @@ namespace System.Unicode
|
||||
}
|
||||
}
|
||||
|
||||
private readonly Version unicodeVersion;
|
||||
private readonly UnicodeCharacterData[] characterData;
|
||||
private readonly UnicodeBlock[] blockEntries;
|
||||
|
||||
public static UnicodeInfo FromStream(Stream stream)
|
||||
{
|
||||
using (var reader = new BinaryReader(stream, Encoding.UTF8))
|
||||
@@ -44,11 +40,11 @@ namespace System.Unicode
|
||||
|
||||
var unicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
|
||||
|
||||
var characterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
|
||||
var unicodeCharacterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
|
||||
|
||||
for (int i = 0; i < characterDataEntries.Length; ++i)
|
||||
for (int i = 0; i < unicodeCharacterDataEntries.Length; ++i)
|
||||
{
|
||||
characterDataEntries[i] = ReadCharacterDataEntry(reader);
|
||||
unicodeCharacterDataEntries[i] = ReadUnicodeCharacterDataEntry(reader);
|
||||
}
|
||||
|
||||
var blockEntries = new UnicodeBlock[reader.ReadByte()];
|
||||
@@ -56,13 +52,20 @@ namespace System.Unicode
|
||||
for (int i = 0; i < blockEntries.Length; ++i)
|
||||
{
|
||||
blockEntries[i] = ReadBlockEntry(reader);
|
||||
}
|
||||
}
|
||||
|
||||
return new UnicodeInfo(unicodeVersion, characterDataEntries, blockEntries);
|
||||
var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)];
|
||||
|
||||
for (int i = 0; i < unihanCharacterDataEntries.Length; ++i)
|
||||
{
|
||||
unihanCharacterDataEntries[i] = ReadUnihanCharacterDataEntry(reader);
|
||||
}
|
||||
|
||||
return new UnicodeInfo(unicodeVersion, unicodeCharacterDataEntries, unihanCharacterDataEntries, blockEntries);
|
||||
}
|
||||
}
|
||||
|
||||
private static UnicodeCharacterData ReadCharacterDataEntry(BinaryReader reader)
|
||||
private static UnicodeCharacterData ReadUnicodeCharacterDataEntry(BinaryReader reader)
|
||||
{
|
||||
var fields = (UcdFields)reader.ReadUInt16();
|
||||
|
||||
@@ -107,6 +110,46 @@ namespace System.Unicode
|
||||
);
|
||||
}
|
||||
|
||||
private static UnihanCharacterData ReadUnihanCharacterDataEntry(BinaryReader reader)
|
||||
{
|
||||
var fields = (UnihanFields)reader.ReadUInt16();
|
||||
|
||||
int codePoint = UnihanCharacterData.UnpackCodePoint(ReadCodePoint(reader));
|
||||
|
||||
var numericType = (UnihanNumericType)((int)(fields & UnihanFields.OtherNumeric));
|
||||
long numericValue = numericType != UnihanNumericType.None ?
|
||||
reader.ReadInt64() :
|
||||
0;
|
||||
|
||||
string definition = (fields & UnihanFields.Definition) != 0 ? reader.ReadString() : null;
|
||||
string mandarinReading = (fields & UnihanFields.MandarinReading) != 0 ? reader.ReadString() : null;
|
||||
string cantoneseReading = (fields & UnihanFields.CantoneseReading) != 0 ? reader.ReadString() : null;
|
||||
string japaneseKunReading = (fields & UnihanFields.JapaneseKunReading) != 0 ? reader.ReadString() : null;
|
||||
string japaneseOnReading = (fields & UnihanFields.JapaneseOnReading) != 0 ? reader.ReadString() : null;
|
||||
string koreanReading = (fields & UnihanFields.KoreanReading) != 0 ? reader.ReadString() : null;
|
||||
string hangulReading = (fields & UnihanFields.HangulReading) != 0 ? reader.ReadString() : null;
|
||||
string vietnameseReading = (fields & UnihanFields.VietnameseReading) != 0 ? reader.ReadString() : null;
|
||||
string simplifiedVariant = (fields & UnihanFields.SimplifiedVariant) != 0 ? reader.ReadString() : null;
|
||||
string traditionalVariant = (fields & UnihanFields.TraditionalVariant) != 0 ? reader.ReadString() : null;
|
||||
|
||||
return new UnihanCharacterData
|
||||
(
|
||||
codePoint,
|
||||
numericType,
|
||||
numericValue,
|
||||
definition,
|
||||
mandarinReading,
|
||||
cantoneseReading,
|
||||
japaneseKunReading,
|
||||
japaneseOnReading,
|
||||
koreanReading,
|
||||
hangulReading,
|
||||
vietnameseReading,
|
||||
simplifiedVariant,
|
||||
traditionalVariant
|
||||
);
|
||||
}
|
||||
|
||||
private static UnicodeBlock ReadBlockEntry(BinaryReader reader)
|
||||
{
|
||||
return new UnicodeBlock(new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)), reader.ReadString());
|
||||
@@ -140,27 +183,57 @@ namespace System.Unicode
|
||||
}
|
||||
}
|
||||
|
||||
internal UnicodeInfo(Version unicodeVersion, UnicodeCharacterData[] characterData, UnicodeBlock[] blockEntries)
|
||||
private readonly Version unicodeVersion;
|
||||
private readonly UnicodeCharacterData[] unicodeCharacterData;
|
||||
private readonly UnihanCharacterData[] unihanCharacterData;
|
||||
private readonly UnicodeBlock[] blockEntries;
|
||||
|
||||
internal UnicodeInfo(Version unicodeVersion, UnicodeCharacterData[] unicodeCharacterData, UnihanCharacterData[] unihanCharacterData, UnicodeBlock[] blockEntries)
|
||||
{
|
||||
this.unicodeVersion = unicodeVersion;
|
||||
this.characterData = characterData;
|
||||
this.unicodeCharacterData = unicodeCharacterData;
|
||||
this.unihanCharacterData = unihanCharacterData;
|
||||
this.blockEntries = blockEntries;
|
||||
}
|
||||
|
||||
public Version UnicodeVersion { get { return unicodeVersion; } }
|
||||
|
||||
private UnicodeCharacterData FindCodePoint(int codePoint)
|
||||
private UnicodeCharacterData FindUnicodeCodePoint(int codePoint)
|
||||
{
|
||||
int minIndex = 0;
|
||||
int maxIndex = characterData.Length - 1;
|
||||
int maxIndex = unicodeCharacterData.Length - 1;
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = characterData[index].CodePointRange.CompareCodePoint(codePoint);
|
||||
int Δ = unicodeCharacterData[index].CodePointRange.CompareCodePoint(codePoint);
|
||||
|
||||
if (Δ == 0) return characterData[index];
|
||||
if (Δ == 0) return unicodeCharacterData[index];
|
||||
else if (Δ < 0) maxIndex = index - 1;
|
||||
else minIndex = index + 1;
|
||||
} while (minIndex <= maxIndex);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private UnihanCharacterData FindUnihanCodePoint(int codePoint)
|
||||
{
|
||||
int minIndex;
|
||||
int maxIndex;
|
||||
|
||||
if (unihanCharacterData.Length == 0 || codePoint < unihanCharacterData[minIndex = 0].CodePoint || codePoint > unihanCharacterData[maxIndex = unicodeCharacterData.Length - 1].CodePoint)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
int index = (minIndex + maxIndex) >> 1;
|
||||
|
||||
int Δ = codePoint - unihanCharacterData[index].CodePoint;
|
||||
|
||||
if (Δ == 0) return unihanCharacterData[index];
|
||||
else if (Δ < 0) maxIndex = index - 1;
|
||||
else minIndex = index + 1;
|
||||
} while (minIndex <= maxIndex);
|
||||
@@ -196,7 +269,7 @@ namespace System.Unicode
|
||||
|
||||
public UnicodeCharInfo GetCharInfo(int codePoint)
|
||||
{
|
||||
return new UnicodeCharInfo(codePoint, FindCodePoint(codePoint), GetBlockName(codePoint));
|
||||
return new UnicodeCharInfo(codePoint, FindUnicodeCodePoint(codePoint), FindUnihanCodePoint(codePoint), GetBlockName(codePoint));
|
||||
}
|
||||
|
||||
public UnicodeBlock[] GetBlocks()
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace System.Unicode
|
||||
{
|
||||
public readonly int CodePoint;
|
||||
public readonly UnihanNumericType NumericType;
|
||||
public readonly ulong NumericValue;
|
||||
public readonly long NumericValue;
|
||||
public readonly string Definition;
|
||||
public readonly string MandarinReading;
|
||||
public readonly string CantoneseReading;
|
||||
@@ -22,6 +22,38 @@ namespace System.Unicode
|
||||
public readonly string SimplifiedVariant;
|
||||
public readonly string TraditionalVariant;
|
||||
|
||||
internal UnihanCharacterData
|
||||
(
|
||||
int codePoint,
|
||||
UnihanNumericType numericType,
|
||||
long numericValue,
|
||||
string definition,
|
||||
string mandarinReading,
|
||||
string cantoneseReading,
|
||||
string japaneseKunReading,
|
||||
string japaneseOnReading,
|
||||
string koreanReading,
|
||||
string hangulReading,
|
||||
string vietnameseReading,
|
||||
string simplifiedVariant,
|
||||
string traditionalVariant
|
||||
)
|
||||
{
|
||||
CodePoint = codePoint;
|
||||
NumericType = numericType;
|
||||
NumericValue = numericValue;
|
||||
Definition = definition;
|
||||
MandarinReading = mandarinReading;
|
||||
CantoneseReading = cantoneseReading;
|
||||
JapaneseKunReading = japaneseKunReading;
|
||||
JapaneseOnReading = japaneseOnReading;
|
||||
KoreanReading = koreanReading;
|
||||
HangulReading = hangulReading;
|
||||
VietnameseReading = vietnameseReading;
|
||||
SimplifiedVariant = simplifiedVariant;
|
||||
TraditionalVariant = traditionalVariant;
|
||||
}
|
||||
|
||||
internal static int PackCodePoint(int codePoint)
|
||||
{
|
||||
if (codePoint >= 0x3400)
|
||||
|
||||
@@ -10,9 +10,9 @@ namespace System.Unicode
|
||||
internal enum UnihanFields : ushort
|
||||
{
|
||||
// NumericType / NumericValue : Not exactly a bit mask here…
|
||||
AccountingNumeric = 1,
|
||||
OtherNumeric = 2,
|
||||
PrimaryNumeric = 3,
|
||||
PrimaryNumeric = 1,
|
||||
AccountingNumeric = 2,
|
||||
OtherNumeric = 3,
|
||||
|
||||
Definition = 4,
|
||||
MandarinReading = 8,
|
||||
|
||||
@@ -10,11 +10,11 @@ namespace System.Unicode
|
||||
public enum UnihanNumericType : byte
|
||||
{
|
||||
None = 0,
|
||||
[ValueName("kAccountingNumeric")]
|
||||
Accounting = 1,
|
||||
[ValueName("kOtherNumeric")]
|
||||
Other = 2,
|
||||
[ValueName("kPrimaryNumeric")]
|
||||
Primary = 3
|
||||
Primary = 1,
|
||||
[ValueName("kAccountingNumeric")]
|
||||
Accounting = 2,
|
||||
[ValueName("kOtherNumeric")]
|
||||
Other = 3,
|
||||
}
|
||||
}
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user