swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added support for emoji properties.

Browse Source
This commit is contained in:
GoldenCrystal
2017-06-25 17:14:03 +02:00
parent f9976c7a03
commit 642b675b3d
15 changed files with 185 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeCharacterInspector/App.xaml
+1
View File
@@ -11,5 +11,6 @@
<local:Utf32ToNameConverter x:Key="Utf32ToNameConverter" />
<local:StringToUtf32Converter x:Key="StringToUtf32Converter" />
<local:RadicalStrokeCountToCharConverter x:Key="RadicalStrokeCountToCharConverter" />
<local:CodePointToFontFamilyConverter x:Key="CodePointToFontFamilyConverter" DefaultFontFamily="Segoe UI" EmojiFontFamily="Segoe UI Emoji" />
</Application.Resources>
</Application>
UnicodeCharacterInspector/CharacterInfoViewModel.cs
+30 -129
View File
@@ -1,9 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Unicode;
namespace UnicodeCharacterInspector
@@ -67,6 +63,7 @@ namespace UnicodeCharacterInspector
NotifyPropertyChanged(nameof(NumericValue));
NotifyPropertyChanged(nameof(ContributoryProperties));
NotifyPropertyChanged(nameof(CoreProperties));
NotifyPropertyChanged(nameof(EmojiProperties));
NotifyPropertyChanged(nameof(RadicalStrokeCounts));
NotifyPropertyChanged(nameof(CrossReferences));
NotifyPropertyChanged(nameof(MandarinReading));
@@ -88,145 +85,49 @@ namespace UnicodeCharacterInspector
displayText = character != null ? UnicodeInfo.GetDisplayText(characterInfo) : null;
if (displayText != oldValue)
NotifyPropertyChanged(nameof(DisplayText));
if (displayText != oldValue) NotifyPropertyChanged(nameof(DisplayText));
}
public string DisplayText { get { return displayText; } }
public int? CodePoint
{
get { return character != null ? codePoint : null as int?; }
}
public int? CodePoint => character != null ? codePoint : null as int?;
public string Name
{
get { return character != null ? characterInfo.Name : null; }
}
public string Name => character != null ? characterInfo.Name : null;
public string OldName => character != null ? characterInfo.OldName : null;
public UnicodeNameAliasCollection NameAliases => character != null ? characterInfo.NameAliases : UnicodeNameAliasCollection.Empty;
public UnicodeNameAliasCollection NameAliases
{
get { return character != null ? characterInfo.NameAliases : UnicodeNameAliasCollection.Empty; }
}
public string Definition => character != null ? characterInfo.Definition : null;
public string OldName
{
get { return character != null ? characterInfo.OldName : null; }
}
public UnicodeCategory? Category => character != null ? characterInfo.Category : null as UnicodeCategory?;
public string Block => character != null ? characterInfo.Block : null;
public string Definition
{
get { return character != null ? characterInfo.Definition : null; }
}
public CanonicalCombiningClass? CanonicalCombiningClass => character != null ? characterInfo.CanonicalCombiningClass : null as CanonicalCombiningClass?;
public BidirectionalClass? BidirectionalClass => character != null ? characterInfo.BidirectionalClass : null as BidirectionalClass?;
public CompatibilityFormattingTag? DecompositionType => character != null && characterInfo.DecompositionMapping != null ? characterInfo.DecompositionType : null as CompatibilityFormattingTag?;
public UnicodeCategory? Category
{
get { return character != null ? characterInfo.Category : null as UnicodeCategory?; }
}
public string DecompositionMapping => character != null ? characterInfo.DecompositionMapping : null;
public string Block
{
get { return character != null ? characterInfo.Block : null; }
}
public UnicodeNumericType? NumericType => character != null ? characterInfo.NumericType : null as UnicodeNumericType?;
public UnihanNumericType? UnihanNumericType => character != null ? characterInfo.UnihanNumericType : null as UnihanNumericType?;
public UnicodeRationalNumber? NumericValue => character != null && characterInfo.NumericType != UnicodeNumericType.None ? characterInfo.NumericValue : null as UnicodeRationalNumber?;
public CanonicalCombiningClass? CanonicalCombiningClass
{
get { return character != null ? characterInfo.CanonicalCombiningClass : null as CanonicalCombiningClass?; }
}
public ContributoryProperties? ContributoryProperties => character != null ? characterInfo.ContributoryProperties : null as ContributoryProperties?;
public CoreProperties? CoreProperties => character != null ? characterInfo.CoreProperties : null as CoreProperties?;
public EmojiProperties? EmojiProperties => character != null ? characterInfo.EmojiProperties : null as EmojiProperties?;
public BidirectionalClass? BidirectionalClass
{
get { return character != null ? characterInfo.BidirectionalClass : null as BidirectionalClass?; }
}
public UnicodeRadicalStrokeCountCollection RadicalStrokeCounts => character != null ? characterInfo.UnicodeRadicalStrokeCounts : UnicodeRadicalStrokeCountCollection.Empty;
public CompatibilityFormattingTag? DecompositionType
{
get { return character != null && characterInfo.DecompositionMapping != null ? characterInfo.DecompositionType : null as CompatibilityFormattingTag?; }
}
public UnicodeCrossReferenceCollection CrossReferences => character != null ? characterInfo.CrossRerefences : UnicodeCrossReferenceCollection.Empty;
public string DecompositionMapping
{
get { return character != null ? characterInfo.DecompositionMapping : null; }
}
public string MandarinReading => character != null ? characterInfo.MandarinReading : null;
public string CantoneseReading => character != null ? characterInfo.CantoneseReading : null;
public string JapaneseKunReading => character != null ? characterInfo.JapaneseKunReading : null;
public string JapaneseOnReading => character != null ? characterInfo.JapaneseOnReading : null;
public string KoreanReading => character != null ? characterInfo.KoreanReading : null;
public string HangulReading => character != null ? characterInfo.HangulReading : null;
public string VietnameseReading => character != null ? characterInfo.VietnameseReading : null;
public UnicodeNumericType? NumericType
{
get { return character != null ? characterInfo.NumericType : null as UnicodeNumericType?; }
}
public UnihanNumericType? UnihanNumericType
{
get { return character != null ? characterInfo.UnihanNumericType : null as UnihanNumericType?; }
}
public UnicodeRationalNumber? NumericValue
{
get { return character != null && characterInfo.NumericType != UnicodeNumericType.None ? characterInfo.NumericValue : null as UnicodeRationalNumber?; }
}
public ContributoryProperties? ContributoryProperties
{
get { return character != null ? characterInfo.ContributoryProperties : null as ContributoryProperties?; }
}
public CoreProperties? CoreProperties
{
get { return character != null ? characterInfo.CoreProperties : null as CoreProperties?; }
}
public UnicodeRadicalStrokeCountCollection RadicalStrokeCounts
{
get { return character != null ? characterInfo.UnicodeRadicalStrokeCounts : UnicodeRadicalStrokeCountCollection.Empty; }
}
public UnicodeCrossReferenceCollection CrossReferences
{
get { return character != null ? characterInfo.CrossRerefences : UnicodeCrossReferenceCollection.Empty; }
}
public string MandarinReading
{
get { return character != null ? characterInfo.MandarinReading : null; }
}
public string CantoneseReading
{
get { return character != null ? characterInfo.CantoneseReading : null; }
}
public string JapaneseKunReading
{
get { return character != null ? characterInfo.JapaneseKunReading : null; }
}
public string JapaneseOnReading
{
get { return character != null ? characterInfo.JapaneseOnReading : null; }
}
public string KoreanReading
{
get { return character != null ? characterInfo.KoreanReading : null; }
}
public string HangulReading
{
get { return character != null ? characterInfo.HangulReading : null; }
}
public string VietnameseReading
{
get { return character != null ? characterInfo.VietnameseReading : null; }
}
public string SimplifiedVariant
{
get { return character != null ? characterInfo.SimplifiedVariant : null; }
}
public string TraditionalVariant
{
get { return character != null ? characterInfo.TraditionalVariant : null; }
}
public string SimplifiedVariant => character != null ? characterInfo.SimplifiedVariant : null;
public string TraditionalVariant => character != null ? characterInfo.TraditionalVariant : null;
}
}
UnicodeCharacterInspector/CodePointToFontFamilyConverter.cs
+23
View File
@@ -0,0 +1,23 @@
using System;
using System.Globalization;
using System.Unicode;
using System.Windows.Data;
using System.Windows.Media;
namespace UnicodeCharacterInspector
{
internal sealed class CodePointToFontFamilyConverter : IValueConverter
{
public FontFamily DefaultFontFamily { get; set; }
public FontFamily EmojiFontFamily { get; set; }
public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
=> value is int codePoint && UnicodeInfo.GetCharInfo(codePoint).EmojiProperties != 0 ?
EmojiFontFamily :
DefaultFontFamily;
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
=> throw new NotSupportedException();
}
}
UnicodeCharacterInspector/MainWindow.xaml
+7 -4
View File
@@ -41,11 +41,11 @@
<ListBox Grid.Row="1" Grid.RowSpan="3" ItemsSource="{Binding Characters}" SelectedIndex="{Binding SelectedCharacterIndex}">
<ListBox.ItemTemplate>
<DataTemplate>
<TextBlock><Run Text="{Binding DisplayText, Mode=OneWay}" /> <Run Text="{Binding CodePoint, Mode=OneWay, StringFormat=U+{0:X4}}" Foreground="{DynamicResource {x:Static SystemColors.GrayTextBrushKey}}" FontStyle="Italic" /></TextBlock>
<TextBlock FontFamily="{Binding CodePoint, Mode=OneWay, Converter={StaticResource CodePointToFontFamilyConverter}}"><Run Text="{Binding DisplayText, Mode=OneWay}" /> <Run Text="{Binding CodePoint, Mode=OneWay, StringFormat=U+{0:X4}}" Foreground="{DynamicResource {x:Static SystemColors.GrayTextBrushKey}}" FontStyle="Italic" /></TextBlock>
</DataTemplate>
</ListBox.ItemTemplate>
</ListBox>
<TextBlock Grid.Row="1" Grid.Column="1" TextOptions.TextHintingMode="Fixed" TextOptions.TextFormattingMode="Ideal" TextOptions.TextRenderingMode="Grayscale" Text="{Binding SelectedCharacterInfo.DisplayText}" FontSize="96" HorizontalAlignment="Center" VerticalAlignment="Center" />
<TextBlock Grid.Row="1" Grid.Column="1" TextOptions.TextHintingMode="Fixed" TextOptions.TextFormattingMode="Ideal" TextOptions.TextRenderingMode="Grayscale" Text="{Binding SelectedCharacterInfo.DisplayText}" FontSize="96" HorizontalAlignment="Center" VerticalAlignment="Center" FontFamily="{Binding SelectedCharacterInfo.CodePoint, Mode=OneWay, Converter={StaticResource CodePointToFontFamilyConverter}}" />
<ScrollViewer Grid.Row="2" Grid.Column="1" VerticalScrollBarVisibility="Auto">
<Grid>
<Grid.Resources>
@@ -86,6 +86,7 @@
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="*" />
@@ -158,8 +159,10 @@
<TextBlock Grid.Row="23" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.ContributoryProperties, Converter={StaticResource ZeroToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.ContributoryProperties}" />
<TextBlock Grid.Row="24" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Core Properties" />
<TextBlock Grid.Row="24" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.CoreProperties, Converter={StaticResource ZeroToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.CoreProperties}" />
<TextBlock Grid.Row="25" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.CrossReferences.Count, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Cross-References" />
<ItemsControl Grid.Row="25" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.CrossReferences.Count, Converter={StaticResource ZeroToVisibilityConverter}}" ItemsSource="{Binding SelectedCharacterInfo.CrossReferences}">
<TextBlock Grid.Row="25" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.EmojiProperties, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Emoji Properties" />
<TextBlock Grid.Row="25" Grid.Column="1" VerticalAlignment="Top" TextWrapping="WrapWithOverflow" Visibility="{Binding SelectedCharacterInfo.EmojiProperties, Converter={StaticResource ZeroToVisibilityConverter}}" Text="{Binding SelectedCharacterInfo.EmojiProperties}" />
<TextBlock Grid.Row="26" Grid.Column="0" VerticalAlignment="Top" Visibility="{Binding SelectedCharacterInfo.CrossReferences.Count, Converter={StaticResource ZeroToVisibilityConverter}}" Text="Cross-References" />
<ItemsControl Grid.Row="26" Grid.Column="1" Visibility="{Binding SelectedCharacterInfo.CrossReferences.Count, Converter={StaticResource ZeroToVisibilityConverter}}" ItemsSource="{Binding SelectedCharacterInfo.CrossReferences}">
<ItemsControl.ItemTemplate>
<DataTemplate>
<TextBlock TextWrapping="WrapWithOverflow">
UnicodeCharacterInspector/UnicodeCharacterInspector.csproj
+1
View File
@@ -79,6 +79,7 @@
<Compile Include="BindableObject.cs" />
<Compile Include="CharacterInfoViewModel.cs" />
<Compile Include="CharacterInspectorViewModel.cs" />
<Compile Include="CodePointToFontFamilyConverter.cs" />
<Compile Include="Placeholder.cs" />
<Compile Include="PlaceholderAdorner.cs">
<DependentUpon>Placeholder.cs</DependentUpon>
UnicodeInformation.Builder/HttpDataSource.cs
+3 -14
View File
@@ -1,28 +1,17 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode.Builder
{
public class HttpDataSource : IDataSource
{
public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute);
private readonly HttpClient httpClient;
private readonly Uri baseUri;
public HttpDataSource()
: this(UnicodeCharacterDataUri)
public HttpDataSource(Uri baseUri, HttpClient httpClient)
{
}
public HttpDataSource(Uri baseUri)
{
this.httpClient = new HttpClient();
this.httpClient = httpClient ?? new HttpClient();
this.baseUri = baseUri;
}
UnicodeInformation.Builder/Program.cs
+25 -19
View File
@@ -11,6 +11,9 @@ namespace System.Unicode.Builder
{
internal class Program
{
public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute);
public static readonly Uri EmojiDataUri = new Uri("http://www.unicode.org/Public/emoji/latest/", UriKind.Absolute);
public const string UnihanDirectoryName = "Unihan";
public const string UnihanArchiveName = "Unihan.zip";
public const string UcdDirectoryName = "UCD";
@@ -36,6 +39,14 @@ namespace System.Unicode.Builder
"Unihan_IRGSources.txt",
};
public static readonly string[] emojiRequiredFiles = new[]
{
"emoji-data.txt",
"emoji-sequences.txt",
"emoji-variation-sequences.txt",
"emoji-zwj-sequences.txt",
};
private static HttpClient httpClient;
// The only purpose of this is for tests…
@@ -44,14 +55,12 @@ namespace System.Unicode.Builder
httpClient = new HttpClient(handler ?? new HttpClientHandler());
}
private static HttpClient HttpClient { get { return httpClient ?? (httpClient = new HttpClient()); } }
internal static HttpClient HttpClient { get { return httpClient ?? (httpClient = new HttpClient()); } }
private static byte[] DownloadDataArchive(string archiveName)
{
return HttpClient.GetByteArrayAsync(HttpDataSource.UnicodeCharacterDataUri + archiveName).Result;
}
private static Task<byte[]> DownloadDataArchiveAsync(string archiveName)
=> HttpClient.GetByteArrayAsync(UnicodeCharacterDataUri + archiveName);
internal static IDataSource GetDataSource(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
internal static async Task<IDataSource> GetDataSourceAsync(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
{
string baseDirectory = Directory.GetCurrentDirectory();
string dataDirectory = Path.Combine(baseDirectory, UcdDirectoryName);
@@ -94,14 +103,13 @@ namespace System.Unicode.Builder
if (shouldDownload != false)
{
var dataArchiveData = DownloadDataArchive(archiveName);
var dataArchiveData = await DownloadDataArchiveAsync(archiveName).ConfigureAwait(false);
if (shouldSaveArchive == true)
{
var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
try
using (var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
stream.Write(dataArchiveData, 0, dataArchiveData.Length);
await stream.WriteAsync(dataArchiveData, 0, dataArchiveData.Length).ConfigureAwait(false);
dataArchiveData = null; // Release the reference now, since we won't need it anymore.
if (shouldExtract == true)
@@ -118,11 +126,6 @@ namespace System.Unicode.Builder
return new ZipDataSource(stream);
}
}
catch
{
stream.Dispose();
throw;
}
}
else
{
@@ -133,18 +136,21 @@ namespace System.Unicode.Builder
throw new InvalidOperationException();
}
private static void Main(string[] args)
private static async Task MainAsync(string[] args)
{
UnicodeInfoBuilder data;
using (var ucdSource = GetDataSource(UcdArchiveName, UcdDirectoryName, ucdRequiredFiles, null, null, null))
using (var unihanSource = GetDataSource(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null))
using (var ucdSource = await GetDataSourceAsync(UcdArchiveName, UcdDirectoryName, ucdRequiredFiles, null, null, null))
using (var unihanSource = await GetDataSourceAsync(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null))
using (var emojiSource = new HttpDataSource(EmojiDataUri, HttpClient))
{
data = UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource).Result;
data = await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource, emojiSource);
}
using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
data.WriteToStream(stream);
}
private static void Main(string[] args) => MainAsync(args).Wait();
}
}
UnicodeInformation.Builder/UnicodeDataProcessor.cs
+18 -1
View File
@@ -23,6 +23,7 @@ namespace System.Unicode.Builder
public const string UnihanVariantsFileName = "Unihan_Variants.txt";
public const string UnihanNumericValuesFileName = "Unihan_NumericValues.txt";
public const string UnihanIrgSourcesFileName = "Unihan_IRGSources.txt";
public const string EmojiDataFileName = "emoji-data.txt";
private static string ParseSimpleCaseMapping(string mapping)
{
@@ -36,13 +37,14 @@ namespace System.Unicode.Builder
return string.IsNullOrEmpty(s) ? null : s;
}
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource)
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource, IDataSource emojiSource)
{
var builder = new UnicodeInfoBuilder(await ReadUnicodeVersionAsync(ucdSource).ConfigureAwait(false));
await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false);
await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
await ProcessDerivedCorePropertiesFile(ucdSource, builder).ConfigureAwait(false);
await ProcessEmojiDataFile(emojiSource, builder).ConfigureAwait(false);
await ProcessCjkRadicalsFile(ucdSource, builder).ConfigureAwait(false);
await ProcessNameAliasesFile(ucdSource, builder).ConfigureAwait(false);
await ProcessNamesListFile(ucdSource, builder).ConfigureAwait(false);
@@ -219,6 +221,21 @@ namespace System.Unicode.Builder
}
}
private static async Task ProcessEmojiDataFile(IDataSource emojiSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await emojiSource.OpenDataFileAsync(EmojiDataFileName).ConfigureAwait(false), ';'))
{
while (reader.MoveToNextLine())
{
var range = UnicodeCodePointRange.Parse(reader.ReadTrimmedField());
if (EnumHelper<EmojiProperties>.TryGetNamedValue(reader.ReadTrimmedField(), out var property))
{
builder.SetProperties(property, range);
}
}
}
}
private static async Task ProcessCjkRadicalsFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(CjkRadicalsFileName).ConfigureAwait(false), ';'))
UnicodeInformation.Builder/UnicodeInfoBuilder.cs
+32 -1
View File
@@ -226,7 +226,7 @@ namespace System.Unicode.Builder
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException();
throw new InvalidOperationException("Unable to find code point for setting core property.");
}
int i = firstIndex;
@@ -241,6 +241,37 @@ namespace System.Unicode.Builder
}
}
public void SetProperties(EmojiProperties property, UnicodeCodePointRange codePointRange)
{
int firstIndex = FindUcdCodePoint(codePointRange.FirstCodePoint);
int lastIndex = FindUcdCodePoint(codePointRange.LastCodePoint);
if (firstIndex < 0 && lastIndex < 0)
{
Insert(new UnicodeCharacterDataBuilder(codePointRange) { EmojiProperties = property });
return;
}
if (firstIndex < 0
|| lastIndex < 0
|| ucdEntries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| ucdEntries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException("Unable to find code point for setting emoji property.");
}
int i = firstIndex;
while (true)
{
ucdEntries[i].EmojiProperties |= property;
if (i == lastIndex) break;
++i;
}
}
public void SetRadicalInfo(int radicalIndex, CjkRadicalData data)
{
if (radicalIndex < 1 || radicalIndex > CjkRadicalCount) throw new ArgumentOutOfRangeException(nameof(radicalIndex));
UnicodeInformation.Builder/ucd.dat
BIN
View File
Binary file not shown.
UnicodeInformation.Tests/UnicodeInfoBuilderTests.cs
+19 -15
View File
@@ -8,6 +8,7 @@ using System.Text;
using System.Net.Http;
using System.Collections.Generic;
using Xunit;
using System.Linq;
namespace System.Unicode.Tests
{
@@ -65,8 +66,9 @@ namespace System.Unicode.Tests
var handler = new FileHttpResponseHandler();
var ucdTask = RegisterAndDownloadFile(handler, httpCacheDirectory, HttpDataSource.UnicodeCharacterDataUri, Program.UcdArchiveName);
var unihanTask = RegisterAndDownloadFile(handler, httpCacheDirectory, HttpDataSource.UnicodeCharacterDataUri, Program.UnihanArchiveName);
var ucdTask = RegisterAndDownloadFile(handler, httpCacheDirectory, Program.UnicodeCharacterDataUri, Program.UcdArchiveName);
var unihanTask = RegisterAndDownloadFile(handler, httpCacheDirectory, Program.UnicodeCharacterDataUri, Program.UnihanArchiveName);
var emojiTasks = Program.emojiRequiredFiles.Select(f => RegisterAndDownloadFile(handler, httpCacheDirectory, Program.EmojiDataUri, f)).ToArray();
Program.SetHttpMessageHandler(handler);
@@ -75,13 +77,13 @@ namespace System.Unicode.Tests
if (Directory.Exists(Program.UnihanDirectoryName)) Directory.Delete(Program.UnihanDirectoryName, true);
if (File.Exists(Program.UnihanArchiveName)) File.Delete(Program.UnihanArchiveName);
Task.WaitAll(ucdTask, unihanTask);
Task.WaitAll(ucdTask, unihanTask, Task.WhenAll(emojiTasks));
}
[Fact]
public void DownloadUcdArchive()
{
using (var source = Program.GetDataSource(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, false, false))
using (var source = Program.GetDataSourceAsync(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, false, false).Result)
{
}
}
@@ -89,7 +91,7 @@ namespace System.Unicode.Tests
[Fact]
public void DownloadAndSaveUcdArchive()
{
using (var source = Program.GetDataSource(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, true, false))
using (var source = Program.GetDataSourceAsync(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, true, false).Result)
{
}
}
@@ -97,7 +99,7 @@ namespace System.Unicode.Tests
[Fact]
public void ExtractUcdArchive()
{
using (var source = Program.GetDataSource(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, true, true))
using (var source = Program.GetDataSourceAsync(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, true, true, true).Result)
{
}
}
@@ -105,7 +107,7 @@ namespace System.Unicode.Tests
[Fact]
public void DownloadUnihanArchive()
{
using (var source = Program.GetDataSource(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, false, false))
using (var source = Program.GetDataSourceAsync(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, false, false).Result)
{
}
}
@@ -113,7 +115,7 @@ namespace System.Unicode.Tests
[Fact]
public void DownloadAndSaveUnihanArchive()
{
using (var source = Program.GetDataSource(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, true, false))
using (var source = Program.GetDataSourceAsync(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, true, false).Result)
{
}
}
@@ -121,7 +123,7 @@ namespace System.Unicode.Tests
[Fact]
public void ExtractUnihanArchive()
{
using (var source = Program.GetDataSource(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, true, true))
using (var source = Program.GetDataSourceAsync(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, true, true, true).Result)
{
}
}
@@ -129,10 +131,11 @@ namespace System.Unicode.Tests
[Fact]
public async Task BuildDataAsync()
{
using (var ucdSource = Program.GetDataSource(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var unihanSource = Program.GetDataSource(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var ucdSource = await Program.GetDataSourceAsync(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var unihanSource = await Program.GetDataSourceAsync(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var emojiSource = new HttpDataSource(Program.EmojiDataUri, Program.HttpClient))
{
var data = (await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource));
var data = (await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource, emojiSource));
Assert.Equal((int)'\t', data.GetUcd('\t').CodePointRange.FirstCodePoint);
}
@@ -141,10 +144,11 @@ namespace System.Unicode.Tests
[Fact]
public async Task BuildAndWriteDataAsync()
{
using (var ucdSource = Program.GetDataSource(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var unihanSource = Program.GetDataSource(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var ucdSource = await Program.GetDataSourceAsync(Program.UcdArchiveName, Program.UcdDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var unihanSource = await Program.GetDataSourceAsync(Program.UnihanArchiveName, Program.UnihanDirectoryName, Program.ucdRequiredFiles, null, null, null))
using (var emojiSource = new HttpDataSource(Program.EmojiDataUri, Program.HttpClient))
{
var data = (await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource));
var data = (await UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource, emojiSource));
//using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
using (var stream = File.Create("ucd.dat"))
UnicodeInformation/EmojiProperties.cs
+4 -7
View File
@@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Text;
using System.ComponentModel.DataAnnotations;
namespace System.Unicode
{
@@ -16,13 +13,13 @@ namespace System.Unicode
/// <summary>Represents the Emoji property.</summary>
[ValueName("Emoji"), Display(Name = "Emoji")]
Emoji = 0x1,
/// <summary>Represents the Emoji property.</summary>
/// <summary>Represents the Emoji_Presentation property.</summary>
[ValueName("Emoji_Presentation"), Display(Name = "Emoji_Presentation")]
EmojiPresentation = 0x2,
/// <summary>Represents the Emoji property.</summary>
/// <summary>Represents the Emoji_Modifier_Base property.</summary>
[ValueName("Emoji_Modifier_Base"), Display(Name = "Emoji_Modifier_Base")]
EmojiModifierBase = 0x4,
/// <summary>Represents the Emoji property.</summary>
/// <summary>Represents the Emoji_Component property.</summary>
[ValueName("Emoji_Component"), Display(Name = "Emoji_Component")]
EmojiComponent = 0x8,
}
UnicodeInformation/UnicodeCharInfo.cs
+3
View File
@@ -123,6 +123,9 @@ namespace System.Unicode
/// <summary>Gets a value indicating which of the boolean core properties are defined for the code point.</summary>
/// <remarks>The core properties are computed by combining various character information together with contributory properties.</remarks>
public CoreProperties CoreProperties => UnicodeCharacterData.CoreProperties;
/// <summary>Gets a value indicating which of the boolean emoji properties are defined for the code point.</summary>
/// <remarks>The emoji properties are provided by the Unicode Emoji standard, which is not directly part of UCD.</remarks>
public EmojiProperties EmojiProperties => UnicodeCharacterData.EmojiProperties;
/// <summary>Gets a collection of cross references associated with the code point.</summary>
/// <remarks>The cross references have been extracted from Unicode data but are not normative.</remarks>
public UnicodeCrossReferenceCollection CrossRerefences => new UnicodeCrossReferenceCollection(UnicodeCharacterData.CrossRerefences);
UnicodeInformation/UnicodeInfo.cs
+9 -11
View File
@@ -1,12 +1,8 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode
{
@@ -82,14 +78,14 @@ namespace System.Unicode
for (i = 0; i < blockEntries.Length; ++i)
{
blockEntries[i] = ReadBlockEntry(reader);
ReadBlockEntry(reader, out blockEntries[i]);
}
var cjkRadicalEntries = new CjkRadicalData[reader.ReadByte()];
for (i = 0; i < cjkRadicalEntries.Length; ++i)
{
cjkRadicalEntries[i] = ReadCjkRadicalInfo(reader);
ReadCjkRadicalInfo(reader, out cjkRadicalEntries[i]);
}
var unihanCharacterDataEntries = new UnihanCharacterData[ReadCodePoint(reader)];
@@ -247,7 +243,7 @@ namespace System.Unicode
);
}
private static CjkRadicalData ReadCjkRadicalInfo(BinaryReader reader)
private static void ReadCjkRadicalInfo(BinaryReader reader, out CjkRadicalData value)
{
char tr;
char tc;
@@ -255,11 +251,13 @@ namespace System.Unicode
tr = (char)reader.ReadUInt16();
tc = (char)reader.ReadUInt16();
return (tr & 0x8000) == 0 ? new CjkRadicalData(tr, tc) : new CjkRadicalData((char)(tr & 0x7FFF), tc, (char)reader.ReadUInt16(), (char)reader.ReadUInt16());
value = (tr & 0x8000) == 0 ?
new CjkRadicalData(tr, tc) :
new CjkRadicalData((char)(tr & 0x7FFF), tc, (char)reader.ReadUInt16(), (char)reader.ReadUInt16());
}
private static UnicodeBlock ReadBlockEntry(BinaryReader reader)
=> new UnicodeBlock(new UnicodeCodePointRange(ReadCodePoint(reader), ReadCodePoint(reader)), reader.ReadString());
private static void ReadBlockEntry(BinaryReader reader, out UnicodeBlock value)
=> value = new UnicodeBlock(new UnicodeCodePointRange(ReadCodePoint(reader), ReadCodePoint(reader)), reader.ReadString());
private static int ReadInt24(BinaryReader reader) => reader.ReadByte() | ((reader.ReadByte() | (reader.ReadByte() << 8)) << 8);
UnicodeInformation/ucd.dat
BIN
View File
Binary file not shown.