swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preparing to integrate parts of Unihan.

Browse Source
This commit is contained in:
GoldenCrystal
2014-11-15 00:58:03 +01:00
parent d7ada9e413
commit 533c9e6d1b
15 changed files with 213 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeCharacterInspector/App.config
+3 -3
View File
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/>
</startup>
</configuration>
</configuration>
UnicodeCharacterInspector/App.manifest
+33
View File
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?>
<asmv1:assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1" xmlns:asmv1="urn:schemas-microsoft-com:asm.v1" xmlns:asmv2="urn:schemas-microsoft-com:asm.v2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<assemblyIdentity version="1.0.0.0" name="UnicodeCharacterInspector"/>
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
<security>
<requestedPrivileges xmlns="urn:schemas-microsoft-com:asm.v3">
<requestedExecutionLevel level="asInvoker" uiAccess="false" />
</requestedPrivileges>
</security>
</trustInfo>
<compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
<application>
<supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
<supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
<supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
<supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
</application>
</compatibility>
<dependency>
<dependentAssembly>
<assemblyIdentity
type="win32"
name="Microsoft.Windows.Common-Controls"
version="6.0.0.0"
processorArchitecture="*"
publicKeyToken="6595b64144ccf1df"
language="*"
/>
</dependentAssembly>
</dependency>
</asmv1:assembly>
UnicodeCharacterInspector/UnicodeCharacterInspector.csproj
+5
View File
@@ -14,6 +14,7 @@
<ProjectTypeGuids>{60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<WarningLevel>4</WarningLevel>
<NuGetPackageImportStamp>f1bfc594</NuGetPackageImportStamp>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
@@ -34,6 +35,9 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup>
<ApplicationManifest>App.manifest</ApplicationManifest>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Data" />
@@ -80,6 +84,7 @@
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="App.manifest" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\UnicodeInformation\UnicodeInformation.csproj">
UnicodeInformation.Builder/FileUcdSource.cs → UnicodeInformation.Builder/FileDataSource.cs
+2 -2
View File
@@ -8,11 +8,11 @@ using System.Unicode;
namespace System.Unicode.Builder
{
public sealed class FileUcdSource : IUcdSource
public sealed class FileDataSource : IDataSource
{
private readonly string baseDirectory;
public FileUcdSource(string baseDirectory)
public FileDataSource(string baseDirectory)
{
this.baseDirectory = Path.GetFullPath(baseDirectory);
}
UnicodeInformation.Builder/HttpUcdSource.cs → UnicodeInformation.Builder/HttpDataSource.cs
+3 -3
View File
@@ -8,19 +8,19 @@ using System.Threading.Tasks;
namespace System.Unicode.Builder
{
public class HttpUcdSource : IUcdSource
public class HttpDataSource : IDataSource
{
public static readonly Uri UnicodeCharacterDataUri = new Uri("http://www.unicode.org/Public/UCD/latest/ucd/", UriKind.Absolute);
private readonly HttpClient httpClient;
private readonly Uri baseUri;
public HttpUcdSource()
public HttpDataSource()
: this(UnicodeCharacterDataUri)
{
}
public HttpUcdSource(Uri baseUri)
public HttpDataSource(Uri baseUri)
{
this.httpClient = new HttpClient();
this.baseUri = baseUri;
UnicodeInformation.Builder/IUcdSource.cs → UnicodeInformation.Builder/IDataSource.cs
+1 -1
View File
@@ -7,7 +7,7 @@ using System.Threading.Tasks;
namespace System.Unicode.Builder
{
public interface IUcdSource : IDisposable
public interface IDataSource : IDisposable
{
Task<Stream> OpenDataFileAsync(string fileName);
}
UnicodeInformation.Builder/Program.cs
+33 -25
View File
@@ -13,7 +13,7 @@ namespace System.Unicode.Builder
{
public const string UnihanDirectoryName = "Unihan";
public const string UnihanArchiveName = "Unihan.zip";
public const string UcdDirectoryName = "UCD";
public const string directoryName = "UCD";
public const string UcdArchiveName = "UCD.zip";
public static readonly string[] ucdRequiredFiles = new[]
@@ -22,32 +22,39 @@ namespace System.Unicode.Builder
"PropList.txt",
"DerivedCoreProperties.txt",
"Blocks.txt",
"Jamo.txt"
"Jamo.txt",
};
private static byte[] DownloadUcdArchive()
public static readonly string[] unihanRequiredFiles = new[]
{
"Unihan_NumericValues.txt",
"Unihan_Readings.txt",
"Unihan_Variants.txt",
};
private static byte[] DownloadDataArchive(string archiveName)
{
using (var httpClient = new HttpClient())
{
return httpClient.GetByteArrayAsync(HttpUcdSource.UnicodeCharacterDataUri + UcdArchiveName).Result;
return httpClient.GetByteArrayAsync(HttpDataSource.UnicodeCharacterDataUri + archiveName).Result;
}
}
internal static IUcdSource GetUcdSource(bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
internal static IDataSource GetDataSource(string archiveName, string directoryName, string[] requiredFiles, bool? shouldDownload, bool? shouldSaveArchive, bool? shouldExtract)
{
string baseDirectory = Environment.CurrentDirectory;
string ucdDirectory = Path.Combine(baseDirectory, UcdDirectoryName);
string ucdArchiveFileName = Path.Combine(baseDirectory, UcdArchiveName);
string dataDirectory = Path.Combine(baseDirectory, Program.directoryName);
string dataArchiveFileName = Path.Combine(baseDirectory, archiveName);
if (shouldDownload != true)
{
bool hasValidDirectory = Directory.Exists(ucdDirectory);
bool hasValidDirectory = Directory.Exists(dataDirectory);
if (hasValidDirectory)
{
foreach (string ucdRequiredFile in ucdRequiredFiles)
foreach (string requiredFile in requiredFiles)
{
if (!File.Exists(Path.Combine(ucdDirectory, ucdRequiredFile)))
if (!File.Exists(Path.Combine(dataDirectory, requiredFile)))
{
hasValidDirectory = false;
break;
@@ -57,47 +64,47 @@ namespace System.Unicode.Builder
if (hasValidDirectory)
{
return new FileUcdSource(ucdDirectory);
return new FileDataSource(dataDirectory);
}
if (File.Exists(ucdArchiveFileName))
if (File.Exists(dataArchiveFileName))
{
if (shouldExtract == true)
{
ZipFile.ExtractToDirectory(ucdArchiveFileName, ucdDirectory);
return new FileUcdSource(ucdDirectory);
ZipFile.ExtractToDirectory(dataArchiveFileName, dataDirectory);
return new FileDataSource(dataDirectory);
}
else
{
return new ZipUcdSource(File.OpenRead(ucdArchiveFileName));
return new ZipDataSource(File.OpenRead(dataArchiveFileName));
}
}
}
if (shouldDownload != false)
{
var ucdArchiveData = DownloadUcdArchive();
var dataArchiveData = DownloadDataArchive(archiveName);
if (shouldSaveArchive == true)
{
var stream = File.Open(ucdArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
var stream = File.Open(dataArchiveFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
try
{
stream.Write(ucdArchiveData, 0, ucdArchiveData.Length);
ucdArchiveData = null; // Release the reference now, since we won't need it anymore.
stream.Write(dataArchiveData, 0, dataArchiveData.Length);
dataArchiveData = null; // Release the reference now, since we won't need it anymore.
if (shouldExtract == true)
{
using (var archive = new ZipArchive(stream, ZipArchiveMode.Read, false))
{
archive.ExtractToDirectory(ucdDirectory);
archive.ExtractToDirectory(dataDirectory);
return new FileUcdSource(ucdDirectory);
return new FileDataSource(dataDirectory);
}
}
else
{
return new ZipUcdSource(stream);
return new ZipDataSource(stream);
}
}
catch
@@ -108,7 +115,7 @@ namespace System.Unicode.Builder
}
else
{
return new ZipUcdSource(new MemoryStream(ucdArchiveData));
return new ZipDataSource(new MemoryStream(dataArchiveData));
}
}
@@ -119,9 +126,10 @@ namespace System.Unicode.Builder
{
UnicodeInfoBuilder data;
using (var ucdSource = GetUcdSource(null, null, null))
using (var ucdSource = GetDataSource(UcdArchiveName, directoryName, ucdRequiredFiles, null, null, null))
using (var unihanSource = GetDataSource(UnihanArchiveName, UnihanDirectoryName, unihanRequiredFiles, null, null, null))
{
data = UnicodeDataProcessor.BuildDataAsync(ucdSource).Result;
data = UnicodeDataProcessor.BuildDataAsync(ucdSource, unihanSource).Result;
}
using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
UnicodeInformation.Builder/UnicodeDataProcessor.cs
+5 -5
View File
@@ -27,7 +27,7 @@ namespace System.Unicode.Builder
return string.IsNullOrEmpty(s) ? null : s;
}
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IUcdSource ucdSource)
public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSource, IDataSource unihanSource)
{
var builder = new UnicodeInfoBuilder(new Version(7, 0));
@@ -39,7 +39,7 @@ namespace System.Unicode.Builder
return builder;
}
private static async Task ProcessUnicodeDataFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
private static async Task ProcessUnicodeDataFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false)))
{
@@ -156,7 +156,7 @@ namespace System.Unicode.Builder
}
}
private static async Task ProcessPropListFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
private static async Task ProcessPropListFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(PropListFileName).ConfigureAwait(false)))
{
@@ -173,7 +173,7 @@ namespace System.Unicode.Builder
}
}
private static async Task ProcessDerivedCorePropertiesFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
private static async Task ProcessDerivedCorePropertiesFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(DerivedCorePropertiesFileName).ConfigureAwait(false)))
{
@@ -190,7 +190,7 @@ namespace System.Unicode.Builder
}
}
private static async Task ProcessBlocksFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
private static async Task ProcessBlocksFile(IDataSource ucdSource, UnicodeInfoBuilder builder)
{
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false)))
{
UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
+4 -4
View File
@@ -54,16 +54,16 @@
<ItemGroup>
<Compile Include="BinaryWriterExtensions.cs" />
<Compile Include="CharacterDecompositionMapping.cs" />
<Compile Include="FileUcdSource.cs" />
<Compile Include="HttpUcdSource.cs" />
<Compile Include="IUcdSource.cs" />
<Compile Include="FileDataSource.cs" />
<Compile Include="HttpDataSource.cs" />
<Compile Include="IDataSource.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="UnicodeCharacterDataBuilder.cs" />
<Compile Include="UnicodeInfoBuilder.cs" />
<Compile Include="UnicodeDataFileReader.cs" />
<Compile Include="UnicodeDataProcessor.cs" />
<Compile Include="ZipUcdSource.cs" />
<Compile Include="ZipDataSource.cs" />
</ItemGroup>
<ItemGroup>
<None Include="..\System.Unicode.snk">
UnicodeInformation.Builder/ZipUcdSource.cs → UnicodeInformation.Builder/ZipDataSource.cs
+2 -2
View File
@@ -8,11 +8,11 @@ using System.Threading.Tasks;
namespace System.Unicode.Builder
{
public sealed class ZipUcdSource : IUcdSource
public sealed class ZipDataSource : IDataSource
{
private readonly ZipArchive archive;
public ZipUcdSource(Stream stream)
public ZipDataSource(Stream stream)
{
archive = new ZipArchive(stream, ZipArchiveMode.Read, false);
}
UnicodeInformation.Tests/UnicodeDataManagerTests.cs
+15 -8
View File
@@ -23,12 +23,19 @@ namespace System.Unicode.Tests
{
Directory.CreateDirectory(directoryName);
var fileName = Path.Combine(directoryName, "UCD.zip");
string ucdFileName = Path.Combine(directoryName, "UCD.zip");
string unihanFileName = Path.Combine(directoryName, "Unihan.zip");
if (!File.Exists(fileName))
if (!File.Exists(ucdFileName))
{
new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/UCD.zip", fileName);
ZipFile.ExtractToDirectory(fileName, directoryName);
new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/UCD.zip", ucdFileName);
ZipFile.ExtractToDirectory(ucdFileName, directoryName);
}
if (!File.Exists(unihanFileName))
{
new WebClient().DownloadFile("http://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip", unihanFileName);
ZipFile.ExtractToDirectory(unihanFileName, directoryName);
}
}
}
@@ -36,9 +43,9 @@ namespace System.Unicode.Tests
[TestMethod]
public async Task BuildDataAsync()
{
var source = new FileUcdSource(UcdDirectoryName);
var source = new FileDataSource(UcdDirectoryName);
var data = (await UnicodeDataProcessor.BuildDataAsync(source)).ToUnicodeData();
var data = (await UnicodeDataProcessor.BuildDataAsync(source, source)).ToUnicodeData();
Assert.AreEqual((int)'\t', data.GetCharInfo('\t').CodePoint);
}
@@ -46,9 +53,9 @@ namespace System.Unicode.Tests
[TestMethod]
public async Task BuildAndWriteDataAsync()
{
var source = new FileUcdSource(UcdDirectoryName);
var source = new FileDataSource(UcdDirectoryName);
var data = await UnicodeDataProcessor.BuildDataAsync(source);
var data = await UnicodeDataProcessor.BuildDataAsync(source, source);
using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
{
UnicodeInformation/UnicodeInformation.csproj
+3
View File
@@ -76,7 +76,10 @@
<Compile Include="UnicodeCharacterRange.cs" />
<Compile Include="UnicodeCharInfo.cs" />
<Compile Include="UnicodeInfo.cs" />
<Compile Include="UnihanNumericType.cs" />
<Compile Include="UnicodeNumericType.cs" />
<Compile Include="UnihanCharacterData.cs" />
<Compile Include="UnihanFields.cs" />
<Compile Include="ValueNameAttribute.cs" />
<Compile Include="UnicodeRationalNumber.cs" />
</ItemGroup>
UnicodeInformation/UnihanCharacterData.cs
+55
View File
@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode
{
internal sealed class UnihanCharacterData
{
public readonly int CodePoint;
public readonly UnihanNumericType NumericType;
public readonly ulong NumericValue;
public readonly string Definition;
public readonly string MandarinReading;
public readonly string CantoneseReading;
public readonly string JapaneseKunReading;
public readonly string JapaneseOnReading;
public readonly string KoreanReading;
public readonly string HangulReading;
public readonly string VietnameseReading;
public readonly string SimplifiedVariant;
public readonly string TraditionalVariant;
internal static int PackCodePoint(int codePoint)
{
if (codePoint >= 0x3400)
{
if (codePoint < 0x4E00) return codePoint + 0x1E00;
else if (codePoint < 0xA000) return codePoint - 0x4E00;
else if (codePoint >= 0xF900 && codePoint < 0xFB00) return codePoint + 0xFD00;
else if (codePoint >= 0x20000)
{
if (codePoint < 0x2F800) return codePoint - 0x19400;
else if (codePoint < 0x30000) return codePoint - 0x10000;
}
}
throw new ArgumentOutOfRangeException("codePoint");
}
internal static int UnpackCodePoint(int packedCodePoint)
{
if (packedCodePoint >= 0)
{
if (packedCodePoint < 0x05200) return packedCodePoint + 0x4E00;
else if (packedCodePoint < 0x06C00) return packedCodePoint - 0x1E00;
else if (packedCodePoint < 0x1F600) return packedCodePoint + 0x19400;
else if (packedCodePoint < 0x1F800) return packedCodePoint - 0xFD00;
else if (packedCodePoint < 0x20000) return packedCodePoint + 0x10000;
}
throw new ArgumentOutOfRangeException("packedCodePoint");
}
}
}
UnicodeInformation/UnihanFields.cs
+29
View File
@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode
{
[Flags]
internal enum UnihanFields : ushort
{
// NumericType / NumericValue : Not exactly a bit mask here…
AccountingNumeric = 1,
OtherNumeric = 2,
PrimaryNumeric = 3,
Definition = 4,
MandarinReading = 8,
CantoneseReading = 16,
JapaneseKunReading = 32,
JapaneseOnReading = 64,
KoreanReading = 128,
HangulReading = 256,
VietnameseReading = 512,
SimplifiedVariant = 1024,
TraditionalVariant = 2048,
}
}
UnicodeInformation/UnihanNumericType.cs
+20
View File
@@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode
{
/// <summary>Represents the different numeric types from the Unihan database.</summary>
public enum UnihanNumericType : byte
{
None = 0,
[ValueName("kAccountingNumeric")]
Accounting = 1,
[ValueName("kOtherNumeric")]
Other = 2,
[ValueName("kPrimaryNumeric")]
Primary = 3
}
}