swcs/NetUnicodeInfo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added the code to save and load data in custom binary format.

Browse Source
This commit is contained in:
GoldenCrystal
2014-11-03 03:44:41 +01:00
parent de36586c64
commit ed1c5d7913
10 changed files with 285 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UnicodeInformation.Builder/BinaryWriterExtensions.cs
+48
View File
@@ -0,0 +1,48 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode.Builder
{
internal static class BinaryWriterExtensions
{
//public static void WriteUInt24(this BinaryWriter writer, int value)
//{
// if (value < 0 || value > 0xFFFFFF) throw new ArgumentOutOfRangeException("value");
// writer.Write((byte)(value));
// writer.Write((byte)(value >> 8));
// writer.Write((byte)(value >> 16));
//}
/// <summary>Writes code point in a custom, but compact encoding.</summary>
/// <remarks>
/// Unlike UTF-8, this encoding will consume at most 3 bytes.
/// It could ideally store values between 0x0 and 0x40407F, but this range is useless at the moment.
/// </remarks>
/// <param name="writer">The binary writer to use.</param>
/// <param name="value">The value to write</param>
public static void WriteCodePoint(this BinaryWriter writer, int value)
{
if (value < 0 || value > 0x40407F) throw new ArgumentOutOfRangeException("value");
if (value < 0x80) writer.Write((byte)value);
else if (value < 0x4080)
{
value -= 0x80;
writer.Write((byte)((byte)(value >> 8) | 0x80));
writer.Write((byte)value);
}
else
{
value -= 0x4080;
writer.Write((byte)((byte)(value >> 16) | 0xC0));
writer.Write((byte)(value >> 8));
writer.Write((byte)value);
}
}
}
}
UnicodeInformation.Builder/UnicodeCharacterDataBuilder.cs
+42
View File
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@@ -147,5 +148,46 @@ namespace System.Unicode.Builder
relatedCodePoints.Count > 0 ? relatedCodePoints.ToArray() : null
);
}
internal void WriteToFile(BinaryWriter writer)
{
UcdFields fields = default(UcdFields);
if (!codePointRange.IsSingleCodePoint) fields = UcdFields.CodePointRange;
if (name != null) fields |= UcdFields.Name;
if (category != UnicodeCategory.OtherNotAssigned) fields |= UcdFields.Category;
if (canonicalCombiningClass != CanonicalCombiningClass.NotReordered) fields |= UcdFields.CanonicalCombiningClass;
/*if (bidirectionalClass != 0)*/fields |= UcdFields.BidirectionalClass;
if (decompositionType != null) fields |= UcdFields.DecompositionType;
fields |= (UcdFields)((int)numericType << 6);
if (bidirectionalMirrored) fields |= UcdFields.BidirectionalMirrored;
if (oldName != null) fields |= UcdFields.OldName;
if (simpleUpperCaseMapping != null) fields |= UcdFields.SimpleUpperCaseMapping;
if (simpleLowerCaseMapping != null) fields |= UcdFields.SimpleLowerCaseMapping;
if (simpleTitleCaseMapping != null) fields |= UcdFields.SimpleTitleCaseMapping;
if (contributoryProperties != 0) fields |= UcdFields.ContributoryProperties;
writer.Write((ushort)fields);
writer.WriteCodePoint(codePointRange.FirstCodePoint);
if ((fields & UcdFields.CodePointRange) != 0) writer.WriteCodePoint(CodePointRange.LastCodePoint);
if ((fields & UcdFields.Name) != 0) writer.Write(name);
if ((fields & UcdFields.Category) != 0) writer.Write((byte)category);
if ((fields & UcdFields.CanonicalCombiningClass) != 0) writer.Write((byte)canonicalCombiningClass);
if ((fields & UcdFields.BidirectionalClass) != 0) writer.Write((byte)bidirectionalClass);
if ((fields & UcdFields.DecompositionType) != 0) writer.Write(decompositionType);
if ((fields & UcdFields.NumericNumeric) != 0)
{
writer.Write(numericValue.Numerator);
writer.Write(numericValue.Denominator);
}
if ((fields & UcdFields.OldName) != 0) writer.Write(oldName);
if ((fields & UcdFields.SimpleUpperCaseMapping) != 0) writer.Write(simpleUpperCaseMapping);
if ((fields & UcdFields.SimpleLowerCaseMapping) != 0) writer.Write(simpleLowerCaseMapping);
if ((fields & UcdFields.SimpleTitleCaseMapping) != 0) writer.Write(simpleTitleCaseMapping);
if ((fields & UcdFields.ContributoryProperties) != 0) writer.Write((int)contributoryProperties);
}
}
}
UnicodeInformation.Builder/UnicodeDataBuilder.cs
+46 -22
View File
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@@ -9,8 +10,8 @@ namespace System.Unicode.Builder
public class UnicodeDataBuilder
{
private readonly Version unicodeVersion;
private UnicodeCharacterDataBuilder[] characterDataBuilders = new UnicodeCharacterDataBuilder[10000];
private int characterCount;
private UnicodeCharacterDataBuilder[] entries = new UnicodeCharacterDataBuilder[10000];
private int entryCount;
public UnicodeDataBuilder(Version unicodeVersion)
{
@@ -19,16 +20,16 @@ namespace System.Unicode.Builder
private int FindCodePoint(int codePoint)
{
if (characterCount == 0) return -1;
if (entryCount == 0) return -1;
int minIndex = 0;
int maxIndex = characterCount - 1;
int maxIndex = entryCount - 1;
do
{
int index = (minIndex + maxIndex) >> 1;
int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(codePoint);
int Δ = entries[index].CodePointRange.CompareCodePoint(codePoint);
if (Δ == 0) return index;
else if (Δ < 0) maxIndex = index - 1;
@@ -43,22 +44,22 @@ namespace System.Unicode.Builder
int minIndex;
int maxIndex;
if (characterCount == 0 || characterDataBuilders[maxIndex = characterCount - 1].CodePointRange.LastCodePoint < startCodePoint) return characterCount;
else if (endCodePoint < characterDataBuilders[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
else if (characterCount == 1) return -1;
if (entryCount == 0 || entries[maxIndex = entryCount - 1].CodePointRange.LastCodePoint < startCodePoint) return entryCount;
else if (endCodePoint < entries[minIndex = 0].CodePointRange.FirstCodePoint) return 0;
else if (entryCount == 1) return -1;
do
{
int index = (minIndex + maxIndex) >> 1;
int Δ = characterDataBuilders[index].CodePointRange.CompareCodePoint(startCodePoint);
int Δ = entries[index].CodePointRange.CompareCodePoint(startCodePoint);
if (Δ == 0) return -1;
else if (Δ < 0) maxIndex = index;
else minIndex = index;
} while (maxIndex - minIndex > 1);
if (characterDataBuilders[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
if (entries[maxIndex].CodePointRange.FirstCodePoint < endCodePoint) return -1;
else return maxIndex;
}
@@ -68,25 +69,25 @@ namespace System.Unicode.Builder
if (insertionPoint < 0) throw new InvalidOperationException("The specified range overlaps with pre-existing ranges.");
if (insertionPoint >= characterDataBuilders.Length)
if (insertionPoint >= entries.Length)
{
Array.Resize(ref characterDataBuilders, characterDataBuilders.Length << 1);
Array.Resize(ref entries, entries.Length << 1);
}
if (insertionPoint < characterCount)
if (insertionPoint < entryCount)
{
Array.Copy(characterDataBuilders, insertionPoint, characterDataBuilders, insertionPoint + 1, characterCount - insertionPoint);
Array.Copy(entries, insertionPoint, entries, insertionPoint + 1, entryCount - insertionPoint);
}
characterDataBuilders[insertionPoint] = data;
++characterCount;
entries[insertionPoint] = data;
++entryCount;
}
public UnicodeCharacterDataBuilder Get(int codePoint)
{
int index = FindCodePoint(codePoint);
return index >= 0 ? characterDataBuilders[index] : null;
return index >= 0 ? entries[index] : null;
}
public void SetProperties(ContributoryProperties property, UnicodeCharacterRange codePointRange)
@@ -102,8 +103,8 @@ namespace System.Unicode.Builder
if (firstIndex < 0
|| lastIndex < 0
|| characterDataBuilders[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| characterDataBuilders[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
|| entries[firstIndex].CodePointRange.FirstCodePoint < codePointRange.FirstCodePoint
|| entries[lastIndex].CodePointRange.LastCodePoint > codePointRange.LastCodePoint)
{
throw new InvalidOperationException();
}
@@ -112,7 +113,7 @@ namespace System.Unicode.Builder
while (true)
{
characterDataBuilders[i].ContributoryProperties |= property;
entries[i].ContributoryProperties |= property;
if (i == lastIndex) break;
@@ -122,12 +123,35 @@ namespace System.Unicode.Builder
public UnicodeData ToUnicodeData()
{
var finalData = new UnicodeCharacterData[characterCount];
var finalData = new UnicodeCharacterData[entryCount];
for (int i = 0; i < finalData.Length; ++i)
finalData[i] = characterDataBuilders[i].ToCharacterData();
finalData[i] = entries[i].ToCharacterData();
return new UnicodeData(unicodeVersion, finalData);
}
public void WriteToStream(Stream stream)
{
using (var writer = new BinaryWriter(stream, Encoding.UTF8, true))
{
writer.Write(new byte[] { (byte)'U', (byte)'C', (byte)'D', 0 });
writer.Write((ushort)7);
writer.Write((byte)0);
writer.WriteCodePoint(entryCount);
for (int i = 0; i < entryCount; ++i)
{
entries[i].WriteToFile(writer);
}
}
}
public void WriteToFile(string fileName)
{
using (var stream = File.Create(fileName))
{
WriteToStream(stream);
}
}
}
}
UnicodeInformation.Builder/UnicodeDataManager.cs
+2 -2
View File
@@ -25,7 +25,7 @@ namespace System.Unicode.Builder
return string.IsNullOrEmpty(s) ? null : s;
}
public static async Task<UnicodeData> DownloadAndBuildDataAsync(IUcdSource ucdSource)
public static async Task<UnicodeDataBuilder> BuildDataAsync(IUcdSource ucdSource)
{
var builder = new UnicodeDataBuilder(new Version(7, 0));
@@ -153,7 +153,7 @@ namespace System.Unicode.Builder
}
}
return builder.ToUnicodeData();
return builder;
}
}
}
UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
+1
View File
@@ -48,6 +48,7 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BinaryWriterExtensions.cs" />
<Compile Include="FileUcdSource.cs" />
<Compile Include="HttpUcdSource.cs" />
<Compile Include="IUcdSource.cs" />
UnicodeInformation.Tests/UnicodeDataManagerTests.cs
+22 -2
View File
@@ -33,13 +33,33 @@ namespace System.Unicode.Tests
}
[TestMethod]
public async Task DownloadAndBuildDataAsync()
public async Task BuildDataAsync()
{
var source = new FileUcdSource(UcdDirectoryName);
var data = await UnicodeDataManager.DownloadAndBuildDataAsync(source);
var data = (await UnicodeDataManager.BuildDataAsync(source)).ToUnicodeData();
Assert.AreEqual((int)'\t', data.GetUnicodeData('\t').CodePointRange.FirstCodePoint);
}
[TestMethod]
public async Task BuildAndWriteDataAsync()
{
var source = new FileUcdSource(UcdDirectoryName);
var data = await UnicodeDataManager.BuildDataAsync(source);
using (var stream = new DeflateStream(File.Create("ucd.dat"), CompressionLevel.Optimal, false))
{
data.WriteToStream(stream);
}
using (var stream = new DeflateStream(File.OpenRead("ucd.dat"), CompressionMode.Decompress, false))
{
var readData = UnicodeData.FromStream(stream);
Assert.AreEqual((int)'\t', data.Get('\t').CodePointRange.FirstCodePoint);
}
}
}
}
UnicodeInformation/UcdFields.cs
+37
View File
@@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace System.Unicode
{
/// <summary>Represents the fields available for an UCD entry.</summary>
/// <remarks>Not all the enumeration member directly map to a field.</remarks>
[Flags]
internal enum UcdFields : ushort
{
// Not really a field, just here to indicate that the entry is a range
CodePointRange = 1,
Name = 2,
Category = 4,
CanonicalCombiningClass = 8,
BidirectionalClass = 16,
DecompositionType = 32,
// NumericType / NumericValue : Not exactly a bit mask here… More like [0…3] << 6
NumericDecimal = 64,
NumericDigit = 128,
NumericNumeric = 192,
// This is a yes/no field, so obviously, no extra storage is required for this one…
BidirectionalMirrored = 256,
OldName = 512,
SimpleUpperCaseMapping = 1024,
SimpleLowerCaseMapping = 2048,
SimpleTitleCaseMapping = 4096,
ContributoryProperties = 8192,
}
}
UnicodeInformation/UnicodeCharacterData.cs
+5 -7
View File
@@ -10,13 +10,13 @@ namespace System.Unicode
public sealed class UnicodeCharacterData
{
public readonly UnicodeCharacterRange CodePointRange;
private readonly string name;
public readonly string Name;
public readonly UnicodeCategory Category;
public readonly CanonicalCombiningClass CanonicalCombiningClass;
public readonly BidirectionalClass BidirectionalClass;
public readonly string DecompositionType;
public readonly UnicodeNumericType NumericType;
public readonly UnicodeRationalNumber NumericValue;
private readonly UnicodeRationalNumber numericValue;
public readonly bool BidirectionalMirrored;
public readonly string OldName;
public readonly string SimpleUpperCaseMapping;
@@ -46,13 +46,13 @@ namespace System.Unicode
)
{
this.CodePointRange = codePointRange;
this.name = name;
this.Name = name;
this.Category = category;
this.CanonicalCombiningClass = canonicalCombiningClass;
this.BidirectionalClass = bidirectionalClass;
this.DecompositionType = decompositionType;
this.NumericType = numericType;
this.NumericValue = numericValue;
this.numericValue = numericValue;
this.BidirectionalMirrored = bidirectionalMirrored;
this.OldName = oldName;
this.SimpleUpperCaseMapping = simpleUpperCaseMapping;
@@ -62,8 +62,6 @@ namespace System.Unicode
this.RelatedCodePoints = relatedCodePoints;
}
public bool IsRange { get { return CodePointRange.FirstCodePoint != CodePointRange.LastCodePoint; } }
public string Name { get { return !IsRange ? name : null; } }
public UnicodeRationalNumber? NumericValue { get { return NumericType != UnicodeNumericType.None ? numericValue : null as UnicodeRationalNumber?; } }
}
}
UnicodeInformation/UnicodeData.cs
+81 -16
View File
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
@@ -12,25 +13,93 @@ namespace System.Unicode
private readonly Version unicodeVersion;
private readonly UnicodeCharacterData[] characterData;
public static async Task<UnicodeData> FromStreamAsync(Stream stream)
public static UnicodeData FromStream(Stream stream)
{
var buffer = new byte[4096];
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.ReadByte() != 'U'
| reader.ReadByte() != 'C'
| reader.ReadByte() != 'D')
throw new InvalidDataException();
if (await stream.ReadAsync(buffer, 0, 6).ConfigureAwait(false) != 6)
throw new EndOfStreamException();
byte formatVersion = reader.ReadByte();
if (buffer[0] != (byte)'U'
|| buffer[1] != (byte)'C'
|| buffer[2] != (byte)'D')
throw new InvalidDataException();
#if !DEBUG
if (formatVersion != 1) throw new InvalidDataException();
#endif
int formatVersion = buffer[3];
var unicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
if (formatVersion != 1) throw new InvalidDataException();
var entries = new UnicodeCharacterData[ReadCodePoint(reader)];
var unicodeVersion = new Version(buffer[4], buffer[6]);
for (int i = 0; i < entries.Length; ++i)
{
entries[i] = ReadEntry(reader);
}
return new UnicodeData(unicodeVersion, null);
return new UnicodeData(unicodeVersion, entries);
}
}
private static UnicodeCharacterData ReadEntry(BinaryReader reader)
{
var fields = (UcdFields)reader.ReadUInt16();
var codePointRange = (fields & UcdFields.CodePointRange) != 0 ? new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)) : new UnicodeCharacterRange(ReadCodePoint(reader));
string name = (fields & UcdFields.Name) != 0 ? reader.ReadString() : null;
var category = (fields & UcdFields.Category) != 0 ? (UnicodeCategory)reader.ReadByte() : UnicodeCategory.OtherNotAssigned;
var canonicalCombiningClass = (fields & UcdFields.CanonicalCombiningClass) != 0 ? (CanonicalCombiningClass)reader.ReadByte() : CanonicalCombiningClass.NotReordered;
var bidirectionalClass = (fields & UcdFields.BidirectionalClass) != 0 ? (BidirectionalClass)reader.ReadByte() : 0;
string decompositionType = (fields & UcdFields.DecompositionType) != 0 ? reader.ReadString() : null;
var numericType = (UnicodeNumericType)((int)(fields & UcdFields.NumericNumeric) >> 6);
UnicodeRationalNumber numericValue = numericType != UnicodeNumericType.None ?
new UnicodeRationalNumber(reader.ReadInt64(), reader.ReadByte()) :
default(UnicodeRationalNumber);
string oldName = (fields & UcdFields.OldName) != 0 ? reader.ReadString() : null;
string simpleUpperCaseMapping = (fields & UcdFields.SimpleUpperCaseMapping) != 0 ? reader.ReadString() : null;
string simpleLowerCaseMapping = (fields & UcdFields.SimpleLowerCaseMapping) != 0 ? reader.ReadString() : null;
string simpleTitleCaseMapping = (fields & UcdFields.SimpleTitleCaseMapping) != 0 ? reader.ReadString() : null;
ContributoryProperties contributoryProperties = (fields & UcdFields.ContributoryProperties) != 0 ? (ContributoryProperties)reader.ReadInt32() : 0;
return new UnicodeCharacterData
(
codePointRange,
name,
category,
canonicalCombiningClass,
bidirectionalClass,
decompositionType,
numericType,
numericValue,
(fields & UcdFields.BidirectionalMirrored) != 0,
oldName,
simpleUpperCaseMapping,
simpleLowerCaseMapping,
simpleTitleCaseMapping,
contributoryProperties,
null
);
}
//private static int ReadInt24(BinaryReader reader)
//{
// return reader.ReadByte() | ((reader.ReadByte() | (reader.ReadByte() << 8)) << 8);
//}
private static int ReadCodePoint(BinaryReader reader)
{
byte b = reader.ReadByte();
if (b < 128) return b;
else if ((b & 0x40) == 0)
{
return 128 + (((b & 0x3F) << 8) | reader.ReadByte());
}
else
{
return 0x4080 + (((((b & 0x3F) << 8) | reader.ReadByte()) << 8) | reader.ReadByte());
}
}
internal UnicodeData(Version unicodeVersion, UnicodeCharacterData[] characterData)
@@ -64,9 +133,5 @@ namespace System.Unicode
{
return FindCodePoint(codePoint);
}
private void LoadData(Stream stream)
{
}
}
}
UnicodeInformation/UnicodeInformation.csproj
+1
View File
@@ -59,6 +59,7 @@
<Compile Include="CanonicalCombiningClass.cs" />
<Compile Include="CodePointEnumerable.cs" />
<Compile Include="CodePointEnumerator.cs" />
<Compile Include="UcdFields.cs" />
<Compile Include="EnumHelper.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="StringExtensions.cs" />