diff --git a/UnicodeCharacterInspector/CharacterInfoViewModel.cs b/UnicodeCharacterInspector/CharacterInfoViewModel.cs
index 27d23d7..1dd67b3 100644
--- a/UnicodeCharacterInspector/CharacterInfoViewModel.cs
+++ b/UnicodeCharacterInspector/CharacterInfoViewModel.cs
@@ -12,7 +12,7 @@ namespace UnicodeCharacterInspector
{
private string character;
private int codePoint;
- private UnicodeCharInfo characterInfo = UnicodeInfo.Default.Get(0);
+ private UnicodeCharInfo characterInfo = UnicodeInfo.Default.GetCharInfo(0);
public CharacterInfoViewModel()
{
@@ -40,12 +40,12 @@ namespace UnicodeCharacterInspector
if ((character = value) != null)
{
codePoint = char.ConvertToUtf32(character, 0);
- characterInfo = UnicodeInfo.Default.Get(codePoint);
+ characterInfo = UnicodeInfo.Default.GetCharInfo(codePoint);
}
else
{
codePoint = 0;
- characterInfo = UnicodeInfo.Default.Get(0);
+ characterInfo = UnicodeInfo.Default.GetCharInfo(0);
}
NotifyPropertyChanged();
@@ -53,6 +53,7 @@ namespace UnicodeCharacterInspector
NotifyPropertyChanged("Name");
NotifyPropertyChanged("OldName");
NotifyPropertyChanged("Category");
+ NotifyPropertyChanged("Block");
NotifyPropertyChanged("CanonicalCombiningClass");
NotifyPropertyChanged("BidirectionalClass");
NotifyPropertyChanged("DecompositionType");
@@ -84,6 +85,11 @@ namespace UnicodeCharacterInspector
get { return character != null ? characterInfo.Category : null as UnicodeCategory?; }
}
+ public string Block
+ {
+ get { return character != null ? characterInfo.Block : null; }
+ }
+
public CanonicalCombiningClass? CanonicalCombiningClass
{
get { return character != null ? characterInfo.CanonicalCombiningClass : null as CanonicalCombiningClass?; }
diff --git a/UnicodeCharacterInspector/MainWindow.xaml b/UnicodeCharacterInspector/MainWindow.xaml
index 2c7c828..05ab2eb 100644
--- a/UnicodeCharacterInspector/MainWindow.xaml
+++ b/UnicodeCharacterInspector/MainWindow.xaml
@@ -69,6 +69,7 @@
+
@@ -79,20 +80,22 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/UnicodeInformation.Builder/Program.cs b/UnicodeInformation.Builder/Program.cs
index bf1fdac..f520cca 100644
--- a/UnicodeInformation.Builder/Program.cs
+++ b/UnicodeInformation.Builder/Program.cs
@@ -18,6 +18,7 @@ namespace System.Unicode.Builder
{
"UnicodeData.txt",
"PropList.txt",
+ "Blocks.txt",
"Jamo.txt"
};
@@ -113,7 +114,7 @@ namespace System.Unicode.Builder
private static void Main(string[] args)
{
- UnicodeDataBuilder data;
+ UnicodeInfoBuilder data;
using (var ucdSource = GetUcdSource(null, null, null))
{
diff --git a/UnicodeInformation.Builder/UnicodeDataFileReader.cs b/UnicodeInformation.Builder/UnicodeDataFileReader.cs
index 174b47e..d00c3ca 100644
--- a/UnicodeInformation.Builder/UnicodeDataFileReader.cs
+++ b/UnicodeInformation.Builder/UnicodeDataFileReader.cs
@@ -9,12 +9,12 @@ namespace System.Unicode.Builder
{
public class UnicodeDataFileReader : IDisposable
{
- private struct AsciiCharBuffer
+ private struct Utf8Buffer
{
private readonly UnicodeDataFileReader reader;
private int length;
- public AsciiCharBuffer(UnicodeDataFileReader reader)
+ public Utf8Buffer(UnicodeDataFileReader reader)
{
this.reader = reader;
this.length = 0;
@@ -25,8 +25,8 @@ namespace System.Unicode.Builder
private void EnsureExtraCapacity(int count)
{
if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity");
- if (reader.charBuffer.Length < checked(length + count))
- Array.Resize(ref reader.charBuffer, Math.Max(length + count, reader.charBuffer.Length << 1));
+ if (reader.utf8StringBuffer.Length < checked(length + count))
+ Array.Resize(ref reader.utf8StringBuffer, Math.Max(length + count, reader.utf8StringBuffer.Length << 1));
}
public void Append(byte[] value, int startIndex, int count)
@@ -37,27 +37,23 @@ namespace System.Unicode.Builder
EnsureExtraCapacity(value.Length);
- var buffer = reader.charBuffer;
+ var buffer = reader.utf8StringBuffer;
for (int i = startIndex; i < count; ++i)
{
- byte b = value[i];
-
- if (b > 127) throw new InvalidDataException(string.Format("Unexpected character value: {0}.", b));
-
- buffer[length++] = (char)b;
+ buffer[length++] = value[i];
}
}
public override string ToString()
{
- return length > 0 ? new string(reader.charBuffer, 0, length) : string.Empty;
+ return length > 0 ? Encoding.UTF8.GetString(reader.utf8StringBuffer, 0, length) : string.Empty;
}
}
private readonly Stream stream;
private readonly byte[] byteBuffer;
- private char[] charBuffer;
+ private byte[] utf8StringBuffer;
private int index;
private int length;
private bool hasField = false;
@@ -72,7 +68,7 @@ namespace System.Unicode.Builder
{
this.stream = stream;
this.byteBuffer = new byte[8192];
- this.charBuffer = new char[100];
+ this.utf8StringBuffer = new byte[100];
this.leaveOpen = leaveOpen;
}
@@ -156,7 +152,7 @@ namespace System.Unicode.Builder
}
}
- var buffer = new AsciiCharBuffer(this);
+ var buffer = new Utf8Buffer(this);
int startOffset;
int endOffset;
diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs
index 035935e..e65aac9 100644
--- a/UnicodeInformation.Builder/UnicodeDataProcessor.cs
+++ b/UnicodeInformation.Builder/UnicodeDataProcessor.cs
@@ -8,10 +8,11 @@ using System.Threading.Tasks;
namespace System.Unicode.Builder
{
- public class UnicodeDataProcessor
+ internal class UnicodeDataProcessor
{
public const string UnicodeDataFileName = "UnicodeData.txt";
public const string PropListFileName = "PropList.txt";
+ public const string BlocksFileName = "Blocks.txt";
private static string ParseSimpleCaseMapping(string mapping)
{
@@ -23,12 +24,21 @@ namespace System.Unicode.Builder
private static string NullIfEmpty(string s)
{
return string.IsNullOrEmpty(s) ? null : s;
- }
+ }
- public static async Task BuildDataAsync(IUcdSource ucdSource)
+ public static async Task BuildDataAsync(IUcdSource ucdSource)
{
- var builder = new UnicodeDataBuilder(new Version(7, 0));
+ var builder = new UnicodeInfoBuilder(new Version(7, 0));
+ await ProcessUnicodeDataFile(ucdSource, builder).ConfigureAwait(false);
+ await ProcessPropListFile(ucdSource, builder).ConfigureAwait(false);
+ await ProcessBlocksFile(ucdSource, builder).ConfigureAwait(false);
+
+ return builder;
+ }
+
+ private static async Task ProcessUnicodeDataFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
+ {
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(UnicodeDataFileName).ConfigureAwait(false)))
{
int rangeStartCodePoint = -1;
@@ -55,11 +65,11 @@ namespace System.Unicode.Builder
codePoint = new UnicodeCharacterRange(rangeStartCodePoint, codePoint.LastCodePoint);
- name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
+ name = name.Substring(1, name.Length - 8).ToUpperInvariant(); // Upper-case the name in order to respect unicode naming scheme. (Spec says all names are uppercase ASCII)
rangeStartCodePoint = -1;
}
- else if (name == "") // Ignore the name of the property for these code points, as it should really be empty by the spec.
+ else if (name == "") // Ignore the name of the property for these code points, as it should really be empty by the spec.
{
// For control characters, we can derive a character label in of the form , which is not the character name.
name = null;
@@ -70,7 +80,7 @@ namespace System.Unicode.Builder
}
}
else if (rangeStartCodePoint >= 0)
- {
+ {
throw new InvalidDataException("Invalid range data in UnicodeData.txt.");
}
@@ -142,7 +152,10 @@ namespace System.Unicode.Builder
builder.Insert(characterData);
}
}
-
+ }
+
+ private static async Task ProcessPropListFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
+ {
using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(PropListFileName).ConfigureAwait(false)))
{
while (reader.MoveToNextLine())
@@ -156,8 +169,17 @@ namespace System.Unicode.Builder
}
}
}
+ }
- return builder;
+ private static async Task ProcessBlocksFile(IUcdSource ucdSource, UnicodeInfoBuilder builder)
+ {
+ using (var reader = new UnicodeDataFileReader(await ucdSource.OpenDataFileAsync(BlocksFileName).ConfigureAwait(false)))
+ {
+ while (reader.MoveToNextLine())
+ {
+ builder.AddBlockEntry(new UnicodeBlock(UnicodeCharacterRange.Parse(reader.ReadField()), reader.ReadField().Trim()));
+ }
+ }
}
}
}
diff --git a/UnicodeInformation.Builder/UnicodeDataBuilder.cs b/UnicodeInformation.Builder/UnicodeInfoBuilder.cs
similarity index 81%
rename from UnicodeInformation.Builder/UnicodeDataBuilder.cs
rename to UnicodeInformation.Builder/UnicodeInfoBuilder.cs
index 53138bc..a3d60e9 100644
--- a/UnicodeInformation.Builder/UnicodeDataBuilder.cs
+++ b/UnicodeInformation.Builder/UnicodeInfoBuilder.cs
@@ -7,13 +7,14 @@ using System.Threading.Tasks;
namespace System.Unicode.Builder
{
- public class UnicodeDataBuilder
+ internal class UnicodeInfoBuilder
{
private readonly Version unicodeVersion;
private UnicodeCharacterDataBuilder[] entries = new UnicodeCharacterDataBuilder[10000];
private int entryCount;
+ private List blockEntries = new List(100);
- public UnicodeDataBuilder(Version unicodeVersion)
+ public UnicodeInfoBuilder(Version unicodeVersion)
{
this.unicodeVersion = unicodeVersion;
}
@@ -121,6 +122,11 @@ namespace System.Unicode.Builder
}
}
+ public void AddBlockEntry(UnicodeBlock block)
+ {
+ blockEntries.Add(block);
+ }
+
public UnicodeInfo ToUnicodeData()
{
var finalData = new UnicodeCharacterData[entryCount];
@@ -128,7 +134,14 @@ namespace System.Unicode.Builder
for (int i = 0; i < finalData.Length; ++i)
finalData[i] = entries[i].ToCharacterData();
- return new UnicodeInfo(unicodeVersion, finalData);
+ return new UnicodeInfo(unicodeVersion, finalData, blockEntries.ToArray());
+ }
+
+ private void WriteUnicodeBlockToFile(BinaryWriter writer, UnicodeBlock block)
+ {
+ writer.WriteCodePoint(block.CodePointRange.FirstCodePoint);
+ writer.WriteCodePoint(block.CodePointRange.LastCodePoint);
+ writer.Write(block.Name);
}
public void WriteToStream(Stream stream)
@@ -143,6 +156,10 @@ namespace System.Unicode.Builder
{
entries[i].WriteToFile(writer);
}
+ if (blockEntries.Count > 255) throw new InvalidOperationException("There are too many block entries. The file format needs to be upgraded.");
+ writer.Write((byte)blockEntries.Count);
+ for (int i = 0; i < blockEntries.Count; ++i)
+ WriteUnicodeBlockToFile(writer, blockEntries[i]);
}
}
diff --git a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
index d70cb1f..f8b431d 100644
--- a/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
+++ b/UnicodeInformation.Builder/UnicodeInformation.Builder.csproj
@@ -60,7 +60,7 @@
-
+
diff --git a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
index 73b1e47..1c25bf4 100644
--- a/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
+++ b/UnicodeInformation.Tests/UnicodeDataManagerTests.cs
@@ -40,7 +40,7 @@ namespace System.Unicode.Tests
var data = (await UnicodeDataProcessor.BuildDataAsync(source)).ToUnicodeData();
- Assert.AreEqual((int)'\t', data.Get('\t').CodePoint);
+ Assert.AreEqual((int)'\t', data.GetCharInfo('\t').CodePoint);
}
[TestMethod]
diff --git a/UnicodeInformation/UnicodeBlock.cs b/UnicodeInformation/UnicodeBlock.cs
new file mode 100644
index 0000000..79b2530
--- /dev/null
+++ b/UnicodeInformation/UnicodeBlock.cs
@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Unicode
+{
+ public struct UnicodeBlock
+ {
+ public readonly UnicodeCharacterRange CodePointRange;
+ public readonly string Name;
+
+ internal UnicodeBlock(UnicodeCharacterRange codePointRange, string name)
+ {
+ this.CodePointRange = codePointRange;
+ this.Name = name;
+ }
+ }
+}
diff --git a/UnicodeInformation/UnicodeCharInfo.cs b/UnicodeInformation/UnicodeCharInfo.cs
index c06e323..274b688 100644
--- a/UnicodeInformation/UnicodeCharInfo.cs
+++ b/UnicodeInformation/UnicodeCharInfo.cs
@@ -11,6 +11,7 @@ namespace System.Unicode
{
private readonly int codePoint;
private readonly UnicodeCharacterData characterData;
+ private readonly string block;
public int CodePoint { get { return codePoint; } }
@@ -25,6 +26,7 @@ namespace System.Unicode
}
public UnicodeCategory Category { get { return characterData.Category; } }
+ public string Block { get { return block; } }
public CanonicalCombiningClass CanonicalCombiningClass { get { return characterData.CanonicalCombiningClass; } }
public BidirectionalClass BidirectionalClass { get { return characterData.BidirectionalClass; } }
public CompatibilityFormattingTag DecompositionType { get { return characterData.DecompositionType; } }
@@ -38,10 +40,11 @@ namespace System.Unicode
public string SimpleTitleCaseMapping { get { return characterData.SimpleTitleCaseMapping; } }
public ContributoryProperties ContributoryProperties { get { return characterData.ContributoryProperties; } }
- internal UnicodeCharInfo(int codePoint, UnicodeCharacterData characterData)
+ internal UnicodeCharInfo(int codePoint, UnicodeCharacterData characterData, string block)
{
this.codePoint = codePoint;
this.characterData = characterData;
+ this.block = block;
}
}
}
diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs
index 7a2cdd5..9cfa1df 100644
--- a/UnicodeInformation/UnicodeInfo.cs
+++ b/UnicodeInformation/UnicodeInfo.cs
@@ -17,7 +17,7 @@ namespace System.Unicode
private static UnicodeInfo ReadEmbeddedUnicodeData()
{
- using (var stream = new DeflateStream(typeof(UnicodeInfo).GetTypeInfo().Assembly.GetManifestResourceStream("System.Unicode.ucd.dat"), CompressionMode.Decompress, false))
+ using (var stream = new DeflateStream(typeof(UnicodeInfo).GetTypeInfo().Assembly.GetManifestResourceStream("ucd.dat"), CompressionMode.Decompress, false))
{
return FromStream(stream);
}
@@ -25,6 +25,7 @@ namespace System.Unicode
private readonly Version unicodeVersion;
private readonly UnicodeCharacterData[] characterData;
+ private readonly UnicodeBlock[] blockEntries;
public static UnicodeInfo FromStream(Stream stream)
{
@@ -43,18 +44,25 @@ namespace System.Unicode
var unicodeVersion = new Version(reader.ReadUInt16(), reader.ReadByte());
- var entries = new UnicodeCharacterData[ReadCodePoint(reader)];
+ var characterDataEntries = new UnicodeCharacterData[ReadCodePoint(reader)];
- for (int i = 0; i < entries.Length; ++i)
+ for (int i = 0; i < characterDataEntries.Length; ++i)
{
- entries[i] = ReadEntry(reader);
+ characterDataEntries[i] = ReadCharacterDataEntry(reader);
}
- return new UnicodeInfo(unicodeVersion, entries);
+ var blockEntries = new UnicodeBlock[reader.ReadByte()];
+
+ for (int i = 0; i < blockEntries.Length; ++i)
+ {
+ blockEntries[i] = ReadBlockEntry(reader);
+ }
+
+ return new UnicodeInfo(unicodeVersion, characterDataEntries, blockEntries);
}
}
- private static UnicodeCharacterData ReadEntry(BinaryReader reader)
+ private static UnicodeCharacterData ReadCharacterDataEntry(BinaryReader reader)
{
var fields = (UcdFields)reader.ReadUInt16();
@@ -97,6 +105,11 @@ namespace System.Unicode
);
}
+ private static UnicodeBlock ReadBlockEntry(BinaryReader reader)
+ {
+ return new UnicodeBlock(new UnicodeCharacterRange(ReadCodePoint(reader), ReadCodePoint(reader)), reader.ReadString());
+ }
+
#if DEBUG
internal static int ReadCodePoint(BinaryReader reader)
#else
@@ -120,10 +133,11 @@ namespace System.Unicode
}
}
- internal UnicodeInfo(Version unicodeVersion, UnicodeCharacterData[] characterData)
+ internal UnicodeInfo(Version unicodeVersion, UnicodeCharacterData[] characterData, UnicodeBlock[] blockEntries)
{
this.unicodeVersion = unicodeVersion;
this.characterData = characterData;
+ this.blockEntries = blockEntries;
}
public Version UnicodeVersion { get { return unicodeVersion; } }
@@ -147,9 +161,40 @@ namespace System.Unicode
return null;
}
- public UnicodeCharInfo Get(int codePoint)
+ private int FindBlockIndex(int codePoint)
{
- return new UnicodeCharInfo(codePoint, FindCodePoint(codePoint));
+ int minIndex = 0;
+ int maxIndex = blockEntries.Length - 1;
+
+ do
+ {
+ int index = (minIndex + maxIndex) >> 1;
+
+ int Δ = blockEntries[index].CodePointRange.CompareCodePoint(codePoint);
+
+ if (Δ == 0) return index;
+ else if (Δ < 0) maxIndex = index - 1;
+ else minIndex = index + 1;
+ } while (minIndex <= maxIndex);
+
+ return -1;
}
+
+ private string GetBlockName(int codePoint)
+ {
+ int i = FindBlockIndex(codePoint);
+
+ return i >= 0 ? blockEntries[i].Name : null;
+ }
+
+ public UnicodeCharInfo GetCharInfo(int codePoint)
+ {
+ return new UnicodeCharInfo(codePoint, FindCodePoint(codePoint), GetBlockName(codePoint));
+ }
+
+ public UnicodeBlock[] GetBlocks()
+ {
+ return (UnicodeBlock[])blockEntries.Clone();
+ }
}
}
diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj
index 014632a..db57fd9 100644
--- a/UnicodeInformation/UnicodeInformation.csproj
+++ b/UnicodeInformation/UnicodeInformation.csproj
@@ -53,7 +53,9 @@
System.Unicode.snk
-
+
+ ucd.dat
+
@@ -66,6 +68,7 @@
+
diff --git a/UnicodeInformation/ucd.dat b/UnicodeInformation/ucd.dat
index c502089..6327a9a 100644
Binary files a/UnicodeInformation/ucd.dat and b/UnicodeInformation/ucd.dat differ