Added Unihan data.

UnicodeInformation.Builder/UnicodeDataFileReader.cs

@@ -9,80 +9,24 @@ namespace System.Unicode.Builder
 {
 	public class UnicodeDataFileReader : IDisposable
 	{
-		private struct Utf8Buffer
-		{
-			private readonly UnicodeDataFileReader reader;
-			private int length;
-
-			public Utf8Buffer(UnicodeDataFileReader reader)
-			{
-				this.reader = reader;
-				this.length = 0;
-			}
-
-			public int Length { get { return length; } }
-
-			private void EnsureExtraCapacity(int count)
-			{
-				if (count < 0) throw new ArgumentOutOfRangeException("requiredExtraCapacity");
-				if (reader.utf8StringBuffer.Length < checked(length + count))
-					Array.Resize(ref reader.utf8StringBuffer, Math.Max(length + count, reader.utf8StringBuffer.Length << 1));
-			}
-
-			public void Append(byte[] value, int startIndex, int count)
-			{
-				if (value == null) throw new ArgumentNullException("value");
-				if (startIndex >= value.Length) throw new ArgumentOutOfRangeException("startIndex");
-				if (checked(count += startIndex) > value.Length) throw new ArgumentOutOfRangeException("count");
-
-				EnsureExtraCapacity(value.Length);
-
-				var buffer = reader.utf8StringBuffer;
-
-				for (int i = startIndex; i < count; ++i)
-				{
-					buffer[length++] = value[i];
-				}
-			}
-
-			public override string ToString()
-			{
-				return length > 0 ? Encoding.UTF8.GetString(reader.utf8StringBuffer, 0, length) : string.Empty;
-			}
-
-			public string ToTrimmedString()
-			{
-				if (length == 0) return string.Empty;
-
-				var buffer = reader.utf8StringBuffer;
-                int start = 0;
-				int end = length;
-
-				while (buffer[start] == ' ') if (++start == length) return string.Empty;
-				while (buffer[--end] == ' ') ;
-
-				return length > 0 ? Encoding.UTF8.GetString(reader.utf8StringBuffer, start, end - start + 1) : string.Empty;
-			}
-		}
-
 		private readonly Stream stream;
 		private readonly byte[] byteBuffer;
-		private byte[] utf8StringBuffer;
 		private int index;
 		private int length;
+		private readonly char fieldSeparator;
 		private bool hasField = false;
 		private readonly bool leaveOpen;
 
-		public UnicodeDataFileReader(Stream stream)
-			: this(stream, false)
+		public UnicodeDataFileReader(Stream stream, char fieldSeparator)
+			: this(stream, fieldSeparator, false)
 		{
 		}
 
-		public UnicodeDataFileReader(Stream stream, bool leaveOpen)
+		public UnicodeDataFileReader(Stream stream, char fieldSeparator, bool leaveOpen)
 		{
 			this.stream = stream;
+			this.fieldSeparator = fieldSeparator;
 			this.byteBuffer = new byte[8192];
-			this.utf8StringBuffer = new byte[100];
 			this.leaveOpen = leaveOpen;
 		}
 
@@ -163,49 +107,51 @@ namespace System.Unicode.Builder
 				}
 			}
 
-			var buffer = new Utf8Buffer(this);
-			int startOffset;
-			int endOffset;
-
-			do
+			using (var buffer = Utf8Buffer.Get())
 			{
-				startOffset = index;
-				endOffset = -1;
+				int startOffset;
+				int endOffset;
 
-				while (index < length)
+				do
 				{
-					byte b = byteBuffer[index];
+					startOffset = index;
+					endOffset = -1;
 
-					if (IsNewLineOrComment(b)) // NB: Do not advance to the next byte when end of line has been reached.
+					while (index < length)
 					{
-						endOffset = index;
-						hasField = false;
+						byte b = byteBuffer[index];
+
+						if (IsNewLineOrComment(b))   // NB: Do not advance to the next byte when end of line has been reached.
+						{
+							endOffset = index;
+							hasField = false;
+							break;
+						}
+						else if (b == fieldSeparator)
+						{
+							endOffset = index++;
+							break;
+						}
+						else
+						{
+							++index;
+						}
+					}
+
+					if (endOffset >= 0)
+					{
+						buffer.Append(byteBuffer, startOffset, endOffset - startOffset);
 						break;
 					}
-					else if(b == ';')
+					else if (index > startOffset)
 					{
-						endOffset = index++;
-						break;
+						buffer.Append(byteBuffer, startOffset, index - startOffset);
 					}
-					else
-					{
-						++index;
-					}
-				}
+				} while (RefillBuffer());
 
-				if (endOffset >= 0)
-				{
-					buffer.Append(byteBuffer, startOffset, endOffset - startOffset);
-					break;
-				}
-				else if (index > startOffset)
-				{
-					buffer.Append(byteBuffer, startOffset, index - startOffset);
-				}
-			} while (RefillBuffer());
-
-			return trim ? buffer.ToTrimmedString() : buffer.ToString();
-        }
+				return trim ? buffer.ToTrimmedString() : buffer.ToString();
+			}
+		}
 
 		/// <summary>Reads the next data field.</summary>
 		/// <remarks>This method will return <see langword="null"/> on end of line.</remarks>
@@ -255,7 +201,7 @@ namespace System.Unicode.Builder
 					{
 						++index;
 
-						if (b == ';')
+						if (b == fieldSeparator)
 						{
 							return true;
 						}