From fd2e8122edd57e5e07061045b792e818373ba81c Mon Sep 17 00:00:00 2001 From: GoldenCrystal Date: Wed, 26 Nov 2014 22:27:09 +0100 Subject: [PATCH] Added a permissive code point enumerator, which doesn't choke on lone surrogates. --- AssemblyInfo.Common.cs | 4 +- .../CharacterInspectorViewModel.cs | 2 +- UnicodeInformation.Tests/UnicodeInfoTests.cs | 85 +++++++++++++++++++ UnicodeInformation/CodePointEnumerable.cs | 2 + .../PermissiveCodePointEnumerable.cs | 38 +++++++++ .../PermissiveCodePointEnumerator.cs | 65 ++++++++++++++ UnicodeInformation/StringExtensions.cs | 5 ++ UnicodeInformation/UnicodeInformation.csproj | 2 + 8 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 UnicodeInformation/PermissiveCodePointEnumerable.cs create mode 100644 UnicodeInformation/PermissiveCodePointEnumerator.cs diff --git a/AssemblyInfo.Common.cs b/AssemblyInfo.Common.cs index a4c2635..4ef604f 100644 --- a/AssemblyInfo.Common.cs +++ b/AssemblyInfo.Common.cs @@ -15,5 +15,5 @@ using System.Runtime.InteropServices; [assembly: ComVisible(false)] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: AssemblyVersion("1.0.1.0")] +[assembly: AssemblyFileVersion("1.0.1.0")] diff --git a/UnicodeCharacterInspector/CharacterInspectorViewModel.cs b/UnicodeCharacterInspector/CharacterInspectorViewModel.cs index b1ada28..c11a5f1 100644 --- a/UnicodeCharacterInspector/CharacterInspectorViewModel.cs +++ b/UnicodeCharacterInspector/CharacterInspectorViewModel.cs @@ -139,7 +139,7 @@ namespace UnicodeCharacterInspector int index = 0; if (!string.IsNullOrEmpty(value)) { - foreach (int codePoint in value.AsCodePointEnumerable()) + foreach (int codePoint in value.AsPermissiveCodePointEnumerable()) { if (index >= codePoints.Length) Array.Resize(ref codePoints, codePoints.Length * 2); diff --git a/UnicodeInformation.Tests/UnicodeInfoTests.cs b/UnicodeInformation.Tests/UnicodeInfoTests.cs index 9331a5a..9f5a4f2 100644 --- a/UnicodeInformation.Tests/UnicodeInfoTests.cs +++ b/UnicodeInformation.Tests/UnicodeInfoTests.cs @@ -50,6 +50,91 @@ namespace UnicodeInformation.Tests Assert.AreEqual(false, legacyEnumerator.MoveNext()); } + private static void EnumerationFailTest(string text) { foreach (int codePoint in text.AsCodePointEnumerable()) { } } + + [TestMethod] + public void CodePointEnumeratorDirtyTest() + { + AssertEx.ThrowsExactly(() => EnumerationFailTest(null)); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDA00")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDCD0")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDCD0\uDA00")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\u0041\uDA00")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\u0041\uDCD0")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDA00\u0041")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDCD0\u0041")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\uDA00\u0041\uDCD0\u0041")); + AssertEx.ThrowsExactly(() => EnumerationFailTest("\u0041\uDA00\u0041\uDCD0\u0041")); + } + + [TestMethod] + public void PermissiveCodePointEnumeratorTest() + { + string text = "\u0041\U0001F600\u00E9"; + + var enumerable = text.AsPermissiveCodePointEnumerable(); + + Assert.AreEqual(text, enumerable.Text); + + var enumerator = enumerable.GetEnumerator(); + + Assert.AreEqual(true, enumerator.MoveNext()); + Assert.AreEqual(0x0041, enumerator.Current); + Assert.AreEqual(true, enumerator.MoveNext()); + Assert.AreEqual(0x1F600, enumerator.Current); + Assert.AreEqual(true, enumerator.MoveNext()); + Assert.AreEqual(0x00E9, enumerator.Current); + Assert.AreEqual(false, enumerator.MoveNext()); + + var genericEnumerator = ((IEnumerable)enumerable).GetEnumerator(); + + Assert.AreEqual(true, genericEnumerator.MoveNext()); + Assert.AreEqual(0x0041, genericEnumerator.Current); + Assert.AreEqual(true, genericEnumerator.MoveNext()); + Assert.AreEqual(0x1F600, genericEnumerator.Current); + Assert.AreEqual(true, genericEnumerator.MoveNext()); + Assert.AreEqual(0x00E9, genericEnumerator.Current); + Assert.AreEqual(false, genericEnumerator.MoveNext()); + + var legacyEnumerator = ((IEnumerable)enumerable).GetEnumerator(); + + Assert.AreEqual(true, legacyEnumerator.MoveNext()); + Assert.AreEqual(0x0041, legacyEnumerator.Current); + Assert.AreEqual(true, legacyEnumerator.MoveNext()); + Assert.AreEqual(0x1F600, legacyEnumerator.Current); + Assert.AreEqual(true, legacyEnumerator.MoveNext()); + Assert.AreEqual(0x00E9, legacyEnumerator.Current); + Assert.AreEqual(false, legacyEnumerator.MoveNext()); + } + + private static void PermissiveEnumerationTest(string text, int[] expectedCharacters) + { + int i = 0; + + foreach (int codePoint in text.AsPermissiveCodePointEnumerable()) + { + Assert.AreEqual(expectedCharacters[i++], codePoint); + } + + Assert.AreEqual(expectedCharacters.Length, i); + } + + [TestMethod] + public void PermissiveCodePointEnumeratorDirtyTest() + { + AssertEx.ThrowsExactly(() => { foreach (int c in (null as string).AsPermissiveCodePointEnumerable()) { } }); + PermissiveEnumerationTest(string.Empty, new int[0]); + PermissiveEnumerationTest("\uDA00", new int[] { 0xDA00 }); + PermissiveEnumerationTest("\uDCD0", new int[] { 0xDCD0 }); + PermissiveEnumerationTest("\uDCD0\uDA00", new int[] { 0xDCD0, 0xDA00 }); + PermissiveEnumerationTest("\u0041\uDA00", new int[] { 0x0041, 0xDA00 }); + PermissiveEnumerationTest("\u0041\uDCD0", new int[] { 0x0041, 0xDCD0 }); + PermissiveEnumerationTest("\uDA00\u0041", new int[] { 0xDA00, 0x0041 }); + PermissiveEnumerationTest("\uDCD0\u0041", new int[] { 0xDCD0, 0x0041 }); + PermissiveEnumerationTest("\uDA00\u0041\uDCD0\u0041", new int[] { 0xDA00, 0x0041, 0xDCD0, 0x0041 }); + PermissiveEnumerationTest("\u0041\uDA00\u0041\uDCD0\u0041", new int[] { 0x0041, 0xDA00, 0x0041, 0xDCD0, 0x0041 }); + } + [TestMethod] public void DisplayTextTest() { diff --git a/UnicodeInformation/CodePointEnumerable.cs b/UnicodeInformation/CodePointEnumerable.cs index dcbf2aa..3c88f1d 100644 --- a/UnicodeInformation/CodePointEnumerable.cs +++ b/UnicodeInformation/CodePointEnumerable.cs @@ -13,6 +13,8 @@ namespace System.Unicode public CodePointEnumerable(string text) { + if (text == null) throw new ArgumentNullException(nameof(text)); + this.text = text; } diff --git a/UnicodeInformation/PermissiveCodePointEnumerable.cs b/UnicodeInformation/PermissiveCodePointEnumerable.cs new file mode 100644 index 0000000..a5f74c2 --- /dev/null +++ b/UnicodeInformation/PermissiveCodePointEnumerable.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + public struct PermissiveCodePointEnumerable : IEnumerable + { + private readonly string text; + + public PermissiveCodePointEnumerable(string text) + { + if (text == null) throw new ArgumentNullException(nameof(text)); + + this.text = text; + } + + public string Text { get { return text; } } + + public PermissiveCodePointEnumerator GetEnumerator() + { + return new PermissiveCodePointEnumerator(text); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/UnicodeInformation/PermissiveCodePointEnumerator.cs b/UnicodeInformation/PermissiveCodePointEnumerator.cs new file mode 100644 index 0000000..7e7efb0 --- /dev/null +++ b/UnicodeInformation/PermissiveCodePointEnumerator.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Unicode +{ + public struct PermissiveCodePointEnumerator : IEnumerator + { + private readonly string text; + private int current; + private int index; + + public PermissiveCodePointEnumerator(string text) + { + if (text == null) throw new ArgumentNullException(nameof(text)); + + this.text = text; + this.current = 0; + this.index = -1; + } + + public int Current { get { return current; } } + + object IEnumerator.Current { get { return current; } } + + void IDisposable.Dispose() { } + + public bool MoveNext() + { + if (index < text.Length && (index += current > 0xFFFF ? 2 : 1) < text.Length) + { + current = GetUtf32(text, index); + return true; + } + else + { + current = 0; + return false; + } + } + + void IEnumerator.Reset() + { + current = 0; + index = -1; + } + + private static int GetUtf32(string s, int index) + { + char c1 = s[index]; + + if (char.IsHighSurrogate(c1) && ++index < s.Length) + { + char c2 = s[index]; + + if (char.IsLowSurrogate(c2)) return char.ConvertToUtf32(c1, c2); + } + + return c1; + } + } +} diff --git a/UnicodeInformation/StringExtensions.cs b/UnicodeInformation/StringExtensions.cs index c7c31c9..99c008f 100644 --- a/UnicodeInformation/StringExtensions.cs +++ b/UnicodeInformation/StringExtensions.cs @@ -12,5 +12,10 @@ namespace System.Unicode { return new CodePointEnumerable(s); } + + public static PermissiveCodePointEnumerable AsPermissiveCodePointEnumerable(this string s) + { + return new PermissiveCodePointEnumerable(s); + } } } diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj index 4dbb8b8..c767af8 100644 --- a/UnicodeInformation/UnicodeInformation.csproj +++ b/UnicodeInformation/UnicodeInformation.csproj @@ -68,6 +68,8 @@ + +