From 31074ec9db67ee055aa7898ae2cd77122f526ef1 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Fri, 16 May 2014 17:19:05 +0100 Subject: [PATCH 01/11] Issue #AR-4-delinter_fixes: trying to sort out issue with \r\n being replaced inside values --- .../Serialization/Base/RegressionTestBase.cs | 1 - .../Delinter/ArchetypeJsonDelinterTest.cs | 31 ++++++++++++++++++- .../Serialization/ArchetypeJsonDelinter.cs | 20 ++++++------ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs index d06383d..85021fc 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs @@ -12,7 +12,6 @@ using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Models.Editors; -using Umbraco.Core.Models.Membership; using Umbraco.Core.Persistence; using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; diff --git a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs index 7c92ba2..a60b582 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs @@ -49,7 +49,36 @@ namespace Archetype.Tests.Serialization.Delinter _nestedClass.Nested.SlideShow = _referenceModel; _nestedClass.Nested.Nested = GetNestedClass(); - } + } + + [Test] + public void Delinter_ByPasses_Values() + { + const string _JSON_WITH_RESERVED_CHARS = @"{ + ""alias"": ""captions"", + ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \\r \\n test1\r\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\n test2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \r\r\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" +}"; + + var referenceModel = new Captions() + { + new TextString() {Text = @"test1 \\r \\n test1\r"}, + new TextString() {Text = @"test2\r\n test2"}, + new TextString() {Text = @"test3b \r\r\rtest3"} + }; + + var actualModel = ConvertArchetypeJsonToModel(_JSON_WITH_RESERVED_CHARS.DelintArchetypeJson()); + + Assert.IsInstanceOf(actualModel); + Assert.IsNotNull(actualModel); + + Assert.AreEqual(3, actualModel.Count); + + foreach (var refItem in referenceModel) + { + var index = referenceModel.IndexOf(refItem); + AssertAreEqual(refItem, actualModel.ElementAt(index)); + } + } [Test] public void ReferenceJsonAndModel_Equal_ExpectedJsonAndModel() diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index ef7b758..ee7f8e2 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -56,13 +56,13 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveNewLine, new Regex(@"\\r|\\n|[\r\n]", RegexOptions.Multiline)}, - {DelinterStep.LabelFieldsets, new Regex(@"\s*\\+""(fieldsets)\\+"":\s*")}, - {DelinterStep.LabelProperties, new Regex(@"\s*\\+""(properties)\\+"":\s*")}, - {DelinterStep.LabelAlias, new Regex(@"\s*\\+""(alias)\\+"":\s*")}, - {DelinterStep.LabelValue, new Regex(@"\s*\\+""(value)\\+"":\s*")}, - {DelinterStep.UnescapeValues, new Regex(@"""(alias|value)"":\\+""(.*?)\\+""")}, - {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""({""fieldsets"".+?})""")}, + {DelinterStep.RemoveNewLine, new Regex(@"(? String.Empty}, {DelinterAction.UnescapeLabels, match => String.Format(@"""{0}"":", match.Groups[1])}, {DelinterAction.UnescapeValues, match => String.Format(@"""{0}"":""{1}""", match.Groups[1], match.Groups[2])}, - {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{1}", match.Groups[1], match.Groups[2])} + {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1], match.Groups[2])} }; Pipeline = new Dictionary { - {DelinterStep.RemoveNewLine, DelinterAction.Remove}, {DelinterStep.LabelFieldsets, DelinterAction.UnescapeLabels}, {DelinterStep.LabelProperties, DelinterAction.UnescapeLabels}, {DelinterStep.LabelAlias, DelinterAction.UnescapeLabels}, {DelinterStep.LabelValue, DelinterAction.UnescapeLabels}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets}, - {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove} + {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, + {DelinterStep.RemoveNewLine, DelinterAction.Remove} }; } } From 6f2eaa9859f2871949e0f8810e9376d02e343c13 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Mon, 19 May 2014 16:08:00 +0100 Subject: [PATCH 02/11] Issue #AR-4-delinter_fixes: trying to fix delinter regex --- .../Delinter/ArchetypeJsonDelinterTest.cs | 2 +- .../Regression/SerializationTestHelper.cs | 2 +- .../Serialization/ArchetypeJsonDelinter.cs | 42 ++++++++----------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs index a60b582..5a75dd2 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs @@ -52,7 +52,7 @@ namespace Archetype.Tests.Serialization.Delinter } [Test] - public void Delinter_ByPasses_Values() + public void Delinter_RemoveNewLine_ByPasses_Values() { const string _JSON_WITH_RESERVED_CHARS = @"{ ""alias"": ""captions"", diff --git a/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs b/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs index fc4f8b9..843580e 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs @@ -27,7 +27,7 @@ namespace Archetype.Tests.Serialization.Regression DateTwo = null, Id = 123, NullableId = null, - Text = "Test Text" + Text = @"Test \r\n Text ""quote"" \r\n" }; } diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index ee7f8e2..017f9bd 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -6,14 +6,12 @@ namespace Archetype.Serialization { public enum DelinterStep { - RemoveNewLine = 0, - LabelFieldsets, - LabelProperties, - LabelAlias, - LabelValue, + RemoveWhiteSpace = 0, + RemoveNewLine, + UnescapeLabels, + UnescapeAlias, UnescapeValues, FixNestedFieldsets, - RemoveWhiteSpace } public enum DelinterAction @@ -56,34 +54,30 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveNewLine, new Regex(@"(?> { {DelinterAction.Remove, match => String.Empty}, - {DelinterAction.UnescapeLabels, match => String.Format(@"""{0}"":", match.Groups[1])}, - {DelinterAction.UnescapeValues, match => String.Format(@"""{0}"":""{1}""", match.Groups[1], match.Groups[2])}, - {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1], match.Groups[2])} + {DelinterAction.UnescapeLabels, match => String.Format(@"""{0}"":", match.Groups[1].Value)}, + {DelinterAction.UnescapeValues, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, match.Groups[2].Value)}, + {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[3].Value)} }; Pipeline = new Dictionary { - {DelinterStep.LabelFieldsets, DelinterAction.UnescapeLabels}, - {DelinterStep.LabelProperties, DelinterAction.UnescapeLabels}, - {DelinterStep.LabelAlias, DelinterAction.UnescapeLabels}, - {DelinterStep.LabelValue, DelinterAction.UnescapeLabels}, + {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, + {DelinterStep.RemoveNewLine, DelinterAction.Remove}, + {DelinterStep.UnescapeLabels, DelinterAction.UnescapeLabels}, + {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, - {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets}, - {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, - {DelinterStep.RemoveNewLine, DelinterAction.Remove} + {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets} }; } } From 1913f8bf14f902b7ca6f009bf8ba358018a650ba Mon Sep 17 00:00:00 2001 From: cankoluman Date: Mon, 19 May 2014 16:25:20 +0100 Subject: [PATCH 03/11] Issue #AR-4-delinter_fixes: create single entry point for deserialization - 15 tests failing now --- .../Serialization/ArchetypeJsonConverter.cs | 77 +++++++++++-------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs index ffb6be5..d7c8475 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs @@ -38,22 +38,9 @@ namespace Archetype.Serialization var jToken = JToken.ReadFrom(reader); if (jToken == null) - return null; - - var obj = Activator.CreateInstance(objectType); + return null; - if (null != obj as IEnumerable) - { - JToken enumerableToken; - - if (TryParseJTokenAsEnumerable(jToken, out enumerableToken) - || TryParseJTokenAsEnumerable(jToken["value"], out enumerableToken)) - return DeserializeEnumerableObject(obj, enumerableToken); - } - - return null == jToken as JArray - ? DeserializeObject(obj, jToken) - : PopulateProperties(obj, jToken); + return DeserializeJson(objectType, GetDelintedJToken(jToken)); } public override bool CanConvert(Type objectType) @@ -63,17 +50,48 @@ namespace Archetype.Serialization #endregion - #region private methods - deserialization + #region private methods - deserialization + + private JToken GetDelintedJToken(JToken jToken) + { + return JToken.Parse(jToken.ToString(Formatting.None).DelintArchetypeJson()); + } + + private object DeserializeJson(Type objectType, JToken jToken) + { + + var obj = Activator.CreateInstance(objectType); + + if (null != obj as IEnumerable) + { + JToken enumerableToken; + + if (TryParseJTokenAsEnumerable(jToken, out enumerableToken) + || (jToken.SelectToken("value") != null && TryParseJTokenAsEnumerable(jToken["value"], out enumerableToken))) + return DeserializeEnumerableObject(obj, enumerableToken); + } + + return null == jToken as JArray + ? DeserializeObject(obj, jToken) + : PopulateProperties(obj, jToken); + } private object DeserializeEnumerableObject(object obj, JToken jToken) { - var model = obj as IEnumerable; - - var itemType = model.GetType().BaseType.GetGenericArguments().First(); - foreach (var fs in jToken["fieldsets"].Where(fs => fs["alias"].ToString().Equals(GetFieldsetName(itemType)))) - { - var item = JsonConvert.DeserializeObject( - fs["properties"].ToString().DelintArchetypeJson(), itemType, this); + var model = obj as IEnumerable; + var fsToken = jToken; + + var itemType = model.GetType().GetGenericArguments().FirstOrDefault(); + + if (itemType == null) + { + itemType = model.GetType().BaseType.GetGenericArguments().First(); + fsToken = fsToken["fieldsets"]; + } + + foreach (var fs in fsToken.Where(fs => fs["alias"].ToString().Equals(GetFieldsetName(itemType)))) + { + var item = DeserializeJson(itemType, fs["properties"]); obj.GetType().GetMethod("Add").Invoke(obj, new[] { item }); } @@ -208,15 +226,12 @@ namespace Archetype.Serialization private object GetPropertyValue(PropertyInfo propertyInfo, JToken propJToken) { return IsTypeArchetypeDatatype(propertyInfo.PropertyType) - ? JsonConvert.DeserializeObject(propJToken.ToString().DelintArchetypeJson(), propertyInfo.PropertyType, - this) + ? DeserializeJson(propertyInfo.PropertyType, propJToken) : IsTypeIEnumerableArchetypeDatatype(propertyInfo.PropertyType) - ? JsonConvert.DeserializeObject(propJToken["value"].SelectToken("fieldsets").ToString(), propertyInfo.PropertyType, - this) + ? DeserializeJson(propertyInfo.PropertyType, propJToken["value"].SelectToken("fieldsets")) : GetDeserializedPropertyValue(propJToken, propertyInfo.PropertyType); } - private JToken ParseJTokenFromItem(JToken jToken, string itemAlias) { return (jToken.SelectToken("alias") != null && jToken["alias"].ToString().Equals(itemAlias)) @@ -266,8 +281,10 @@ namespace Archetype.Serialization resultToken = null; //To Do: Strange newtonsoft behaviour - var fsToken = jToken.Parent == null ? jToken["fieldsets"] : jToken.Parent.Children().First(); - var jTokenEnumerable = jToken != null && fsToken != null && fsToken.Any(); + //var fsToken = jToken.Parent == null ? jToken["fieldsets"] : jToken.Parent.Children().First()["fieldsets"]; + var fsToken = jToken.SelectToken("fieldsets") != null ? jToken["fieldsets"] : jToken.Parent.Children().First(); + var jTokenEnumerable = jToken != null + && fsToken != null && fsToken is JArray; if (jTokenEnumerable) resultToken = jToken; From d5329935aa609169116768cb80ab5fd29668db56 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 09:32:01 +0100 Subject: [PATCH 04/11] Issue #AR-4-delinter_fixes: fixed all test failures due to jToken navigation errors. To Do: fix erros due to incorrect space removal --- .../Serialization/Regression/JsonTestStrings.cs | 2 +- .../Serialization/ArchetypeJsonConverter.cs | 8 ++++++-- .../Serialization/ArchetypeJsonDelinter.cs | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs b/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs index 72a503a..3e31c01 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs @@ -9,7 +9,7 @@ ""properties"": [ { ""alias"": ""text"", - ""value"": ""Test Text"" + ""value"": ""Test \r\n Text \""quote\"" \r\n"" }, { ""alias"": ""integer"", diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs index d7c8475..ed50204 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonConverter.cs @@ -281,8 +281,12 @@ namespace Archetype.Serialization resultToken = null; //To Do: Strange newtonsoft behaviour - //var fsToken = jToken.Parent == null ? jToken["fieldsets"] : jToken.Parent.Children().First()["fieldsets"]; - var fsToken = jToken.SelectToken("fieldsets") != null ? jToken["fieldsets"] : jToken.Parent.Children().First(); + var fsToken = jToken.Parent != null + ? jToken.Parent.Children().First() is JArray + ? jToken.Parent.Children().First() + : jToken.Parent.Children().First()["fieldsets"] + : jToken.SelectToken("fieldsets"); + var jTokenEnumerable = jToken != null && fsToken != null && fsToken is JArray; diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index 017f9bd..98f1d56 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -54,7 +54,7 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveWhiteSpace, new Regex(@"\s+(?=([^""]*""[^""]*"")*[^""]*$)", RegexOptions.Multiline)}, + {DelinterStep.RemoveWhiteSpace, new Regex(@"\s+(?!([^""]*""[^""]*"")*[^""]*$)", RegexOptions.Multiline)}, {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\+?""(alias|value|properties|fieldsets)\\+?"":\{*?(\\*|""|\[))|(\\+r\\+n)(?=(\{|\}|\]))")}, {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":")}, {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""", RegexOptions.Multiline)}, From cda17ac7542649b02a8535bdab428cf3c598414b Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 10:39:09 +0100 Subject: [PATCH 05/11] Issue #AR-4-delinter_fixes: update remove new line --- .../Serialization/ArchetypeJsonDelinter.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index 98f1d56..b98f7bb 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -54,12 +54,12 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveWhiteSpace, new Regex(@"\s+(?!([^""]*""[^""]*"")*[^""]*$)", RegexOptions.Multiline)}, - {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\+?""(alias|value|properties|fieldsets)\\+?"":\{*?(\\*|""|\[))|(\\+r\\+n)(?=(\{|\}|\]))")}, + {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""(alias|value|properties|fieldsets)\\*?"":\{*?(\\*|""|\[))|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\r|\n)+|(\\+r\\n+)(?=\s+)")}, + {DelinterStep.RemoveWhiteSpace, new Regex(@"\s+")}, {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":")}, - {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""", RegexOptions.Multiline)}, - {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\})", RegexOptions.Multiline)}, - {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{(\\+r\\+n)*?(""fieldsets""[\S\s]+?)}""", RegexOptions.Multiline)} + {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""")}, + {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\})")}, + {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{(\\+r\\+n)*?(""fieldsets""[\S\s]+?)}""")} }; Actions = new Dictionary> @@ -72,8 +72,8 @@ namespace Archetype.Serialization Pipeline = new Dictionary { - {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, - {DelinterStep.RemoveNewLine, DelinterAction.Remove}, + {DelinterStep.RemoveNewLine, DelinterAction.Remove}, + {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, {DelinterStep.UnescapeLabels, DelinterAction.UnescapeLabels}, {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, From d9a25d1bb3b0473782061111c35068b852a2094d Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 12:18:48 +0100 Subject: [PATCH 06/11] Issue #AR-4-delinter_fixes: fix all regexes except white space --- .../Serialization/ArchetypeJsonDelinter.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index b98f7bb..628ad43 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -17,6 +17,7 @@ namespace Archetype.Serialization public enum DelinterAction { Remove = 0, + RemoveWhiteSpace, UnescapeLabels, UnescapeValues, FixNestedFieldsets @@ -54,26 +55,27 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""(alias|value|properties|fieldsets)\\*?"":\{*?(\\*|""|\[))|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\r|\n)+|(\\+r\\n+)(?=\s+)")}, - {DelinterStep.RemoveWhiteSpace, new Regex(@"\s+")}, - {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":")}, + {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""""(alias|value|properties|fieldsets)\\*?"""":\{*?(\\*|""""|\[))|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\r|\n)+|(\\+r\\n+)(?!\s+.*?\\+""})")}, + {DelinterStep.RemoveWhiteSpace, new Regex(@"(\s*?)\\+""(fieldsets|properties|alias|value)\\+"":(\s+)|""(\s*?)\},(\s+)\{|\[(\s+)\{|[\]\}](\s+)[\]\}]")}, + {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":(\s*)")}, {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""")}, - {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\})")}, - {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{(\\+r\\+n)*?(""fieldsets""[\S\s]+?)}""")} + {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\s*?\})")}, + {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{\s*?(""fieldsets""[\S\s]+?)}""")} }; Actions = new Dictionary> { {DelinterAction.Remove, match => String.Empty}, + {DelinterAction.RemoveWhiteSpace, match => match.Groups[2].Success ? match.Groups[0].Value : String.Empty}, {DelinterAction.UnescapeLabels, match => String.Format(@"""{0}"":", match.Groups[1].Value)}, {DelinterAction.UnescapeValues, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, match.Groups[2].Value)}, - {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[3].Value)} + {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[2].Value)} }; Pipeline = new Dictionary { {DelinterStep.RemoveNewLine, DelinterAction.Remove}, - {DelinterStep.RemoveWhiteSpace, DelinterAction.Remove}, + //{DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, {DelinterStep.UnescapeLabels, DelinterAction.UnescapeLabels}, {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, From d124622089ba507b8c39ea0c28fd2ad1cb6e4649 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 13:07:18 +0100 Subject: [PATCH 07/11] Issue #AR-4-delinter_fixes: refactor delinter --- .../Delinter/ArchetypeJsonDelinterTest.cs | 2 +- .../Serialization/ArchetypeJsonDelinter.cs | 60 +++++++++++++------ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs index 5a75dd2..e254e4d 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs @@ -56,7 +56,7 @@ namespace Archetype.Tests.Serialization.Delinter { const string _JSON_WITH_RESERVED_CHARS = @"{ ""alias"": ""captions"", - ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \\r \\n test1\r\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\n test2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \r\r\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" + ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \r \n test1\r\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\n test2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \r\r\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" }"; var referenceModel = new Captions() diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index 628ad43..0398952 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -12,6 +12,7 @@ namespace Archetype.Serialization UnescapeAlias, UnescapeValues, FixNestedFieldsets, + FixValueContent } public enum DelinterAction @@ -20,14 +21,15 @@ namespace Archetype.Serialization RemoveWhiteSpace, UnescapeLabels, UnescapeValues, - FixNestedFieldsets + FixNestedFieldsets, + FixValueContent } public class ArchetypeJsonDelinter { public IDictionary Pipeline { get; set; } public IDictionary Tokens { get; set; } - public IDictionary> Actions { get; set; } + public IDictionary> Actions { get; set; } public ArchetypeJsonDelinter() { @@ -41,11 +43,7 @@ namespace Archetype.Serialization foreach (var step in Pipeline.Keys) { var action = Pipeline[step]; - while (Tokens[step].IsMatch(buffer)) - { - buffer = Tokens[step] - .Replace(buffer, match => Actions[action](match)); - } + buffer = Actions[action](buffer, Tokens[step]); } return buffer; @@ -60,27 +58,53 @@ namespace Archetype.Serialization {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":(\s*)")}, {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""")}, {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\s*?\})")}, - {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{\s*?(""fieldsets""[\S\s]+?)}""")} + {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{\s*?(""fieldsets""[\S\s]+?)}""")}, + {DelinterStep.FixValueContent, new Regex(@"""(value)"":""(.*?)(?> + + Actions = new Dictionary> { - {DelinterAction.Remove, match => String.Empty}, - {DelinterAction.RemoveWhiteSpace, match => match.Groups[2].Success ? match.Groups[0].Value : String.Empty}, - {DelinterAction.UnescapeLabels, match => String.Format(@"""{0}"":", match.Groups[1].Value)}, - {DelinterAction.UnescapeValues, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, match.Groups[2].Value)}, - {DelinterAction.FixNestedFieldsets, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[2].Value)} + {DelinterAction.Remove, (input, pattern) => + RecursiveReplace(input, pattern, match => String.Empty) + }, + {DelinterAction.RemoveWhiteSpace, (input, pattern) => + RecursiveReplace(input, pattern, match => match.Groups[2].Success ? match.Groups[0].Value : String.Empty) + }, + {DelinterAction.UnescapeLabels, (input, pattern) => + RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":", match.Groups[1].Value)) + }, + {DelinterAction.UnescapeValues, (input, pattern) => + RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, match.Groups[2].Value)) + }, + {DelinterAction.FixNestedFieldsets, (input, pattern) => + RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[2].Value)) + }, + {DelinterAction.FixValueContent, (input, pattern) => + pattern.Replace(input, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, + (new Regex(@"\\+")).Replace(match.Groups[2].Value, @"\"))) + }, }; Pipeline = new Dictionary { - {DelinterStep.RemoveNewLine, DelinterAction.Remove}, - //{DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, + {DelinterStep.RemoveNewLine, DelinterAction.Remove}, {DelinterStep.UnescapeLabels, DelinterAction.UnescapeLabels}, {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, - {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets} + {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets}, + {DelinterStep.FixValueContent, DelinterAction.FixValueContent} + //{DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, }; } + + private string RecursiveReplace(string input, Regex pattern, Func matchFunc) + { + var buffer = input; + while (pattern.IsMatch(buffer)) + { + buffer = pattern.Replace(input, match => matchFunc(match)); + } + return buffer; + } } } From a306424162b4ea0ab0434c61beaf8c562dd4f457 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 15:05:44 +0100 Subject: [PATCH 08/11] Issue #AR-4-delinter_fixes: fix issue where escaped new lines were being removed inside value quotes --- .../Serialization/Regression/JsonTestStrings.cs | 2 +- .../Serialization/Regression/SerializationTestHelper.cs | 4 +++- .../Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs b/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs index 3e31c01..df58e5d 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Regression/JsonTestStrings.cs @@ -9,7 +9,7 @@ ""properties"": [ { ""alias"": ""text"", - ""value"": ""Test \r\n Text \""quote\"" \r\n"" + ""value"": ""Test \r\nText \""quote\"" \r\n"" }, { ""alias"": ""integer"", diff --git a/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs b/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs index 843580e..d8e8a54 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Regression/SerializationTestHelper.cs @@ -27,7 +27,9 @@ namespace Archetype.Tests.Serialization.Regression DateTwo = null, Id = 123, NullableId = null, - Text = @"Test \r\n Text ""quote"" \r\n" + Text = @"Test +Text ""quote"" +" }; } diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index 0398952..a06cf9f 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -53,7 +53,7 @@ namespace Archetype.Serialization { Tokens = new Dictionary { - {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""""(alias|value|properties|fieldsets)\\*?"""":\{*?(\\*|""""|\[))|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\r|\n)+|(\\+r\\n+)(?!\s+.*?\\+""})")}, + {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""(alias|value|properties|fieldsets)\\*?"":\{*?(\\*|""|\[))|(\r|\n)+|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\\+r\\n+)(?=\s+\\*?""(alias|value|properties|fieldsets))")}, {DelinterStep.RemoveWhiteSpace, new Regex(@"(\s*?)\\+""(fieldsets|properties|alias|value)\\+"":(\s+)|""(\s*?)\},(\s+)\{|\[(\s+)\{|[\]\}](\s+)[\]\}]")}, {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":(\s*)")}, {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""")}, @@ -81,7 +81,7 @@ namespace Archetype.Serialization }, {DelinterAction.FixValueContent, (input, pattern) => pattern.Replace(input, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, - (new Regex(@"\\+")).Replace(match.Groups[2].Value, @"\"))) + (new Regex(@"\\{2,}")).Replace(match.Groups[2].Value, @"\"))) }, }; From 14777a98dc6aa675fc39efc4d0adacedc9f23181 Mon Sep 17 00:00:00 2001 From: cankoluman Date: Tue, 20 May 2014 16:22:47 +0100 Subject: [PATCH 09/11] Issue #AR-4-delinter_fixes: removed buggy escaped json tests --- .../Serialization/Base/RegressionTestBase.cs | 49 +------------ .../Delinter/ArchetypeJsonDelinterTest.cs | 9 ++- .../Regression/ArchetypeJsonConverterTest.cs | 70 ------------------- .../ArcheTypePropertyEditor.cs | 4 +- .../Serialization/ArchetypeJsonDelinter.cs | 16 ++--- 5 files changed, 13 insertions(+), 135 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs index 85021fc..6ff02c5 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs @@ -15,6 +15,7 @@ using Umbraco.Core.Models.Editors; using Umbraco.Core.Persistence; using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; +using Umbraco.Web.PropertyEditors; namespace Archetype.Tests.Serialization.Base { @@ -92,59 +93,11 @@ namespace Archetype.Tests.Serialization.Base AssertAreEqual(model, resultfromArchetypeJson); } - protected void CompoundModel_WithEscapedJson_Regression_Battery(T model) - { - var propGuid = new Guid(); - var dtd = new DataTypeDefinition(-1, String.Empty) {Id = 0}; - - var dataTypeService = new Mock(); - dataTypeService.Setup(dts => dts.GetDataTypeDefinitionById(Guid.Parse(propGuid.ToString()))).Returns(dtd); - dataTypeService.Setup(dts => dts.GetPreValuesCollectionByDataTypeId(dtd.Id)).Returns(new PreValueCollection(new Dictionary())); - - var propValueEditor = new Mock(new PropertyValueEditor()); - propValueEditor.Setup(pe => pe.GetPropertyEditor(dtd)).Returns(new ArchetypePropertyEditor()); - - var archetype = ConvertModelToArchetype(model); - - foreach (var prop in archetype.Fieldsets.SelectMany(fs => fs.Properties)) - { - prop.DataTypeGuid = propGuid.ToString(); - if (prop.Value.ToString().Contains("fieldsets")) - prop.PropertyEditorAlias = Constants.PropertyEditorAlias; - } - - var archetypeJson = JsonConvert.SerializeObject(archetype); - Assert.IsNotNullOrEmpty(archetypeJson); - - EnsureUmbracoContext(dataTypeService); - - var propEditor = new ArchetypePropertyEditor.ArchetypePropertyValueEditor(new PropertyValueEditor()); - var convertedJson = propEditor.ConvertEditorToDb(new ContentPropertyData(JObject.FromObject(archetype), - new PreValueCollection(new Dictionary()), new Dictionary()), - archetypeJson); - - Assert.IsNotNull(convertedJson); - - var resultfromArchetypeJson = ConvertArchetypeJsonToModel(archetypeJson); - var resultfromConvertedJson = ConvertArchetypeJsonToModel(convertedJson.ToString().DelintArchetypeJson()); - - Assert.IsInstanceOf(resultfromArchetypeJson); - AssertAreEqual(model, resultfromArchetypeJson); - - Assert.IsInstanceOf(resultfromConvertedJson); - AssertAreEqual(model, resultfromConvertedJson); - } - protected void NestedModel_Regression_Battery(T model) { CompoundModel_Regression_Battery(model); } - protected void NestedModel_WithEscapedJson_Regression_Battery(T model) - { - CompoundModel_WithEscapedJson_Regression_Battery(model); - } - protected void ComplexModel_Regression_Battery(T model) { CompoundModel_Regression_Battery(model); diff --git a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs index e254e4d..aa06ce0 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs @@ -56,13 +56,16 @@ namespace Archetype.Tests.Serialization.Delinter { const string _JSON_WITH_RESERVED_CHARS = @"{ ""alias"": ""captions"", - ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \r \n test1\r\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\n test2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \r\r\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" + ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \r\n test1\r\n\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\ntest2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \\r\\r\\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" }"; var referenceModel = new Captions() { - new TextString() {Text = @"test1 \\r \\n test1\r"}, - new TextString() {Text = @"test2\r\n test2"}, + new TextString() {Text = @"test1 + test1 +"}, + new TextString() {Text = @"test2 +test2"}, new TextString() {Text = @"test3b \r\r\rtest3"} }; diff --git a/app/Umbraco/Archetype.Tests/Serialization/Regression/ArchetypeJsonConverterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Regression/ArchetypeJsonConverterTest.cs index 4246963..83f4384 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Regression/ArchetypeJsonConverterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Regression/ArchetypeJsonConverterTest.cs @@ -67,13 +67,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModel); } - [Test] - public void CompoundModel_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithMixedFieldsetVariant1_Regression_Battery() { @@ -81,13 +74,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModel); } - [Test] - public void CompoundModelWithMixedFieldsetVariant1_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithMixedFieldsetVariant2_Regression_Battery() { @@ -95,13 +81,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModel); } - [Test] - public void CompoundModelWithMixedFieldsetVariant2_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithFieldset_Regression_Battery() { @@ -109,13 +88,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModel); } - [Test] - public void CompoundModelWithFieldset_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithList_Regression_Battery() { @@ -123,13 +95,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModelWithList); } - [Test] - public void CompoundModelWithList_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithMixedFieldsetWithList_Regression_Battery() { @@ -137,13 +102,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModelWithList); } - [Test] - public void CompoundModelWithMixedFieldsetWithList_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void CompoundModelWithFieldsetWithList_Regression_Battery() { @@ -151,13 +109,6 @@ namespace Archetype.Tests.Serialization.Regression CompoundModel_Regression_Battery(compoundModelWithList); } - [Test] - public void CompoundModelWithFieldsetWithList_WithEscapedJson_Regression_Battery() - { - var compoundModel = _testHelper.GetModel(); - CompoundModel_WithEscapedJson_Regression_Battery(compoundModel); - } - [Test] public void NestedModel_Regression_Battery() { @@ -165,13 +116,6 @@ namespace Archetype.Tests.Serialization.Regression NestedModel_Regression_Battery(nestedModel); } - [Test] - public void NestedModel_WithEscapedJson_Regression_Battery() - { - var nestedModel = _testHelper.GetModel(); - NestedModel_WithEscapedJson_Regression_Battery(nestedModel); - } - [Test] public void NestedModelWithMixedFieldset_Regression_Battery() { @@ -179,13 +123,6 @@ namespace Archetype.Tests.Serialization.Regression NestedModel_Regression_Battery(nestedModel); } - [Test] - public void NestedModelWithMixedFieldset_WithEscapedJson_Regression_Battery() - { - var nestedModel = _testHelper.GetModel(); - NestedModel_WithEscapedJson_Regression_Battery(nestedModel); - } - [Test] public void NestedModelWithFieldset_Regression_Battery() { @@ -193,13 +130,6 @@ namespace Archetype.Tests.Serialization.Regression NestedModel_Regression_Battery(nestedModel); } - [Test] - public void NestedModelWithFieldset_WithEscapedJson_Regression_Battery() - { - var nestedModel = _testHelper.GetModel(); - NestedModel_WithEscapedJson_Regression_Battery(nestedModel); - } - [Test] public void ComplexModel_Regression_Battery() { diff --git a/app/Umbraco/Umbraco.Archetype/PropertyEditors/ArcheTypePropertyEditor.cs b/app/Umbraco/Umbraco.Archetype/PropertyEditors/ArcheTypePropertyEditor.cs index b5b21ba..a4d3e98 100644 --- a/app/Umbraco/Umbraco.Archetype/PropertyEditors/ArcheTypePropertyEditor.cs +++ b/app/Umbraco/Umbraco.Archetype/PropertyEditors/ArcheTypePropertyEditor.cs @@ -126,8 +126,8 @@ namespace Archetype.PropertyEditors { var dtd = ApplicationContext.Current.Services.DataTypeService.GetDataTypeDefinitionById(Guid.Parse(propDef.DataTypeGuid)); var preValues = ApplicationContext.Current.Services.DataTypeService.GetPreValuesCollectionByDataTypeId(dtd.Id); - var propData = new ContentPropertyData(propDef.Value, preValues, new Dictionary()); - var propEditor = GetPropertyEditor(dtd); + var propData = new ContentPropertyData(propDef.Value, preValues, new Dictionary()); + var propEditor = PropertyEditorResolver.Current.GetByAlias(dtd.PropertyEditorAlias); propDef.Value = propEditor.ValueEditor.ConvertEditorToDb(propData, propDef.Value); } catch (Exception ex) diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index a06cf9f..b1b2b63 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -11,8 +11,7 @@ namespace Archetype.Serialization UnescapeLabels, UnescapeAlias, UnescapeValues, - FixNestedFieldsets, - FixValueContent + FixNestedFieldsets } public enum DelinterAction @@ -21,8 +20,7 @@ namespace Archetype.Serialization RemoveWhiteSpace, UnescapeLabels, UnescapeValues, - FixNestedFieldsets, - FixValueContent + FixNestedFieldsets } public class ArchetypeJsonDelinter @@ -58,8 +56,7 @@ namespace Archetype.Serialization {DelinterStep.UnescapeLabels, new Regex(@"\\+""(fieldsets|properties|alias|value)\\+"":(\s*)")}, {DelinterStep.UnescapeAlias, new Regex(@"""(alias)"":\\+""(.*?)\\+""")}, {DelinterStep.UnescapeValues, new Regex(@"""(value)"":\\+""(.*?)\\+""(?=\s*?\})")}, - {DelinterStep.FixNestedFieldsets, new Regex(@"""(value)"":\s*?""\{\s*?(""fieldsets""[\S\s]+?)}""")}, - {DelinterStep.FixValueContent, new Regex(@"""(value)"":""(.*?)(?> @@ -78,11 +75,7 @@ namespace Archetype.Serialization }, {DelinterAction.FixNestedFieldsets, (input, pattern) => RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":{{{1}}}", match.Groups[1].Value, match.Groups[2].Value)) - }, - {DelinterAction.FixValueContent, (input, pattern) => - pattern.Replace(input, match => String.Format(@"""{0}"":""{1}""", match.Groups[1].Value, - (new Regex(@"\\{2,}")).Replace(match.Groups[2].Value, @"\"))) - }, + } }; Pipeline = new Dictionary @@ -92,7 +85,6 @@ namespace Archetype.Serialization {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets}, - {DelinterStep.FixValueContent, DelinterAction.FixValueContent} //{DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, }; } From 0cfcaa336404d5be90035bafd1215907271b153b Mon Sep 17 00:00:00 2001 From: cankoluman Date: Wed, 21 May 2014 14:15:54 +0100 Subject: [PATCH 10/11] Issue #AR-4-delinter_fixes: change heuristic for space removal --- .../Serialization/Base/RegressionTestBase.cs | 10 +------ .../Delinter/ArchetypeJsonDelinterTest.cs | 13 ++++++--- .../Serialization/ArchetypeJsonDelinter.cs | 27 ++++++++++++------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs index 6ff02c5..db9b84b 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Base/RegressionTestBase.cs @@ -1,21 +1,13 @@ -using System; -using System.Collections; -using System.Collections.Generic; +using System.Collections; using System.Linq; using System.Reflection; -using Archetype.PropertyEditors; using Archetype.Serialization; using Moq; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NUnit.Framework; using Umbraco.Core; -using Umbraco.Core.Models; -using Umbraco.Core.Models.Editors; using Umbraco.Core.Persistence; -using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; -using Umbraco.Web.PropertyEditors; namespace Archetype.Tests.Serialization.Base { diff --git a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs index aa06ce0..dcb02a4 100644 --- a/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs +++ b/app/Umbraco/Archetype.Tests/Serialization/Delinter/ArchetypeJsonDelinterTest.cs @@ -56,7 +56,11 @@ namespace Archetype.Tests.Serialization.Delinter { const string _JSON_WITH_RESERVED_CHARS = @"{ ""alias"": ""captions"", - ""value"": ""{\""fieldsets\"":[{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test1 \r\n test1\r\n\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test2\r\ntest2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \\r\\r\\rtest3\""}],\""alias\"":\""textstringArray\""}]}"" + ""value"": ""{ \""fieldsets\"": [ +{ + \""properties\"": + [ +{ \""alias\"": \""textstring\"" , \""value\"":\""test1 \r\n test1\r\n\""}], \""alias\"":\""textstringArray\"" },{\""properties\"": [ { \""alias\"":\""textstring\"",\""value\"":\""test2\r\ntest2\""}],\""alias\"":\""textstringArray\""},{\""properties\"":[{\""alias\"":\""textstring\"",\""value\"":\""test3b \""quote\"" \\r\\r\\rtest3\""}],\""alias\"":\""textstringArray\"" } ] }"" }"; var referenceModel = new Captions() @@ -66,10 +70,13 @@ namespace Archetype.Tests.Serialization.Delinter "}, new TextString() {Text = @"test2 test2"}, - new TextString() {Text = @"test3b \r\r\rtest3"} + new TextString() {Text = @"test3b ""quote"" \r\r\rtest3"} }; - var actualModel = ConvertArchetypeJsonToModel(_JSON_WITH_RESERVED_CHARS.DelintArchetypeJson()); + var delintedJson = _JSON_WITH_RESERVED_CHARS.DelintArchetypeJson(); + var actualModel = ConvertArchetypeJsonToModel(delintedJson); + + var t = JToken.Parse(delintedJson).ToString(Formatting.None); Assert.IsInstanceOf(actualModel); Assert.IsNotNull(actualModel); diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index b1b2b63..894a442 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -1,13 +1,14 @@ using System; +using System.Linq; using System.Collections.Generic; using System.Text.RegularExpressions; namespace Archetype.Serialization { public enum DelinterStep - { - RemoveWhiteSpace = 0, - RemoveNewLine, + { + RemoveNewLine = 0, + RemoveWhiteSpace, UnescapeLabels, UnescapeAlias, UnescapeValues, @@ -25,9 +26,9 @@ namespace Archetype.Serialization public class ArchetypeJsonDelinter { - public IDictionary Pipeline { get; set; } - public IDictionary Tokens { get; set; } - public IDictionary> Actions { get; set; } + public IDictionary Pipeline { get; private set; } + public IDictionary Tokens { get; private set; } + public IDictionary> Actions { get; private set; } public ArchetypeJsonDelinter() { @@ -52,7 +53,7 @@ namespace Archetype.Serialization Tokens = new Dictionary { {DelinterStep.RemoveNewLine, new Regex(@"(?<=\,){0,1}(\\+r\\+n)(?=\\*?""(alias|value|properties|fieldsets)\\*?"":\{*?(\\*|""|\[))|(\r|\n)+|(\\+r\\+n)(?=\s*?(\{|\}|\]))|(\\+r\\n+)(?=\s+\\*?""(alias|value|properties|fieldsets))")}, - {DelinterStep.RemoveWhiteSpace, new Regex(@"(\s*?)\\+""(fieldsets|properties|alias|value)\\+"":(\s+)|""(\s*?)\},(\s+)\{|\[(\s+)\{|[\]\}](\s+)[\]\}]")}, + {DelinterStep.RemoveWhiteSpace, new Regex(@"(""\S+?"":)(\s+?)("")|([\{\[\}\],]|(? String.Empty) }, {DelinterAction.RemoveWhiteSpace, (input, pattern) => - RecursiveReplace(input, pattern, match => match.Groups[2].Success ? match.Groups[0].Value : String.Empty) + RecursiveReplace(input.Trim(), pattern, match => + String.Join(String.Empty, + match.Groups + .OfType() + .Select((g, i) => String.IsNullOrWhiteSpace(g.Value) ? String.Empty : g.Value) + .Skip(1) + .ToArray())) }, {DelinterAction.UnescapeLabels, (input, pattern) => RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":", match.Groups[1].Value)) @@ -85,7 +92,7 @@ namespace Archetype.Serialization {DelinterStep.UnescapeAlias, DelinterAction.UnescapeValues}, {DelinterStep.UnescapeValues, DelinterAction.UnescapeValues}, {DelinterStep.FixNestedFieldsets, DelinterAction.FixNestedFieldsets}, - //{DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, + {DelinterStep.RemoveWhiteSpace, DelinterAction.RemoveWhiteSpace}, }; } @@ -94,7 +101,7 @@ namespace Archetype.Serialization var buffer = input; while (pattern.IsMatch(buffer)) { - buffer = pattern.Replace(input, match => matchFunc(match)); + buffer = pattern.Replace(buffer, match => matchFunc(match)); } return buffer; } From 11791470417151296e04127d459699685f54121d Mon Sep 17 00:00:00 2001 From: cankoluman Date: Wed, 21 May 2014 14:22:11 +0100 Subject: [PATCH 11/11] Issue #AR-4-delinter_fixes: simplify replace --- .../Serialization/ArchetypeJsonDelinter.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs index 894a442..ea2c55c 100644 --- a/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs +++ b/app/Umbraco/Umbraco.Archetype/Serialization/ArchetypeJsonDelinter.cs @@ -67,12 +67,8 @@ namespace Archetype.Serialization }, {DelinterAction.RemoveWhiteSpace, (input, pattern) => RecursiveReplace(input.Trim(), pattern, match => - String.Join(String.Empty, - match.Groups - .OfType() - .Select((g, i) => String.IsNullOrWhiteSpace(g.Value) ? String.Empty : g.Value) - .Skip(1) - .ToArray())) + String.Format("{0}{1}{2}{3}", match.Groups[1].Value, + match.Groups[3].Value, match.Groups[4].Value, match.Groups[6].Value)) }, {DelinterAction.UnescapeLabels, (input, pattern) => RecursiveReplace(input, pattern, match => String.Format(@"""{0}"":", match.Groups[1].Value))