Merge branch 'dev-v7.6' of https://github.com/umbraco/Umbraco-CMS into dev-v7.6

U4-9497 Update Related Links Property Editor to support UDIs

build/NuSpecs/UmbracoCms.Core.nuspec

@@ -24,7 +24,7 @@
       <dependency id="Microsoft.Owin.Security.OAuth" version="[3.0.1, 4.0.0)" />
       <dependency id="Microsoft.Owin.Host.SystemWeb" version="[3.0.1, 4.0.0)" />
       <dependency id="MiniProfiler" version="[2.1.0, 3.0.0)" />
-      <dependency id="HtmlAgilityPack" version="[1.4.9, 2.0.0)" />
+      <dependency id="HtmlAgilityPack" version="[1.4.9.5, 2.0.0)" />
       <dependency id="Lucene.Net" version="[2.9.4.1, 3.0.0.0)" />
       <dependency id="SharpZipLib" version="[0.86.0, 1.0.0)" />
       <dependency id="MySql.Data" version="[6.9.8, 7.0.0)" />
@@ -37,9 +37,8 @@
       <dependency id="ImageProcessor" version="[2.5.2, 3.0.0)" />
       <dependency id="ImageProcessor.Web" version="[4.8.2, 5.0.0)" />
       <dependency id="semver" version="[2.0.4, 3.0.0)" />
-      <dependency id="UrlRewritingNet" version="[2.0.7, 3.0.0)" />	  
 	  <!-- Markdown can not be updated due to: https://github.com/hey-red/markdownsharp/issues/71#issuecomment-233585487 -->
-      <dependency id="Markdown" version="[1.14.7, 2.0.0)" />	  
+      <dependency id="Markdown" version="[1.14.7, 2.0.0)" />
       <dependency id="System.Threading.Tasks.Dataflow" version="[4.6.0, 5.0.0)" />
     </dependencies>
   </metadata>

build/NuSpecs/tools/Dashboard.config.install.xdt

@@ -41,16 +41,6 @@
         views/dashboard/developer/examinemanagement.html
       </control>
     </tab>
-    <tab caption="Health Check" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/healthcheck.html
-      </control>
-    </tab>
-    <tab caption="Redirect URL Management" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/redirecturls.html
-      </control>
-    </tab>	
   </section>
   
   <section alias="StartupMediaDashboardSection" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
@@ -80,4 +70,26 @@
 	<tab caption="Change Password" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
 	<tab caption="Last Edits" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
   </section>
+
+  <section alias="RedirectUrlManagement" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>content</area>
+    </areas>
+    <tab caption="Redirect URL Management">
+      <control>
+        views/dashboard/developer/redirecturls.html
+      </control>
+    </tab>
+  </section>
+
+  <section alias="UmbracoHealthCheck" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>developer</area>
+    </areas>
+    <tab caption="Health Check">
+      <control>
+        views/dashboard/developer/healthcheck.html
+      </control>
+    </tab>
+  </section>
 </dashBoard>

build/NuSpecs/tools/install.core.ps1

@@ -53,7 +53,6 @@ if ($project) {
 		if(Test-Path $umbracoBinFolder\umbraco.providers.dll) { Remove-Item $umbracoBinFolder\umbraco.providers.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\Umbraco.Web.UI.dll) { Remove-Item $umbracoBinFolder\Umbraco.Web.UI.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\UmbracoExamine.dll) { Remove-Item $umbracoBinFolder\UmbracoExamine.dll -Force -Confirm:$false }
-		if(Test-Path $umbracoBinFolder\UrlRewritingNet.UrlRewriter.dll) { Remove-Item $umbracoBinFolder\UrlRewritingNet.UrlRewriter.dll -Force -Confirm:$false }
 		
 		# Delete files Umbraco depends upon
 		$amd64Folder = Join-Path $umbracoBinFolder "amd64"

build/UmbracoVersion.txt

@@ -1,3 +1,3 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
 7.6.0
-alpha064
+alpha075

src/SQLCE4Umbraco/SqlCE4Umbraco.csproj

@@ -10,7 +10,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>SQLCE4Umbraco</RootNamespace>
     <AssemblyName>SQLCE4Umbraco</AssemblyName>
-    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <SccProjectName>
     </SccProjectName>
@@ -49,9 +49,11 @@
     <Reference Include="System.Core" />
     <Reference Include="System.Data.SqlServerCe, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.Data.SqlServerCe.Entity, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.Entity.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />

src/SQLCE4Umbraco/app.config

@@ -32,4 +32,4 @@
       </dependentAssembly>
     </assemblyBinding>
   </runtime>
-<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" /></startup></configuration>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup></configuration>

src/SolutionInfo.cs

@@ -12,4 +12,4 @@ using System.Resources;
 [assembly: AssemblyVersion("1.0.*")]
 
 [assembly: AssemblyFileVersion("7.6.0")]
-[assembly: AssemblyInformationalVersion("7.6.0-alpha064")]
+[assembly: AssemblyInformationalVersion("7.6.0-alpha075")]

src/Umbraco.Core/Configuration/ContentXmlStorage.cs

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Configuration
+{
+    internal enum ContentXmlStorage
+    {
+        Default,
+        AspNetTemp,
+        EnvironmentTemp
+    }
+}

src/Umbraco.Core/Configuration/CoreDebug.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace Umbraco.Core.Configuration
+{
+    internal static class CoreDebugExtensions
+    {
+        private static CoreDebug _coreDebug;
+
+        public static CoreDebug CoreDebug(this UmbracoConfig config)
+        {
+            return _coreDebug ?? (_coreDebug = new CoreDebug());
+        }
+    }
+
+    internal class CoreDebug
+    {
+        public CoreDebug()
+        {
+            var appSettings = System.Configuration.ConfigurationManager.AppSettings;
+            LogUncompletedScopes = string.Equals("true", appSettings["Umbraco.CoreDebug.LogUncompletedScopes"], StringComparison.OrdinalIgnoreCase);
+        }
+
+        // when true, Scope logs the stack trace for any scope that gets disposed without being completed.
+        // this helps troubleshooting rogue scopes that we forget to complete
+        public bool LogUncompletedScopes { get; private set; }
+    }
+}

src/Umbraco.Core/Configuration/GlobalSettings.cs

@@ -517,12 +517,25 @@ namespace Umbraco.Core.Configuration
         }
 
         internal static bool ContentCacheXmlStoredInCodeGen
+        {
+            get { return ContentCacheXmlStorageLocation == ContentXmlStorage.AspNetTemp; }
+        }
+
+        internal static ContentXmlStorage ContentCacheXmlStorageLocation
         {
             get
             {
-                //defaults to false
-                return ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp") 
-                    && bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]); //default to false
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLStorage"))
+                {
+                    return Enum<ContentXmlStorage>.Parse(ConfigurationManager.AppSettings["umbracoContentXMLStorage"]);
+                }
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp"))
+                {
+                    return bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]) 
+                        ? ContentXmlStorage.AspNetTemp 
+                        : ContentXmlStorage.Default;
+                }
+                return ContentXmlStorage.Default;
             }
         }
 

src/Umbraco.Core/Configuration/UmbracoSettings/ContentElement.cs

@@ -177,6 +177,12 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
             get { return GetOptionalTextElement("EnableInheritedMediaTypes", true); }
         }
 
+        [ConfigurationProperty("EnablePropertyValueConverters")]
+        internal InnerTextConfigurationElement<bool> EnablePropertyValueConverters
+        {
+            get { return GetOptionalTextElement("EnablePropertyValueConverters", false); }
+        }
+
         string IContentSection.NotificationEmailAddress
         {
             get { return Notifications.NotificationEmailAddress; }
@@ -326,5 +332,9 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
         {
             get { return EnableInheritedMediaTypes; }
         }
+        bool IContentSection.EnablePropertyValueConverters
+        {
+            get { return EnablePropertyValueConverters; }
+        }
     }
 }

src/Umbraco.Core/Configuration/UmbracoSettings/IContentSection.cs

@@ -69,5 +69,8 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
         bool EnableInheritedDocumentTypes { get; }
 
         bool EnableInheritedMediaTypes { get; }
+
+        bool EnablePropertyValueConverters { get; }
+
     }
 }

src/Umbraco.Core/Configuration/UmbracoVersion.cs

@@ -24,7 +24,7 @@ namespace Umbraco.Core.Configuration
         /// Gets the version comment (like beta or RC).
         /// </summary>
         /// <value>The version comment.</value>
-        public static string CurrentComment { get { return "alpha064"; } }
+        public static string CurrentComment { get { return "alpha075"; } }
 
         // Get the version of the umbraco.dll by looking at a class in that dll
         // Had to do it like this due to medium trust issues, see: http://haacked.com/archive/2010/11/04/assembly-location-and-medium-trust.aspx

src/Umbraco.Core/Constants-ObjectTypes.cs

@@ -109,6 +109,11 @@ namespace Umbraco.Core
             /// </summary>
             public const string Member = "39EB0F98-B348-42A1-8662-E7EB18487560";
 
+            /// <summary>
+            /// Guid for a Media Type object.
+            /// </summary>
+            public static readonly Guid MemberGuid = new Guid(Member);
+
             /// <summary>
             /// Guid for a Member Group object.
             /// </summary>

src/Umbraco.Core/Constants-PropertyEditors.cs

@@ -289,11 +289,14 @@ namespace Umbraco.Core
             /// </summary>
             [Obsolete("GUIDs are no longer used to reference Property Editors, use the Alias constant instead. This will be removed in future versions")]
             public const string RelatedLinks = "71B8AD1A-8DC2-425C-B6B8-FAA158075E63";
+            
+            [Obsolete("This is an obsoleted picker, use RelatedLinks2Alias instead")]
+            public const string RelatedLinksAlias = "Umbraco.RelatedLinks";
 
             /// <summary>
-            /// Alias for the Related Links datatype.
+            /// Alias for the Related Links property editor.
             /// </summary>
-            public const string RelatedLinksAlias = "Umbraco.RelatedLinks";
+            public const string RelatedLinks2Alias = "Umbraco.RelatedLinks2";
 
             /// <summary>
             /// Guid for the Slider datatype.

src/Umbraco.Core/Deploy/ArtifactBase.cs

@@ -16,6 +16,7 @@ namespace Umbraco.Core.Deploy
             if (udi == null)
                 throw new ArgumentNullException("udi");
             Udi = udi;
+            Name = Udi.ToString();
 
             Dependencies = dependencies ?? Enumerable.Empty<ArtifactDependency>();
             _checksum = new Lazy<string>(GetChecksum);
@@ -48,5 +49,8 @@ namespace Umbraco.Core.Deploy
         }
 
         #endregion
+
+        public string Name { get; set; }
+        public string Alias { get; set; }
     }
 }

src/Umbraco.Core/Deploy/IArtifact.cs

@@ -4,5 +4,8 @@ namespace Umbraco.Core.Deploy
     /// Represents an artifact ie an object that can be transfered between environments.
     /// </summary>
     public interface IArtifact : IArtifactSignature
-    { }
+    {
+        string Name { get; }
+        string Alias { get; }
+    }
 }

src/Umbraco.Core/Deploy/IFileSource.cs

@@ -66,10 +66,20 @@ namespace Umbraco.Core.Deploy
         /// <returns>The length of the file, or -1 if the file does not exist.</returns>
         Task<long> GetFileLengthAsync(StringUdi udi, CancellationToken token);
 
-        // fixme - doc
-        void GetFiles(IEnumerable<StringUdi> udis, IFileStore fileStore);
+        /// <summary>
+        /// Gets files and store them using a file store.
+        /// </summary>
+        /// <param name="udis">The udis of the files to get.</param>
+        /// <param name="fileTypes">A collection of file types which can store the files.</param>
+        void GetFiles(IEnumerable<StringUdi> udis, IFileTypeCollection fileTypes);
 
-        Task GetFilesAsync(IEnumerable<StringUdi> udis, IFileStore fileStore, CancellationToken token);
+        /// <summary>
+        /// Gets files and store them using a file store.
+        /// </summary>
+        /// <param name="udis">The udis of the files to get.</param>
+        /// <param name="fileTypes">A collection of file types which can store the files.</param>
+        /// <param name="token">A cancellation token.</param>
+        Task GetFilesAsync(IEnumerable<StringUdi> udis, IFileTypeCollection fileTypes, CancellationToken token);
 
         ///// <summary>
         ///// Gets the content of a file as a bytes array.

src/Umbraco.Core/Deploy/IFileStore.cs

@@ -1,9 +0,0 @@
-using System.IO;
-
-namespace Umbraco.Core.Deploy
-{
-    public interface IFileStore
-    {
-        void SaveStream(StringUdi udi, Stream stream);
-    }
-}

src/Umbraco.Core/Deploy/IFileType.cs

@@ -0,0 +1,32 @@
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Umbraco.Core.Deploy
+{
+    public interface IFileType
+    {
+        Stream GetStream(StringUdi udi);
+
+        Task<Stream> GetStreamAsync(StringUdi udi, CancellationToken token);
+
+        Stream GetChecksumStream(StringUdi udi);
+
+        long GetLength(StringUdi udi);
+
+        void SetStream(StringUdi udi, Stream stream);
+
+        Task SetStreamAsync(StringUdi udi, Stream stream, CancellationToken token);
+
+        bool CanSetPhysical { get; }
+
+        void Set(StringUdi udi, string physicalPath, bool copy = false);
+
+        // this is not pretty as *everywhere* in Deploy we take care of ignoring
+        // the physical path and always rely on Core's virtual IFileSystem but
+        // Cloud wants to add some of these files to Git and needs the path...
+        string GetPhysicalPath(StringUdi udi);
+
+        string GetVirtualPath(StringUdi udi);
+    }
+}

src/Umbraco.Core/Deploy/IFileTypeCollection.cs

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Deploy
+{
+    public interface IFileTypeCollection
+    {
+        IFileType this[string entityType] { get; }
+
+        bool Contains(string entityType);
+    }
+}

src/Umbraco.Core/Deploy/IPrettyArtifact.cs

@@ -1,8 +0,0 @@
-namespace Umbraco.Core.Deploy
-{
-    public interface IPrettyArtifact
-    {
-        string Name { get; }
-        string Alias { get; }
-    }
-}

src/Umbraco.Core/EnumerableExtensions.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Text;
@@ -67,22 +68,15 @@ namespace Umbraco.Core
             }
         }
 
-        /// <summary>The for each.</summary>
-        /// <param name="items">The items.</param>
-        /// <param name="func">The func.</param>
-        /// <typeparam name="TItem">item type</typeparam>
-        /// <typeparam name="TResult">Result type</typeparam>
-        /// <returns>the Results</returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Use a normal foreach loop instead, this adds more allocations than necessary")]
         public static TResult[] ForEach<TItem, TResult>(this IEnumerable<TItem> items, Func<TItem, TResult> func)
         {
             return items.Select(func).ToArray();
         }
 
-        /// <summary>The for each.</summary>
-        /// <param name="items">The items.</param>
-        /// <param name="action">The action.</param>
-        /// <typeparam name="TItem">Item type</typeparam>
-        /// <returns>list of TItem</returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Use a normal foreach loop instead, this adds more allocations than necessary")]
         public static IEnumerable<TItem> ForEach<TItem>(this IEnumerable<TItem> items, Action<TItem> action)
         {
             if (items != null)
@@ -101,6 +95,7 @@ namespace Umbraco.Core
         /// <param name="f">The select child.</param>
         /// <typeparam name="T">Item type</typeparam>
         /// <returns>list of TItem</returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("Do not use, use SelectRecursive instead which has far less potential of re-iterating an iterator which may cause significantly more SQL queries")]
         public static IEnumerable<T> FlattenList<T>(this IEnumerable<T> e, Func<T, IEnumerable<T>> f)
         {
@@ -301,5 +296,21 @@ namespace Umbraco.Core
                && list1Groups.All(g => g.Count() == list2Groups[g.Key].Count());
         }
 
+        ///<summary>
+        /// Returns the items of the given enumerable as a pure enumerable.
+        /// <remarks>
+        /// When quering lists using methods such as <see cref="M:List.Where"/>, the result, despite appearing to look like and quack like an 
+        /// <see cref="T:Enumerable{T}"/> the type is actually an instance of <see cref="T:System.Linq.Enumerable.WhereEnumerableIterator"/>
+        /// </remarks>
+        /// </summary>
+        ///<param name="source">The item to find.</param>
+        ///<returns>The index of the first matching item, or -1 if the item was not found.</returns>
+        internal static IEnumerable<T> Yield<T>(this IEnumerable<T> source)
+        {
+            foreach (var element in source)
+            {
+                yield return element;
+            }
+        }
     }
 }

src/Umbraco.Core/Events/ScopeLifespanMessagesFactory.cs

@@ -0,0 +1,58 @@
+using System;
+using Umbraco.Core.Scoping;
+
+namespace Umbraco.Core.Events
+{
+    /// <summary>
+    /// Stores the instance of EventMessages in the current scope.
+    /// </summary>
+    internal class ScopeLifespanMessagesFactory : IEventMessagesFactory
+    {
+        public const string ContextKey = "Umbraco.Core.Events.ScopeLifespanMessagesFactory";
+
+        private readonly IHttpContextAccessor _contextAccessor;
+        private readonly IScopeProviderInternal _scopeProvider;
+
+        public static ScopeLifespanMessagesFactory Current { get; private set; }
+
+        public ScopeLifespanMessagesFactory(IHttpContextAccessor contextAccesor, IScopeProvider scopeProvider)
+        {
+            if (contextAccesor == null) throw new ArgumentNullException("contextAccesor");
+            if (scopeProvider == null) throw new ArgumentNullException("scopeProvider");
+            if (scopeProvider is IScopeProviderInternal == false) throw new ArgumentException("Not IScopeProviderInternal.", "scopeProvider");
+            _contextAccessor = contextAccesor;
+            _scopeProvider = (IScopeProviderInternal) scopeProvider;
+            Current = this;
+        }
+
+        public EventMessages Get()
+        {
+            var messages = GetFromHttpContext();
+            if (messages != null) return messages;
+
+            var scope = _scopeProvider.GetAmbientOrNoScope();
+            return scope.Messages;
+        }
+
+        public EventMessages GetFromHttpContext()
+        {
+            if (_contextAccessor == null || _contextAccessor.Value == null) return null;
+            return (EventMessages)_contextAccessor.Value.Items[ContextKey];
+        }
+
+        public EventMessages TryGet()
+        {
+            var messages = GetFromHttpContext();
+            if (messages != null) return messages;
+
+            var scope = _scopeProvider.AmbientScope;
+            return scope == null ? null : scope.MessagesOrNull;
+        }
+
+        public void Set(EventMessages messages)
+        {
+            if (_contextAccessor.Value == null) return;
+            _contextAccessor.Value.Items[ContextKey] = messages;
+        }
+    }
+}

src/Umbraco.Core/GuidUdi.cs

@@ -1,10 +1,12 @@
 using System;
+using System.ComponentModel;
 
 namespace Umbraco.Core
 {
     /// <summary>
     /// Represents a guid-based entity identifier.
     /// </summary>
+    [TypeConverter(typeof(UdiTypeConverter))]
     public class GuidUdi : Udi
     {
         /// <summary>

src/Umbraco.Core/IHttpContextAccessor.cs

@@ -0,0 +1,9 @@
+using System.Web;
+
+namespace Umbraco.Core
+{
+    public interface IHttpContextAccessor
+    {
+        HttpContextBase Value { get; }
+    }
+}

src/Umbraco.Core/IO/FileSystemProviderManager.cs

@@ -12,25 +12,18 @@ namespace Umbraco.Core.IO
     public class FileSystemProviderManager
     {
         private readonly FileSystemProvidersSection _config;
-        private readonly WeakSet<ShadowWrapper> _wrappers = new WeakSet<ShadowWrapper>();
+        private readonly ConcurrentSet<ShadowWrapper> _wrappers = new ConcurrentSet<ShadowWrapper>();
 
-        // actual well-known filesystems returned by properties
-        private readonly IFileSystem2 _macroPartialFileSystem;
-        private readonly IFileSystem2 _partialViewsFileSystem;
-        private readonly IFileSystem2 _stylesheetsFileSystem;
-        private readonly IFileSystem2 _scriptsFileSystem;
-        private readonly IFileSystem2 _xsltFileSystem;
-        private readonly IFileSystem2 _masterPagesFileSystem;
-        private readonly IFileSystem2 _mvcViewsFileSystem;
+        private readonly ConcurrentDictionary<string, ProviderConstructionInfo> _providerLookup = new ConcurrentDictionary<string, ProviderConstructionInfo>();
+        private readonly ConcurrentDictionary<string, IFileSystem2> _filesystems = new ConcurrentDictionary<string, IFileSystem2>();
 
-        // when shadowing is enabled, above filesystems, as wrappers
-        private readonly ShadowWrapper _macroPartialFileSystemWrapper;
-        private readonly ShadowWrapper _partialViewsFileSystemWrapper;
-        private readonly ShadowWrapper _stylesheetsFileSystemWrapper;
-        private readonly ShadowWrapper _scriptsFileSystemWrapper;
-        private readonly ShadowWrapper _xsltFileSystemWrapper;
-        private readonly ShadowWrapper _masterPagesFileSystemWrapper;
-        private readonly ShadowWrapper _mvcViewsFileSystemWrapper;
+        private ShadowWrapper _macroPartialFileSystem;
+        private ShadowWrapper _partialViewsFileSystem;
+        private ShadowWrapper _stylesheetsFileSystem;
+        private ShadowWrapper _scriptsFileSystem;
+        private ShadowWrapper _xsltFileSystem;
+        private ShadowWrapper _masterPagesFileSystem;
+        private ShadowWrapper _mvcViewsFileSystem;
 
         #region Singleton & Constructor
 
@@ -41,31 +34,45 @@ namespace Umbraco.Core.IO
             get { return Instance; }
         }
 
+        // for tests only, totally unsafe
+        internal void Reset()
+        {
+            _wrappers.Clear();
+            _providerLookup.Clear();
+            _filesystems.Clear();
+            CreateWellKnownFileSystems();
+        }
+
         private IScopeProviderInternal ScopeProvider
         {
             // fixme - 'course this is bad, but enough for now
+            // beware: means that we capture the "current" scope provider - take care in tests!
             get { return ApplicationContext.Current == null ? null : ApplicationContext.Current.ScopeProvider as IScopeProviderInternal; }
         }
 
         internal FileSystemProviderManager()
         {
             _config = (FileSystemProvidersSection) ConfigurationManager.GetSection("umbracoConfiguration/FileSystemProviders");
+            CreateWellKnownFileSystems();
+        }
 
-            _macroPartialFileSystem = new PhysicalFileSystem(SystemDirectories.MacroPartials);
-            _partialViewsFileSystem = new PhysicalFileSystem(SystemDirectories.PartialViews);
-            _stylesheetsFileSystem = new PhysicalFileSystem(SystemDirectories.Css);
-            _scriptsFileSystem = new PhysicalFileSystem(SystemDirectories.Scripts);
-            _xsltFileSystem = new PhysicalFileSystem(SystemDirectories.Xslt);
-            _masterPagesFileSystem = new PhysicalFileSystem(SystemDirectories.Masterpages);
-            _mvcViewsFileSystem = new PhysicalFileSystem(SystemDirectories.MvcViews);
+        private void CreateWellKnownFileSystems()
+        {
+            var macroPartialFileSystem = new PhysicalFileSystem(SystemDirectories.MacroPartials);
+            var partialViewsFileSystem = new PhysicalFileSystem(SystemDirectories.PartialViews);
+            var stylesheetsFileSystem = new PhysicalFileSystem(SystemDirectories.Css);
+            var scriptsFileSystem = new PhysicalFileSystem(SystemDirectories.Scripts);
+            var xsltFileSystem = new PhysicalFileSystem(SystemDirectories.Xslt);
+            var masterPagesFileSystem = new PhysicalFileSystem(SystemDirectories.Masterpages);
+            var mvcViewsFileSystem = new PhysicalFileSystem(SystemDirectories.MvcViews);
 
-            _macroPartialFileSystem = _macroPartialFileSystemWrapper = new ShadowWrapper(_macroPartialFileSystem, "Views/MacroPartials", ScopeProvider);
-            _partialViewsFileSystem = _partialViewsFileSystemWrapper = new ShadowWrapper(_partialViewsFileSystem, "Views/Partials", ScopeProvider);
-            _stylesheetsFileSystem = _stylesheetsFileSystemWrapper = new ShadowWrapper(_stylesheetsFileSystem, "css", ScopeProvider);
-            _scriptsFileSystem = _scriptsFileSystemWrapper = new ShadowWrapper(_scriptsFileSystem, "scripts", ScopeProvider);
-            _xsltFileSystem = _xsltFileSystemWrapper = new ShadowWrapper(_xsltFileSystem, "xslt", ScopeProvider);
-            _masterPagesFileSystem = _masterPagesFileSystemWrapper = new ShadowWrapper(_masterPagesFileSystem, "masterpages", ScopeProvider);
-            _mvcViewsFileSystem = _mvcViewsFileSystemWrapper = new ShadowWrapper(_mvcViewsFileSystem, "Views", ScopeProvider);
+            _macroPartialFileSystem = new ShadowWrapper(macroPartialFileSystem, "Views/MacroPartials", ScopeProvider);
+            _partialViewsFileSystem = new ShadowWrapper(partialViewsFileSystem, "Views/Partials", ScopeProvider);
+            _stylesheetsFileSystem = new ShadowWrapper(stylesheetsFileSystem, "css", ScopeProvider);
+            _scriptsFileSystem = new ShadowWrapper(scriptsFileSystem, "scripts", ScopeProvider);
+            _xsltFileSystem = new ShadowWrapper(xsltFileSystem, "xslt", ScopeProvider);
+            _masterPagesFileSystem = new ShadowWrapper(masterPagesFileSystem, "masterpages", ScopeProvider);
+            _mvcViewsFileSystem = new ShadowWrapper(mvcViewsFileSystem, "Views", ScopeProvider);
 
             // filesystems obtained from GetFileSystemProvider are already wrapped and do not need to be wrapped again
             MediaFileSystem = GetFileSystemProvider<MediaFileSystem>();
@@ -80,7 +87,7 @@ namespace Umbraco.Core.IO
         public IFileSystem2 StylesheetsFileSystem { get { return _stylesheetsFileSystem; } }
         public IFileSystem2 ScriptsFileSystem { get { return _scriptsFileSystem; } }
         public IFileSystem2 XsltFileSystem { get { return _xsltFileSystem; } }
-        public IFileSystem2 MasterPagesFileSystem { get { return _masterPagesFileSystem; } }
+        public IFileSystem2 MasterPagesFileSystem { get { return _mvcViewsFileSystem; } }
         public IFileSystem2 MvcViewsFileSystem { get { return _mvcViewsFileSystem; } }
         public MediaFileSystem MediaFileSystem { get; private set; }
 
@@ -98,56 +105,74 @@ namespace Umbraco.Core.IO
 			//public string ProviderAlias { get; set; }
 		}
 
-		private readonly ConcurrentDictionary<string, ProviderConstructionInfo> _providerLookup = new ConcurrentDictionary<string, ProviderConstructionInfo>();
-		private readonly ConcurrentDictionary<Type, string> _aliases = new ConcurrentDictionary<Type, string>();
-
         /// <summary>
         /// Gets an underlying (non-typed) filesystem supporting a strongly-typed filesystem.
         /// </summary>
         /// <param name="alias">The alias of the strongly-typed filesystem.</param>
         /// <returns>The non-typed filesystem supporting the strongly-typed filesystem with the specified alias.</returns>
-        /// <remarks>This method should not be used directly, used <see cref="GetFileSystemProvider{TFileSystem}"/> instead.</remarks>
+        /// <remarks>This method should not be used directly, used <see cref="GetFileSystemProvider{TFileSystem}()"/> instead.</remarks>
         public IFileSystem GetUnderlyingFileSystemProvider(string alias)
+        {
+            return GetUnderlyingFileSystemProvider(alias, null);
+        }
+
+        /// <summary>
+        /// Gets an underlying (non-typed) filesystem supporting a strongly-typed filesystem.
+        /// </summary>
+        /// <param name="alias">The alias of the strongly-typed filesystem.</param>
+        /// /// <param name="fallback">A fallback creator for the filesystem.</param>
+        /// <returns>The non-typed filesystem supporting the strongly-typed filesystem with the specified alias.</returns>
+        /// <remarks>This method should not be used directly, used <see cref="GetFileSystemProvider{TFileSystem}()"/> instead.</remarks>
+        private IFileSystem GetUnderlyingFileSystemProvider(string alias, Func<IFileSystem> fallback)
         {
 			// either get the constructor info from cache or create it and add to cache
-	        var ctorInfo = _providerLookup.GetOrAdd(alias, s =>
-		        {
-                    // get config
-			        var providerConfig = _config.Providers[s];
-			        if (providerConfig == null)
-				        throw new ArgumentException(string.Format("No provider found with alias {0}.", s));
+            var ctorInfo = _providerLookup.GetOrAdd(alias, _ => GetUnderlyingFileSystemCtor(alias, fallback));
+            return ctorInfo == null ? fallback() : (IFileSystem) ctorInfo.Constructor.Invoke(ctorInfo.Parameters);
+        }
 
-                    // get the filesystem type
-			        var providerType = Type.GetType(providerConfig.Type);
-			        if (providerType == null)
-				        throw new InvalidOperationException(string.Format("Could not find type {0}.", providerConfig.Type));
+        private IFileSystem GetUnderlyingFileSystemNoCache(string alias, Func<IFileSystem> fallback)
+        {
+            var ctorInfo = GetUnderlyingFileSystemCtor(alias, fallback);
+            return ctorInfo == null ? fallback() : (IFileSystem) ctorInfo.Constructor.Invoke(ctorInfo.Parameters);
+        }
 
-                    // ensure it implements IFileSystem
-			        if (providerType.IsAssignableFrom(typeof (IFileSystem)))
-				        throw new InvalidOperationException(string.Format("Type {0} does not implement IFileSystem.", providerType.FullName));
+        private ProviderConstructionInfo GetUnderlyingFileSystemCtor(string alias, Func<IFileSystem> fallback)
+        {
+            // get config
+            var providerConfig = _config.Providers[alias];
+            if (providerConfig == null)
+            {
+                if (fallback != null) return null;
+                throw new ArgumentException(string.Format("No provider found with alias {0}.", alias));
+            }
 
-                    // find a ctor matching the config parameters
-			        var paramCount = providerConfig.Parameters != null ? providerConfig.Parameters.Count : 0;
-			        var constructor = providerType.GetConstructors().SingleOrDefault(x
-                        => x.GetParameters().Length == paramCount && x.GetParameters().All(y => providerConfig.Parameters.AllKeys.Contains(y.Name)));
-			        if (constructor == null)
-				        throw new InvalidOperationException(string.Format("Type {0} has no ctor matching the {1} configuration parameter(s).", providerType.FullName, paramCount));
+            // get the filesystem type
+            var providerType = Type.GetType(providerConfig.Type);
+            if (providerType == null)
+                throw new InvalidOperationException(string.Format("Could not find type {0}.", providerConfig.Type));
 
-			        var parameters = new object[paramCount];
-                    if (providerConfig.Parameters != null) // keeps ReSharper happy
-			            for (var i = 0; i < paramCount; i++)
-				            parameters[i] = providerConfig.Parameters[providerConfig.Parameters.AllKeys[i]].Value;
+            // ensure it implements IFileSystem
+            if (providerType.IsAssignableFrom(typeof(IFileSystem)))
+                throw new InvalidOperationException(string.Format("Type {0} does not implement IFileSystem.", providerType.FullName));
 
-			        return new ProviderConstructionInfo
-				        {
-					        Constructor = constructor,
-					        Parameters = parameters,
-					        //ProviderAlias = s
-				        };
-		        });
+            // find a ctor matching the config parameters
+            var paramCount = providerConfig.Parameters != null ? providerConfig.Parameters.Count : 0;
+            var constructor = providerType.GetConstructors().SingleOrDefault(x
+                => x.GetParameters().Length == paramCount && x.GetParameters().All(y => providerConfig.Parameters.AllKeys.Contains(y.Name)));
+            if (constructor == null)
+                throw new InvalidOperationException(string.Format("Type {0} has no ctor matching the {1} configuration parameter(s).", providerType.FullName, paramCount));
 
-            // create the fs and return
-			return (IFileSystem) ctorInfo.Constructor.Invoke(ctorInfo.Parameters);
+            var parameters = new object[paramCount];
+            if (providerConfig.Parameters != null) // keeps ReSharper happy
+                for (var i = 0; i < paramCount; i++)
+                    parameters[i] = providerConfig.Parameters[providerConfig.Parameters.AllKeys[i]].Value;
+
+            return new ProviderConstructionInfo
+            {
+                Constructor = constructor,
+                Parameters = parameters,
+                //ProviderAlias = s
+            };
         }
 
         /// <summary>
@@ -155,38 +180,68 @@ namespace Umbraco.Core.IO
         /// </summary>
         /// <typeparam name="TFileSystem">The type of the filesystem.</typeparam>
         /// <returns>A strongly-typed filesystem of the specified type.</returns>
+        /// <remarks>
+        /// <para>Ideally, this should cache the instances, but that would break backward compatibility, so we
+        /// only do it for our own MediaFileSystem - for everything else, it's the responsibility of the caller
+        /// to ensure that they maintain singletons. This is important for singletons, as each filesystem maintains
+        /// its own shadow and having multiple instances would lead to inconsistencies.</para>
+        /// <para>Note that any filesystem created by this method *after* shadowing begins, will *not* be
+        /// shadowing (and an exception will be thrown by the ShadowWrapper).</para>
+        /// </remarks>
         public TFileSystem GetFileSystemProvider<TFileSystem>()
-			where TFileSystem : FileSystemWrapper
+            where TFileSystem : FileSystemWrapper
         {
-            // deal with known types - avoid infinite loops!
-            if (typeof(TFileSystem) == typeof(MediaFileSystem) && MediaFileSystem != null)
-                return MediaFileSystem as TFileSystem; // else create and return
+            return GetFileSystemProvider<TFileSystem>(null);
+        }
 
-			// get/cache the alias for the filesystem type
-	        var alias = _aliases.GetOrAdd(typeof (TFileSystem), fsType =>
-		        {
-					// validate the ctor
-					var constructor = fsType.GetConstructors().SingleOrDefault(x
-                        => x.GetParameters().Length == 1 && TypeHelper.IsTypeAssignableFrom<IFileSystem>(x.GetParameters().Single().ParameterType));
-					if (constructor == null)
-						throw new InvalidOperationException("Type " + fsType.FullName + " must inherit from FileSystemWrapper and have a constructor that accepts one parameter of type " + typeof(IFileSystem).FullName + ".");
+        /// <summary>
+        /// Gets a strongly-typed filesystem.
+        /// </summary>
+        /// <typeparam name="TFileSystem">The type of the filesystem.</typeparam>
+        /// <param name="fallback">A fallback creator for the inner filesystem.</param>
+        /// <returns>A strongly-typed filesystem of the specified type.</returns>
+        /// <remarks>
+        /// <para>The fallback creator is used only if nothing is configured.</para>
+        /// <para>Ideally, this should cache the instances, but that would break backward compatibility, so we
+        /// only do it for our own MediaFileSystem - for everything else, it's the responsibility of the caller
+        /// to ensure that they maintain singletons. This is important for singletons, as each filesystem maintains
+        /// its own shadow and having multiple instances would lead to inconsistencies.</para>
+        /// <para>Note that any filesystem created by this method *after* shadowing begins, will *not* be
+        /// shadowing (and an exception will be thrown by the ShadowWrapper).</para>
+        /// </remarks>
+        public TFileSystem GetFileSystemProvider<TFileSystem>(Func<IFileSystem> fallback)
+            where TFileSystem : FileSystemWrapper
+        {
+            var alias = GetFileSystemAlias<TFileSystem>();
+            return (TFileSystem) _filesystems.GetOrAdd(alias, _ =>
+            {
+                // gets the inner fs, create the strongly-typed fs wrapping the inner fs, register & return
+                // so we are double-wrapping here
+                // could be optimized by having FileSystemWrapper inherit from ShadowWrapper, maybe
+                var innerFs = GetUnderlyingFileSystemNoCache(alias, fallback);
+                var shadowWrapper = new ShadowWrapper(innerFs, "typed/" + alias, ScopeProvider);
+                var fs = (IFileSystem2) Activator.CreateInstance(typeof (TFileSystem), shadowWrapper);
+                _wrappers.Add(shadowWrapper); // keeping a reference to the wrapper
+                return fs;
+            });
+        }
 
-                    // find the attribute and get the alias
-					var attr = (FileSystemProviderAttribute) fsType.GetCustomAttributes(typeof(FileSystemProviderAttribute), false).SingleOrDefault();
-					if (attr == null)
-						throw new InvalidOperationException("Type " + fsType.FullName + "is missing the required FileSystemProviderAttribute.");
+        private string GetFileSystemAlias<TFileSystem>()
+        {
+            var fsType = typeof(TFileSystem);
 
-			        return attr.Alias;
-		        });
+            // validate the ctor
+            var constructor = fsType.GetConstructors().SingleOrDefault(x
+                => x.GetParameters().Length == 1 && TypeHelper.IsTypeAssignableFrom<IFileSystem>(x.GetParameters().Single().ParameterType));
+            if (constructor == null)
+                throw new InvalidOperationException("Type " + fsType.FullName + " must inherit from FileSystemWrapper and have a constructor that accepts one parameter of type " + typeof(IFileSystem).FullName + ".");
 
-            // gets the inner fs, create the strongly-typed fs wrapping the inner fs, register & return
-            // so we are double-wrapping here
-            // could be optimized by having FileSystemWrapper inherit from ShadowWrapper, maybe
-            var innerFs = GetUnderlyingFileSystemProvider(alias);
-            var shadowWrapper = new ShadowWrapper(innerFs, "typed/" + alias, ScopeProvider);
-	        var fs = (TFileSystem) Activator.CreateInstance(typeof (TFileSystem), shadowWrapper);
-            _wrappers.Add(shadowWrapper); // keeping a weak reference to the wrapper
-	        return fs;
+            // find the attribute and get the alias
+            var attr = (FileSystemProviderAttribute)fsType.GetCustomAttributes(typeof(FileSystemProviderAttribute), false).SingleOrDefault();
+            if (attr == null)
+                throw new InvalidOperationException("Type " + fsType.FullName + "is missing the required FileSystemProviderAttribute.");
+
+            return attr.Alias;
         }
 
         #endregion
@@ -199,30 +254,37 @@ namespace Umbraco.Core.IO
             var wrappers = new ShadowWrapper[typed.Length + 7];
             var i = 0;
             while (i < typed.Length) wrappers[i] = typed[i++];
-            wrappers[i++] = _macroPartialFileSystemWrapper;
-            wrappers[i++] = _partialViewsFileSystemWrapper;
-            wrappers[i++] = _stylesheetsFileSystemWrapper;
-            wrappers[i++] = _scriptsFileSystemWrapper;
-            wrappers[i++] = _xsltFileSystemWrapper;
-            wrappers[i++] = _masterPagesFileSystemWrapper;
-            wrappers[i] = _mvcViewsFileSystemWrapper;
+            wrappers[i++] = _macroPartialFileSystem;
+            wrappers[i++] = _partialViewsFileSystem;
+            wrappers[i++] = _stylesheetsFileSystem;
+            wrappers[i++] = _scriptsFileSystem;
+            wrappers[i++] = _xsltFileSystem;
+            wrappers[i++] = _masterPagesFileSystem;
+            wrappers[i] = _mvcViewsFileSystem;
 
             return new ShadowFileSystems(id, wrappers);
         }
 
         #endregion
 
-        private class WeakSet<T>
+        private class ConcurrentSet<T>
             where T : class
         {
-            private readonly HashSet<WeakReference<T>> _set = new HashSet<WeakReference<T>>();
+            private readonly HashSet<T> _set = new HashSet<T>();
 
             public void Add(T item)
             {
                 lock (_set)
                 {
-                    _set.Add(new WeakReference<T>(item));
-                    CollectLocked();
+                    _set.Add(item);
+                }
+            }
+
+            public void Clear()
+            {
+                lock (_set)
+                {
+                    _set.Clear();
                 }
             }
 
@@ -230,23 +292,9 @@ namespace Umbraco.Core.IO
             {
                 lock (_set)
                 {
-                    CollectLocked();
-                    return _set.Select(x =>
-                    {
-                        T target;
-                        return x.TryGetTarget(out target) ? target : null;
-                    }).WhereNotNull().ToArray();
+                    return _set.ToArray();
                 }
             }
-
-            private void CollectLocked()
-            {
-                _set.RemoveWhere(x =>
-                {
-                    T target;
-                    return x.TryGetTarget(out target) == false;
-                });
-            }
         }
     }
 }

src/Umbraco.Core/IO/ShadowFileSystem.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 
 namespace Umbraco.Core.IO
 {
@@ -209,19 +210,20 @@ namespace Umbraco.Core.IO
 
         public IEnumerable<string> GetFiles(string path)
         {
-            var normPath = NormPath(path);
-            var shadows = Nodes.Where(kvp => IsChild(normPath, kvp.Key)).ToArray();
-            var files = _fs.GetFiles(path);
-            return files
-                .Except(shadows.Where(kvp => (kvp.Value.IsFile && kvp.Value.IsDelete) || kvp.Value.IsDir)
-                    .Select(kvp => kvp.Key))
-                .Union(shadows.Where(kvp => kvp.Value.IsFile && kvp.Value.IsExist).Select(kvp => kvp.Key))
-                .Distinct();
+            return GetFiles(path, null);
         }
 
         public IEnumerable<string> GetFiles(string path, string filter)
         {
-            return _fs.GetFiles(path, filter);
+            var normPath = NormPath(path);
+            var shadows = Nodes.Where(kvp => IsChild(normPath, kvp.Key)).ToArray();
+            var files = filter != null ? _fs.GetFiles(path, filter) : _fs.GetFiles(path);
+            var regexFilter = FilterToRegex(filter);
+            return files
+                .Except(shadows.Where(kvp => (kvp.Value.IsFile && kvp.Value.IsDelete) || kvp.Value.IsDir)
+                    .Select(kvp => kvp.Key))
+                .Union(shadows.Where(kvp => kvp.Value.IsFile && kvp.Value.IsExist && FilterByRegex(kvp.Key, regexFilter)).Select(kvp => kvp.Key))
+                .Distinct();
         }
 
         public Stream OpenFile(string path)
@@ -253,6 +255,9 @@ namespace Umbraco.Core.IO
 
         public string GetFullPath(string path)
         {
+            ShadowNode sf;
+            if (Nodes.TryGetValue(NormPath(path), out sf))
+                return sf.IsDir || sf.IsDelete ? null : _sfs.GetFullPath(path);
             return _fs.GetFullPath(path);
         }
 
@@ -321,5 +326,32 @@ namespace Umbraco.Core.IO
             _sfs.AddFile(path, physicalPath, overrideIfExists, copy);
             Nodes[normPath] = new ShadowNode(false, false);
         }
+        
+        /// <summary>
+        /// Helper function for filtering keys by Regex if a filter is specified.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <param name="regexFilter"></param>
+        /// <returns></returns>
+        internal static bool FilterByRegex(string input, string regexFilter)
+        {
+            if (regexFilter == null) return true;
+            return regexFilter != string.Empty && Regex.IsMatch(input, regexFilter);
+        }
+
+        /// <summary>
+        /// Transforms a filter pattern into a Regex pattern
+        /// </summary>
+        /// <param name="pattern"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// Appending '$' only if not containing wildcard is stupid and broken.
+        /// It is however what seems to be what they're doing in .NET so we need to match the functionality.
+        /// </remarks>
+        internal static string FilterToRegex(string pattern)
+        {
+            if (pattern == null) return null;
+            return "^" + Regex.Escape(pattern).Replace("\\*", ".*").Replace("\\?", ".") + (pattern.Contains("*") ? string.Empty : "$");
+        }
     }
 }

src/Umbraco.Core/IO/ShadowWrapper.cs

@@ -27,7 +27,7 @@ namespace Umbraco.Core.IO
             // on ShadowFileSystemsScope.None - and if None is false then we should be running
             // in a single thread anyways
 
-            var virt = "~/App_Data/Shadow/" + id + "/" + _shadowPath;
+            var virt = "~/App_Data/TEMP/ShadowFs/" + id + "/" + _shadowPath;
             _shadowDir = IOHelper.MapPath(virt);
             Directory.CreateDirectory(_shadowDir);
             var tempfs = new PhysicalFileSystem(virt);
@@ -69,6 +69,11 @@ namespace Umbraco.Core.IO
             {
                 var isScoped = _scopeProvider != null && _scopeProvider.AmbientScope != null && _scopeProvider.AmbientScope.ScopedFileSystems;
 
+                // if the filesystem is created *after* shadowing starts, it won't be shadowing
+                // better not ignore that situation and raised a meaningful (?) exception
+                if (isScoped && _shadowFileSystem == null)
+                    throw new Exception("The filesystems are shadowing, but this filesystem is not.");
+
                 return isScoped
                     ? _shadowFileSystem
                     : _innerFileSystem;

src/Umbraco.Core/IO/SystemFiles.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -72,15 +73,28 @@ namespace Umbraco.Core.IO
         {
             get
             {
-                if (GlobalSettings.ContentCacheXmlStoredInCodeGen && SystemUtilities.GetCurrentTrustLevel() == AspNetHostingPermissionLevel.Unrestricted)
-                {
-                    return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                switch (GlobalSettings.ContentCacheXmlStorageLocation)
+                {                    
+                    case ContentXmlStorage.AspNetTemp:
+                        return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                    case ContentXmlStorage.EnvironmentTemp:
+                        var appDomainHash = HttpRuntime.AppDomainAppId.ToSHA1();
+                        var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "UmbracoXml",
+                            //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back
+                            // to worker A again, in theory the %temp%  folder should already be empty but we really want to make sure that its not
+                            // utilizing an old path
+                            appDomainHash);
+                        return Path.Combine(cachePath, "umbraco.config");
+                    case ContentXmlStorage.Default:
+                        return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
+                    default:
+                        throw new ArgumentOutOfRangeException();
                 }
-                return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
             }
         }
 
         [Obsolete("Use GlobalSettings.ContentCacheXmlStoredInCodeGen instead")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         internal static bool ContentCacheXmlStoredInCodeGen
         {
             get { return GlobalSettings.ContentCacheXmlStoredInCodeGen; }

src/Umbraco.Core/Logging/Logger.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Web;
 using log4net;
@@ -62,12 +63,34 @@ namespace Umbraco.Core.Logging
         
 		public void Error(Type callingType, string message, Exception exception)
 		{
-			var logger = LogManager.GetLogger(callingType);
-			if (logger != null)
-				logger.Error((message), exception);
+            var logger = LogManager.GetLogger(callingType);
+		    if (logger == null) return;
+
+		    if (IsTimeoutThreadAbortException(exception))
+		    {
+		        message += "\r\nThe thread has been aborted, because the request has timed out.";
+		    }
+
+		    logger.Error(message, exception);
 		}
 
-		public void Warn(Type callingType, string message, params Func<object>[] formatItems)
+        private static bool IsTimeoutThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            if (abort.ExceptionState == null) return false;
+
+            var stateType = abort.ExceptionState.GetType();
+            if (stateType.FullName != "System.Web.HttpApplication+CancelModuleException") return false;
+
+            var timeoutField = stateType.GetField("_timeout", BindingFlags.Instance | BindingFlags.NonPublic);
+            if (timeoutField == null) return false;
+
+            return (bool) timeoutField.GetValue(abort.ExceptionState);
+        }
+
+        public void Warn(Type callingType, string message, params Func<object>[] formatItems)
 		{
 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;

src/Umbraco.Core/Models/ContentTypeExtensions.cs

@@ -30,7 +30,8 @@ namespace Umbraco.Core.Models
                 return descendants;
             }
 
-            throw new NotSupportedException("The content type must be " + typeof(IContentType) + " or " + typeof(IMediaType));
+            //No other content types have children (i.e. member types)
+            return Enumerable.Empty<IContentTypeBase>();
         }
 
         /// <summary>

src/Umbraco.Core/Models/PropertyTypeCollection.cs

@@ -69,6 +69,7 @@ namespace Umbraco.Core.Models
             OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
         }
 
+        //TODO: Instead of 'new' this should explicitly implement one of the collection interfaces members
         internal new void Add(PropertyType item)
         {
             using (new WriteLock(_addLocker))

src/Umbraco.Core/Models/PublicAccessEntry.cs

@@ -110,6 +110,12 @@ namespace Umbraco.Core.Models
             _ruleCollection.Clear();
         }
 
+
+        internal void ClearRemovedRules()
+        {
+            _removedRules.Clear();
+        }
+
         [DataMember]
         public int LoginNodeId
         {

src/Umbraco.Core/Models/PublishedContent/PublishedContentEnumerable.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Umbraco.Core.Models.PublishedContent
+{
+    /// <summary>
+    /// The published content enumerable, this model is to allow ToString to be overriden for value converters to support legacy requests for string values
+    /// </summary>
+    public class PublishedContentEnumerable : IEnumerable<IPublishedContent>
+    {
+        /// <summary>
+        /// The items in the collection
+        /// </summary>
+        private readonly IEnumerable<IPublishedContent> _items;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="PublishedContentEnumerable"/> class.
+        /// </summary>
+        /// <param name="publishedContent">
+        /// The published content items
+        /// </param>
+        public PublishedContentEnumerable(IEnumerable<IPublishedContent> publishedContent)
+        {
+            if (publishedContent == null) throw new ArgumentNullException("publishedContent");
+            _items = publishedContent;
+        }
+
+        /// <summary>
+        /// The ToString method to convert the objects back to CSV
+        /// </summary>
+        /// <returns>
+        /// The <see cref="string"/>.
+        /// </returns>
+        public override string ToString()
+        {
+            return string.Join(",", _items.Select(x => x.Id));
+        }
+
+        /// <summary>
+        /// The get enumerator.
+        /// </summary>
+        /// <returns>
+        /// The <see cref="IEnumerator"/>.
+        /// </returns>
+        public IEnumerator<IPublishedContent> GetEnumerator()
+        {
+            return _items.GetEnumerator();
+        }
+
+        /// <summary>
+        /// The get enumerator.
+        /// </summary>
+        /// <returns>
+        /// The <see cref="IEnumerator"/>.
+        /// </returns>
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

src/Umbraco.Core/Models/PublishedContent/PublishedContentExtended.cs

@@ -1,11 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.ComponentModel;
 using System.Linq;
 
 namespace Umbraco.Core.Models.PublishedContent
 {
-
+    [TypeConverter(typeof(PublishedContentTypeConverter))]
     public class PublishedContentExtended : PublishedContentWrapped, IPublishedContentExtended
     {
         #region Constructor
@@ -186,6 +187,11 @@ namespace Umbraco.Core.Models.PublishedContent
         }
 
         #endregion
+
+        public override string ToString()
+        {
+            return Id.ToString();
+        }
     }
 
 }

src/Umbraco.Core/Models/PublishedContent/PublishedContentTypeConverter.cs

@@ -0,0 +1,39 @@
+using System;
+using System.ComponentModel;
+using System.Globalization;
+using System.Linq;
+
+namespace Umbraco.Core.Models.PublishedContent
+{
+    internal class PublishedContentTypeConverter : TypeConverter
+    {
+        private static readonly Type[] ConvertableTypes = new[]
+        {
+            typeof(int)
+        };
+
+        public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType)
+        {
+            return ConvertableTypes.Any(x => TypeHelper.IsTypeAssignableFrom(x, destinationType))
+                   || base.CanConvertFrom(context, destinationType);
+        }
+
+        public override object ConvertTo(
+            ITypeDescriptorContext context,
+            CultureInfo culture,
+            object value,
+            Type destinationType)
+        {
+            var publishedContent = value as IPublishedContent;
+            if (publishedContent == null)
+                return null;
+
+            if (TypeHelper.IsTypeAssignableFrom<int>(destinationType))
+            {
+                return publishedContent.Id;
+            }
+
+            return base.ConvertTo(context, culture, value, destinationType);
+        }
+    }
+}

src/Umbraco.Core/Models/Range.cs

@@ -0,0 +1,52 @@
+using System;
+namespace Umbraco.Core.Models
+{
+    // <summary>The Range class. Adapted from http://stackoverflow.com/questions/5343006/is-there-a-c-sharp-type-for-representing-an-integer-range</summary>
+    /// <typeparam name="T">Generic parameter.</typeparam>
+    public class Range<T> where T : IComparable<T>
+    {
+        /// <summary>Minimum value of the range.</summary>
+        public T Minimum { get; set; }
+
+        /// <summary>Maximum value of the range.</summary>
+        public T Maximum { get; set; }
+
+        /// <summary>Presents the Range in readable format.</summary>
+        /// <returns>String representation of the Range</returns>
+        public override string ToString()
+        {
+            return string.Format("{0},{1}", this.Minimum, this.Maximum);
+        }
+
+        /// <summary>Determines if the range is valid.</summary>
+        /// <returns>True if range is valid, else false</returns>
+        public bool IsValid()
+        {
+            return this.Minimum.CompareTo(this.Maximum) <= 0;
+        }
+
+        /// <summary>Determines if the provided value is inside the range.</summary>
+        /// <param name="value">The value to test</param>
+        /// <returns>True if the value is inside Range, else false</returns>
+        public bool ContainsValue(T value)
+        {
+            return (this.Minimum.CompareTo(value) <= 0) && (value.CompareTo(this.Maximum) <= 0);
+        }
+
+        /// <summary>Determines if this Range is inside the bounds of another range.</summary>
+        /// <param name="Range">The parent range to test on</param>
+        /// <returns>True if range is inclusive, else false</returns>
+        public bool IsInsideRange(Range<T> range)
+        {
+            return this.IsValid() && range.IsValid() && range.ContainsValue(this.Minimum) && range.ContainsValue(this.Maximum);
+        }
+
+        /// <summary>Determines if another range is inside the bounds of this range.</summary>
+        /// <param name="Range">The child range to test</param>
+        /// <returns>True if range is inside, else false</returns>
+        public bool ContainsRange(Range<T> range)
+        {
+            return this.IsValid() && range.IsValid() && this.ContainsValue(range.Minimum) && this.ContainsValue(range.Maximum);
+        }
+    }
+}

src/Umbraco.Core/Models/Rdbms/DataTypeDto.cs

@@ -10,7 +10,7 @@ namespace Umbraco.Core.Models.Rdbms
     internal class DataTypeDto
     {
         [Column("pk")]
-        [PrimaryKeyColumn(IdentitySeed = 30)]
+        [PrimaryKeyColumn(IdentitySeed = 40)]
         public int PrimaryKey { get; set; }
 
         [Column("nodeId")]

src/Umbraco.Core/Models/Rdbms/NodeDto.cs

@@ -10,7 +10,7 @@ namespace Umbraco.Core.Models.Rdbms
     [ExplicitColumns]
     internal class NodeDto
     {
-        public const int NodeIdSeed = 1050;
+        public const int NodeIdSeed = 1060;
 
         [Column("id")]
         [PrimaryKeyColumn(Name = "PK_structure", IdentitySeed = NodeIdSeed)]

src/Umbraco.Core/Models/UmbracoEntityExtensions.cs

@@ -113,6 +113,11 @@ namespace Umbraco.Core.Models
             }
         }
 
+        /// <summary>
+        /// When resolved from EntityService this checks if the entity has the HasChildren flag
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <returns></returns>
         public static bool HasChildren(this IUmbracoEntity entity)
         {
             if (entity.AdditionalData.ContainsKey("HasChildren"))
@@ -133,6 +138,11 @@ namespace Umbraco.Core.Models
             return entity.AdditionalData.GetValueIgnoreCase(key, defaultVal);
         }
 
+        /// <summary>
+        /// When resolved from EntityService this checks if the entity has the IsContainer flag
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <returns></returns>
         public static bool IsContainer(this IUmbracoEntity entity)
         {
             if (entity.AdditionalData.ContainsKeyIgnoreCase("IsContainer") == false) return false;

src/Umbraco.Core/Persistence/Factories/MemberFactory.cs

@@ -28,17 +28,17 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IMedia,ContentVersionDto>
 
-        public IMember BuildEntity(MemberDto dto)
+        public static IMember BuildEntity(MemberDto dto, IMemberType contentType)
         {
             var member = new Member(
                 dto.ContentVersionDto.ContentDto.NodeDto.Text,
-                dto.Email, dto.LoginName, dto.Password, _contentType);
+                dto.Email, dto.LoginName, dto.Password, contentType);
 
             try
             {
                 member.DisableChangeTracking();
 
-                member.Id = _id;
+                member.Id = dto.NodeId;
                 member.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                 member.Path = dto.ContentVersionDto.ContentDto.NodeDto.Path;
                 member.CreatorId = dto.ContentVersionDto.ContentDto.NodeDto.UserId.Value;
@@ -62,6 +62,12 @@ namespace Umbraco.Core.Persistence.Factories
             }
         }
 
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMember BuildEntity(MemberDto dto)
+        {
+            return BuildEntity(dto, _contentType);
+        }
+
         public MemberDto BuildDto(IMember entity)
         {
             var dto = new MemberDto

src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Factories
 {
@@ -17,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories
             
             //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data
             foreach (var k in originalEntityProperties.Keys
-                .Select(x => new { orig = x, title = x.ConvertCase(StringAliasCaseType.PascalCase) })
+                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase | CleanStringType.Ascii | CleanStringType.ConvertCase) })
                 .Where(x => entityProps.InvariantContains(x.title) == false))
             {
                 entity.AdditionalData[k.title] = originalEntityProperties[k.orig];
@@ -50,7 +51,7 @@ namespace Umbraco.Core.Persistence.Factories
                 entity.ContentTypeThumbnail = asDictionary.ContainsKey("thumbnail") ? (d.thumbnail ?? string.Empty) : string.Empty;
 
                 var publishedVersion = default(Guid);
-                //some content items don't have a published version
+                //some content items don't have a published/newest version
                 if (asDictionary.ContainsKey("publishedVersion") && asDictionary["publishedVersion"] != null)
                 {
                     Guid.TryParse(d.publishedVersion.ToString(), out publishedVersion);
@@ -75,65 +76,6 @@ namespace Umbraco.Core.Persistence.Factories
                 entity.EnableChangeTracking();
             }
         }
-
-        public UmbracoEntity BuildEntity(EntityRepository.UmbracoEntityDto dto)
-        {
-            var entity = new UmbracoEntity(dto.Trashed)
-                             {
-                                 CreateDate = dto.CreateDate,
-                                 CreatorId = dto.UserId.Value,
-                                 Id = dto.NodeId,
-                                 Key = dto.UniqueId,
-                                 Level = dto.Level,
-                                 Name = dto.Text,
-                                 NodeObjectTypeId = dto.NodeObjectType.Value,
-                                 ParentId = dto.ParentId,
-                                 Path = dto.Path,
-                                 SortOrder = dto.SortOrder,
-                                 HasChildren = dto.Children > 0,
-                                 ContentTypeAlias = dto.Alias ?? string.Empty,
-                                 ContentTypeIcon = dto.Icon ?? string.Empty,
-                                 ContentTypeThumbnail = dto.Thumbnail ?? string.Empty,
-                             };
-
-            entity.IsPublished = dto.PublishedVersion != default(Guid) || (dto.NewestVersion != default(Guid) && dto.PublishedVersion == dto.NewestVersion);
-            entity.IsDraft = dto.NewestVersion != default(Guid) && (dto.PublishedVersion == default(Guid) || dto.PublishedVersion != dto.NewestVersion);
-            entity.HasPendingChanges = (dto.PublishedVersion != default(Guid) && dto.NewestVersion != default(Guid)) && dto.PublishedVersion != dto.NewestVersion;
-
-            if (dto.UmbracoPropertyDtos != null)
-            {
-                foreach (var propertyDto in dto.UmbracoPropertyDtos)
-                {
-                    entity.AdditionalData[propertyDto.PropertyAlias] = new UmbracoEntity.EntityProperty
-                    {
-                        PropertyEditorAlias = propertyDto.PropertyEditorAlias,
-                        Value = propertyDto.NTextValue.IsNullOrWhiteSpace()
-                            ? propertyDto.NVarcharValue
-                            : propertyDto.NTextValue.ConvertToJsonIfPossible()
-                    };
-                }
-            }
-
-            return entity;
-        }
-
-        public EntityRepository.UmbracoEntityDto BuildDto(UmbracoEntity entity)
-        {
-            var node = new EntityRepository.UmbracoEntityDto
-                           {
-                               CreateDate = entity.CreateDate,
-                               Level = short.Parse(entity.Level.ToString(CultureInfo.InvariantCulture)),
-                               NodeId = entity.Id,
-                               NodeObjectType = entity.NodeObjectTypeId,
-                               ParentId = entity.ParentId,
-                               Path = entity.Path,
-                               SortOrder = entity.SortOrder,
-                               Text = entity.Name,
-                               Trashed = entity.Trashed,
-                               UniqueId = entity.Key,
-                               UserId = entity.CreatorId
-                           };
-            return node;
-        }
+        
     }
 }

src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs

@@ -31,7 +31,7 @@ namespace Umbraco.Core.Persistence.Mappers
         /// </summary>
         /// <param name="type"></param>
         /// <returns></returns>
-        internal BaseMapper ResolveMapperByType(Type type)
+        public virtual BaseMapper ResolveMapperByType(Type type)
         {
             return _mapperCache.GetOrAdd(type, type1 =>
                 {
@@ -67,7 +67,7 @@ namespace Umbraco.Core.Persistence.Mappers
             return Attempt<BaseMapper>.Succeed(mapper);
         }  
 
-        internal string GetMapping(Type type, string propertyName)
+        public virtual string GetMapping(Type type, string propertyName)
         {
             var mapper = ResolveMapperByType(type);
             var result = mapper.Map(propertyName);

src/Umbraco.Core/Persistence/Migrations/Initial/BaseDataCreation.cs

@@ -130,8 +130,7 @@ namespace Umbraco.Core.Persistence.Migrations.Initial
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = Constants.System.DefaultMembersListViewDataTypeId, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,-97", SortOrder = 2, UniqueId = new Guid("AA2C52A0-CE87-4E65-A47C-7DF09358585D"), Text = Constants.Conventions.DataTypes.ListViewPrefix + "Members", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });                                    
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1031, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1031", SortOrder = 2, UniqueId = new Guid("f38bd2d7-65d0-48e6-95dc-87ce06ec2d3d"), Text = Constants.Conventions.MediaTypes.Folder, NodeObjectType = new Guid(Constants.ObjectTypes.MediaType), CreateDate = DateTime.Now });
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1032, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1032", SortOrder = 2, UniqueId = new Guid("cc07b313-0843-4aa8-bbda-871c8da728c8"), Text = Constants.Conventions.MediaTypes.Image, NodeObjectType = new Guid(Constants.ObjectTypes.MediaType), CreateDate = DateTime.Now });
-            _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1033, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1033", SortOrder = 2, UniqueId = new Guid("4c52d8ab-54e6-40cd-999c-7a5f24903e4d"), Text = Constants.Conventions.MediaTypes.File, NodeObjectType = new Guid(Constants.ObjectTypes.MediaType), CreateDate = DateTime.Now });            
-            _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1040, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1040", SortOrder = 2, UniqueId = new Guid("21e798da-e06e-4eda-a511-ed257f78d4fa"), Text = "Related Links", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
+            _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1033, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1033", SortOrder = 2, UniqueId = new Guid("4c52d8ab-54e6-40cd-999c-7a5f24903e4d"), Text = Constants.Conventions.MediaTypes.File, NodeObjectType = new Guid(Constants.ObjectTypes.MediaType), CreateDate = DateTime.Now });                        
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1041, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1041", SortOrder = 2, UniqueId = new Guid("b6b73142-b9c1-4bf8-a16d-e1c23320b549"), Text = "Tags", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1043, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1043", SortOrder = 2, UniqueId = new Guid("1df9f033-e6d4-451f-b8d2-e0cbc50a836f"), Text = "Image Cropper", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1044, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1044", SortOrder = 0, UniqueId = new Guid("d59be02f-1df9-4228-aa1e-01917d806cda"), Text = Constants.Conventions.MemberTypes.DefaultAlias, NodeObjectType = new Guid(Constants.ObjectTypes.MemberType), CreateDate = DateTime.Now });
@@ -141,6 +140,7 @@ namespace Umbraco.Core.Persistence.Migrations.Initial
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1047, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1047", SortOrder = 2, UniqueId = new Guid("1EA2E01F-EBD8-4CE1-8D71-6B1149E63548"), Text = "Member Picker", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1048, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1048", SortOrder = 2, UniqueId = new Guid("135D60E0-64D9-49ED-AB08-893C9BA44AE5"), Text = "Media Picker", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
             _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1049, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1049", SortOrder = 2, UniqueId = new Guid("9DBBCBBB-2327-434A-B355-AF1B84E5010A"), Text = "Multiple Media Picker", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
+            _database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1050, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1050", SortOrder = 2, UniqueId = new Guid("B4E3535A-1753-47E2-8568-602CF8CFEE6F"), Text = "Related Links", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
 
             //TODO: We're not creating these for 7.0
             //_database.Insert("umbracoNode", "id", false, new NodeDto { NodeId = 1039, Trashed = false, ParentId = -1, UserId = 0, Level = 1, Path = "-1,1039", SortOrder = 2, UniqueId = new Guid("06f349a9-c949-4b6a-8660-59c10451af42"), Text = "Ultimate Picker", NodeObjectType = new Guid(Constants.ObjectTypes.DataType), CreateDate = DateTime.Now });
@@ -246,8 +246,7 @@ namespace Umbraco.Core.Persistence.Migrations.Initial
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 12, DataTypeId = -40, PropertyEditorAlias = Constants.PropertyEditors.RadioButtonListAlias, DbType = "Nvarchar" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 13, DataTypeId = -41, PropertyEditorAlias = Constants.PropertyEditors.DateAlias, DbType = "Date" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 14, DataTypeId = -42, PropertyEditorAlias = Constants.PropertyEditors.DropDownListAlias, DbType = "Integer" });
-            _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 15, DataTypeId = -43, PropertyEditorAlias = Constants.PropertyEditors.CheckBoxListAlias, DbType = "Nvarchar" });            
-            _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 21, DataTypeId = 1040, PropertyEditorAlias = Constants.PropertyEditors.RelatedLinksAlias, DbType = "Ntext" });
+            _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 15, DataTypeId = -43, PropertyEditorAlias = Constants.PropertyEditors.CheckBoxListAlias, DbType = "Nvarchar" });                        
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 22, DataTypeId = 1041, PropertyEditorAlias = Constants.PropertyEditors.TagsAlias, DbType = "Ntext" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 24, DataTypeId = 1043, PropertyEditorAlias = Constants.PropertyEditors.ImageCropperAlias, DbType = "Ntext" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = -26, DataTypeId = Constants.System.DefaultContentListViewDataTypeId, PropertyEditorAlias = Constants.PropertyEditors.ListViewAlias, DbType = "Nvarchar" });
@@ -259,6 +258,7 @@ namespace Umbraco.Core.Persistence.Migrations.Initial
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 27, DataTypeId = 1047, PropertyEditorAlias = Constants.PropertyEditors.MemberPicker2Alias, DbType = "Nvarchar" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 28, DataTypeId = 1048, PropertyEditorAlias = Constants.PropertyEditors.MediaPicker2Alias, DbType = "Ntext" });
             _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 29, DataTypeId = 1049, PropertyEditorAlias = Constants.PropertyEditors.MediaPicker2Alias, DbType = "Ntext" });
+            _database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 30, DataTypeId = 1050, PropertyEditorAlias = Constants.PropertyEditors.RelatedLinks2Alias, DbType = "Ntext" });
 
             //TODO: We're not creating these for 7.0
             //_database.Insert("cmsDataType", "pk", false, new DataTypeDto { PrimaryKey = 19, DataTypeId = 1038, PropertyEditorAlias = Constants.PropertyEditors.MarkdownEditorAlias, DbType = "Ntext" });            

src/Umbraco.Core/Persistence/PetaPoco.cs

@@ -703,7 +703,7 @@ namespace Umbraco.Core.Persistence
 
 		static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?<depth>)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?<!,\s+)\bFROM\b", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled);
 		static Regex rxOrderBy = new Regex(@"\bORDER\s+BY\s+(?:\((?>\((?<depth>)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?<depth>)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled);
-		static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled);
+        static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled);
 		public static bool SplitSqlForPaging(string sql, out string sqlCount, out string sqlSelectRemoved, out string sqlOrderBy)
 		{
 			sqlSelectRemoved = null;
@@ -723,10 +723,9 @@ namespace Umbraco.Core.Persistence
 				sqlCount = sql.Substring(0, g.Index) + "COUNT(" + m.Groups[1].ToString().Trim() + ") " + sql.Substring(g.Index + g.Length);
 			else
 				sqlCount = sql.Substring(0, g.Index) + "COUNT(*) " + sql.Substring(g.Index + g.Length);
-
-
-			// Look for an "ORDER BY <whatever>" clause
-			m = rxOrderBy.Match(sqlCount);
+            
+		    // Look for an "ORDER BY <whatever>" clause
+            m = rxOrderBy.Match(sqlCount);
 			if (!m.Success)
 			{
 				sqlOrderBy = null;
@@ -737,8 +736,7 @@ namespace Umbraco.Core.Persistence
 				sqlOrderBy = g.ToString();
 				sqlCount = sqlCount.Substring(0, g.Index) + sqlCount.Substring(g.Index + g.Length);
 			}
-
-			return true;
+            return true;
 		}
 
 	    /// <summary>
@@ -820,7 +818,7 @@ namespace Umbraco.Core.Persistence
 			result.CurrentPage = page;
 			result.ItemsPerPage = itemsPerPage;
 			result.TotalItems = ExecuteScalar<long>(sqlCount, args);
-			result.TotalPages = result.TotalItems / itemsPerPage;
+		    result.TotalPages = result.TotalItems / itemsPerPage;
 			if ((result.TotalItems % itemsPerPage) != 0)
 				result.TotalPages++;
 

src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs

@@ -546,7 +546,7 @@ namespace Umbraco.Core.Persistence.Querying
                         && methodArgs.Length == 1 
                         && methodArgs[0].NodeType == ExpressionType.MemberAccess
                         && TypeHelper.IsTypeAssignableFrom<IEnumerable>(m.Arguments[0].Type))
-                    {                        
+                    {
                         goto case "SqlIn";
                     }
 

src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq.Expressions;
+using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Mappers;
 using Umbraco.Core.Persistence.SqlSyntax;
 
@@ -12,17 +13,19 @@ namespace Umbraco.Core.Persistence.Querying
     /// <remarks>This object is stateful and cannot be re-used to parse an expression.</remarks>
     internal class ModelToSqlExpressionVisitor<T> : ExpressionVisitorBase
     {
+        private readonly MappingResolver _mappingResolver;
         private readonly BaseMapper _mapper;
 
-        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, BaseMapper mapper)
+        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, MappingResolver mappingResolver)
             : base(sqlSyntax)
         {
-            _mapper = mapper;
+            _mapper = mappingResolver.ResolveMapperByType(typeof(T));
+            _mappingResolver = mappingResolver;
         }
 
         [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public ModelToSqlExpressionVisitor()
-            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T)))
+            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current)
         { }
 
         protected override string VisitMemberAccess(MemberExpression m)
@@ -36,7 +39,7 @@ namespace Umbraco.Core.Persistence.Querying
                 {
                     var field = _mapper.Map(m.Member.Name, true);
                     if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                     return field;
                 }
                 //already compiled, return
@@ -50,13 +53,42 @@ namespace Umbraco.Core.Persistence.Querying
                 {
                     var field = _mapper.Map(m.Member.Name, true);
                     if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                     return field;
                 }
                 //already compiled, return
                 return string.Empty;
             }
 
+            if (m.Expression != null 
+                && m.Expression.Type != typeof(T) 
+                && TypeHelper.IsTypeAssignableFrom<IUmbracoEntity>(m.Expression.Type)
+                && EndsWithConstant(m) == false)
+            {
+                //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test";
+                //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column
+
+                //don't execute if compiled
+                if (Visited == false)
+                {
+                    var subMapper = _mappingResolver.ResolveMapperByType(m.Expression.Type);
+                    if (subMapper == null)
+                        throw new NullReferenceException("No mapper found for type " + m.Expression.Type);
+                    var field = subMapper.Map(m.Member.Name, true);
+                    if (field.IsNullOrWhiteSpace())
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
+                    return field;
+                }
+                //already compiled, return
+                return string.Empty;
+            }
+
+            //TODO: When m.Expression.NodeType == ExpressionType.Constant and it's an expression like: content => aliases.Contains(content.ContentType.Alias);
+            // then an SQL parameter will be added for aliases as an array, however in SqlIn on the subclass it will manually add these SqlParameters anyways,
+            // however the query will still execute because the SQL that is written will only contain the correct indexes of SQL parameters, this would be ignored,
+            // I'm just unsure right now due to time constraints how to make it correct. It won't matter right now and has been working already with this bug but I've
+            // only just discovered what it is actually doing.
+
             var member = Expression.Convert(m, typeof(object));
             var lambda = Expression.Lambda<Func<object>>(member);
             var getter = lambda.Compile();
@@ -71,5 +103,24 @@ namespace Umbraco.Core.Persistence.Querying
             return string.Empty;
 
         }
+
+        /// <summary>
+        /// Determines if the MemberExpression ends in a Constant value
+        /// </summary>
+        /// <param name="m"></param>
+        /// <returns></returns>
+        private bool EndsWithConstant(MemberExpression m)
+        {
+            Expression expr = m;
+            
+            while (expr is MemberExpression)
+            {
+                var memberExpr = expr as MemberExpression;
+                expr = memberExpr.Expression;
+            }
+            
+            var constExpr = expr as ConstantExpression;
+            return constExpr != null;
+        }
     }
 }

src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
 

src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs

@@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateContentFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateContentFromDto(dto, sql);
 
             return content;
         }
@@ -134,6 +134,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 .InnerJoin<NodeDto>(SqlSyntax)
                 .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+            //TODO: IF we want to enable querying on content type information this will need to be joined
+            //.InnerJoin<ContentTypeDto>(SqlSyntax)
+            //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
 
             if (queryType == BaseQueryType.FullSingle)
             {
@@ -260,6 +263,39 @@ namespace Umbraco.Core.Persistence.Repositories
                 }
                 baseId = xmlItems[xmlItems.Count - 1].NodeId;
             }
+
+            //now delete the items that shouldn't be there
+            var sqlAllIds = translate(0, GetBaseQuery(BaseQueryType.Ids));
+            var allContentIds = Database.Fetch<int>(sqlAllIds);
+            var docObjectType = Guid.Parse(Constants.ObjectTypes.Document);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allContentIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }                
+            }
+                
         }
 
         public override IEnumerable<IContent> GetAllVersions(int id)
@@ -273,12 +309,13 @@ namespace Umbraco.Core.Persistence.Repositories
             var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple));
             var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
             
-            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true);
+            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true,  includeAllVersions:true);
         }
 
         public override IContent GetByVersion(Guid versionId)
         {
             var sql = GetBaseQuery(BaseQueryType.FullSingle);
+            //TODO: cmsContentVersion.VersionId has a Unique Index constraint applied, seems silly then to also add OrderByDescending since it would be impossible to return more than one.
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
             sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
@@ -287,7 +324,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateContentFromDto(dto, versionId, sql);
+            var content = CreateContentFromDto(dto, sql);
 
             return content;
         }
@@ -297,7 +334,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From<DocumentDto>(SqlSyntax)
-                .InnerJoin<ContentVersionDto>(SqlSyntax).On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
                 .Where<ContentVersionDto>(x => x.VersionId == versionId, SqlSyntax)
                 .Where<DocumentDto>(x => x.Newest != true, SqlSyntax);
             var dto = Database.Fetch<DocumentDto, ContentVersionDto>(sql).FirstOrDefault();
@@ -317,7 +355,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentVersionDto>()
+                .On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
                 .Where<ContentVersionDto>(x => x.NodeId == id)
                 .Where<ContentVersionDto>(x => x.VersionDate < versionDate)
                 .Where<DocumentDto>(x => x.Newest != true);
@@ -558,6 +597,7 @@ namespace Umbraco.Core.Persistence.Repositories
             //if (((ICanBeDirty)entity).IsPropertyDirty("Published") && (entity.Published || publishedState == PublishedState.Unpublished))
             if (entity.ShouldClearPublishedFlagForPreviousVersions(publishedState, shouldCreateNewVersion))
             {
+                //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
                 var publishedDocs = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND published = @IsPublished", new { Id = entity.Id, IsPublished = true });
                 foreach (var doc in publishedDocs)
                 {
@@ -571,6 +611,7 @@ namespace Umbraco.Core.Persistence.Repositories
             }
 
             //Look up (newest) entries by id in cmsDocument table to set newest = false
+            //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
             var documentDtos = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND newest = @IsNewest", new { Id = entity.Id, IsNewest = true });
             foreach (var documentDto in documentDtos)
             {
@@ -898,7 +939,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
             return base.GetDatabaseFieldNameForOrderBy(orderBy);
         }
-        
+
         /// <summary>
         /// This is the underlying method that processes most queries for this repository
         /// </summary>
@@ -909,8 +950,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
         /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
         /// </param>
         /// <param name="withCache"></param>
+        /// <param name="includeAllVersions">
+        /// Generally when querying for content we only want to return the most recent version of the content item, however in some cases like when 
+        /// we want to return all versions of a content item, we can't simply return the latest
+        /// </param>
         /// <returns></returns>
-        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
+        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false, bool includeAllVersions = false)
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
@@ -929,13 +974,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
                 parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
             }            
 
-            var publishedSql = new Sql(@"SELECT *
-FROM cmsDocument AS doc2
-INNER JOIN 
-	(" + parsedOriginalSql + @") as docData
-ON doc2.nodeId = docData.nodeId
-WHERE doc2.published = 1
-ORDER BY doc2.nodeId
+            //order by update date DESC, if there is corrupted published flags we only want the latest!
+            var publishedSql = new Sql(@"SELECT cmsDocument.nodeId, cmsDocument.published, cmsDocument.versionId, cmsDocument.newest
+FROM cmsDocument INNER JOIN cmsContentVersion ON cmsContentVersion.VersionId = cmsDocument.versionId
+WHERE cmsDocument.published = 1 AND cmsDocument.nodeId IN 
+(" + parsedOriginalSql + @")
+ORDER BY cmsContentVersion.id DESC
 ", sqlFull.Arguments);
 
             //go and get the published version data, we do a Query here and not a Fetch so we are
@@ -950,9 +994,9 @@ ORDER BY doc2.nodeId
                     publishedDataCollection.Add(publishedDto);
             }
 
-
-            var content = new IContent[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IContent, bool>>();
+            var defs = new DocumentDefinitionCollection(includeAllVersions);
             var templateIds = new List<int>();
             
             //track the looked up content types, even though the content types are cached
@@ -960,9 +1004,8 @@ ORDER BY doc2.nodeId
             // the overhead of deep cloning them on every item in this loop
             var contentTypes = new Dictionary<int, IContentType>();
             
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
                 DocumentPublishedReadOnlyDto publishedDto;
                 publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto);
 
@@ -970,10 +1013,10 @@ ORDER BY doc2.nodeId
                 if (withCache)
                 {
                     var cached = IsolatedCache.GetCacheItem<IContent>(GetCacheIdKey<IContent>(dto.NodeId));
-                    //only use this cached version if the dto returned is also the publish version, they must match
-                    if (cached != null && cached.Published && dto.Published)
+                    //only use this cached version if the dto returned is also the publish version, they must match and be teh same version
+                    if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IContent, bool>(cached, true));
                         continue;
                     }
                 }
@@ -991,21 +1034,16 @@ ORDER BY doc2.nodeId
                     contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
                     contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
                 }
-                
-                content[i] = ContentFactory.BuildEntity(dto, contentType, publishedDto);
 
-                // need template
-                if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
-                    templateIds.Add(dto.TemplateId.Value);
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    // assign template
+                    if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
+                        templateIds.Add(dto.TemplateId.Value);
 
-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                    content.Add(new Tuple<IContent, bool>(ContentFactory.BuildEntity(dto, contentType, publishedDto), false));
+                }
             }
 
             // load all required templates in 1 query
@@ -1015,38 +1053,38 @@ ORDER BY doc2.nodeId
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign template and property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id];
 
-                // complete the item
-                var cc = content[dtoIndex];
-                var dto = dtos[dtoIndex];
                 ITemplate template = null;
-                if (dto.TemplateId.HasValue)
-                    templates.TryGetValue(dto.TemplateId.Value, out template); // else null
+                if (def.DocumentDto.TemplateId.HasValue)
+                    templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null
                 cc.Template = template;
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// <summary>
         /// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion.
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql)
+        private IContent CreateContentFromDto(DocumentDto dto, Sql docSql)
         {
             var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
@@ -1058,11 +1096,11 @@ ORDER BY doc2.nodeId
                 content.Template = _templateRepository.Get(dto.TemplateId.Value);
             }
 
-            var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType);
+            var docDef = new DocumentDefinition(dto, contentType);
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            content.Properties = properties[dto.NodeId];
+            content.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs

@@ -104,7 +104,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             if (objectTypes.Any())
             {
-                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", objectTypes);
+                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", new {objectTypes = objectTypes});
             }
 
             return Database.Fetch<string>(sql);

src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs

@@ -1,18 +1,14 @@
 using System;
 using System.Collections.Generic;
-using System.Dynamic;
-using System.Globalization;
+using System.Collections.ObjectModel;
 using System.Linq;
-using System.Reflection;
-using System.Text;
 using Umbraco.Core.Models;
-using Umbraco.Core;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
@@ -49,6 +45,124 @@ namespace Umbraco.Core.Persistence.Repositories
 
         #region Query Methods
 
+        public IEnumerable<IUmbracoEntity> GetPagedResultsByQuery(IQuery<IUmbracoEntity> query, Guid objectTypeId, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy, Direction orderDirection, IQuery<IUmbracoEntity> filter = null)
+        {   
+            bool isContent = objectTypeId == new Guid(Constants.ObjectTypes.Document);
+            bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
+            var factory = new UmbracoEntityFactory();
+            
+            var sqlClause = GetBaseWhere(GetBase, isContent, isMedia, sql =>
+            {
+                if (filter != null)
+                {
+                    foreach (var filterClaus in filter.GetWhereClauses())
+                    {
+                        sql.Where(filterClaus.Item1, filterClaus.Item2);
+                    }
+                }
+            }, objectTypeId);
+            var translator = new SqlTranslator<IUmbracoEntity>(sqlClause, query);
+            var entitySql = translator.Translate();
+            var pagedSql = entitySql.Append(GetGroupBy(isContent, isMedia, false)).OrderBy("umbracoNode.id");
+
+            IEnumerable<IUmbracoEntity> result;
+
+            if (isMedia)
+            {
+                //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag!
+                var pagedResult = _work.Database.Page<dynamic>(pageIndex + 1, pageSize, pagedSql);
+                
+                var ids = pagedResult.Items.Select(x => (int) x.id).InGroupsOf(2000);
+                var entities = pagedResult.Items.Select(factory.BuildEntityFromDynamic).Cast<IUmbracoEntity>().ToList();
+
+                //Now we need to merge in the property data since we need paging and we can't do this the way that the big media query was working before
+                foreach (var idGroup in ids)
+                {
+                    var propSql = GetPropertySql(Constants.ObjectTypes.Media)
+                        .Where("contentNodeId IN (@ids)", new {ids = idGroup})
+                        .OrderBy("contentNodeId");
+
+                    //This does NOT fetch all data into memory in a list, this will read
+                    // over the records as a data reader, this is much better for performance and memory,
+                    // but it means that during the reading of this data set, nothing else can be read
+                    // from SQL server otherwise we'll get an exception.
+                    var allPropertyData = _work.Database.Query<dynamic>(propSql);
+
+                    //keep track of the current property data item being enumerated
+                    var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
+                    var hasCurrent = false; // initially there is no enumerator.Current
+
+                    try
+                    {
+                        //This must be sorted by node id (which is done by SQL) because this is how we are sorting the query to lookup property types above,
+                        // which allows us to more efficiently iterate over the large data set of property values.
+                        foreach (var entity in entities)
+                        {
+                            // assemble the dtos for this def
+                            // use the available enumerator.Current if any else move to next
+                            while (hasCurrent || propertyDataSetEnumerator.MoveNext())
+                            {
+                                if (propertyDataSetEnumerator.Current.contentNodeId == entity.Id)
+                                {
+                                    hasCurrent = false; // enumerator.Current is not available
+
+                                    //the property data goes into the additional data
+                                    entity.AdditionalData[propertyDataSetEnumerator.Current.propertyTypeAlias] = new UmbracoEntity.EntityProperty
+                                    {
+                                        PropertyEditorAlias = propertyDataSetEnumerator.Current.propertyEditorAlias,
+                                        Value = StringExtensions.IsNullOrWhiteSpace(propertyDataSetEnumerator.Current.dataNtext)
+                                            ? propertyDataSetEnumerator.Current.dataNvarchar
+                                            : StringExtensions.ConvertToJsonIfPossible(propertyDataSetEnumerator.Current.dataNtext)
+                                    };
+                                }
+                                else
+                                {
+                                    hasCurrent = true; // enumerator.Current is available for another def
+                                    break; // no more propertyDataDto for this def
+                                }
+                            }
+                        }
+                    }
+                    finally
+                    {
+                        propertyDataSetEnumerator.Dispose();
+                    }
+                }
+                
+                result = entities;
+            }
+            else
+            {   
+                var pagedResult = _work.Database.Page<dynamic>(pageIndex + 1, pageSize, pagedSql);
+                result = pagedResult.Items.Select(factory.BuildEntityFromDynamic).Cast<IUmbracoEntity>().ToList();
+            }
+
+            //The total items from the PetaPoco page query will be wrong due to the Outer join used on parent, depending on the search this will 
+            //return duplicate results when the COUNT is used in conjuction with it, so we need to get the total on our own.
+
+            //generate a query that does not contain the LEFT Join for parent, this would cause
+            //the COUNT(*) query to return the wrong 
+            var sqlCountClause = GetBaseWhere(
+                (isC, isM, f) => GetBase(isC, isM, f, true), //true == is a count query
+                isContent, isMedia, sql =>
+                {
+                    if (filter != null)
+                    {
+                        foreach (var filterClaus in filter.GetWhereClauses())
+                        {
+                            sql.Where(filterClaus.Item1, filterClaus.Item2);
+                        }
+                    }
+                }, objectTypeId);
+            var translatorCount = new SqlTranslator<IUmbracoEntity>(sqlCountClause, query);
+            var countSql = translatorCount.Translate();
+
+            totalRecords = _work.Database.ExecuteScalar<int>(countSql);
+
+            return result;
+        }
+
         public IUmbracoEntity GetByKey(Guid key)
         {
             var sql = GetBaseWhere(GetBase, false, false, key);
@@ -68,11 +182,12 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(key, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
 
@@ -80,14 +195,16 @@ namespace Umbraco.Core.Persistence.Repositories
             }
             else
             {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;
 
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;                
             }
             
             
@@ -112,29 +229,29 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(id, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
 
                 return entities.FirstOrDefault();
             }
             else
-            {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;
-
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+            {                
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;
             }
-
-            
         }
 
         public virtual IEnumerable<IUmbracoEntity> GetAll(Guid objectTypeId, params int[] ids)
@@ -171,22 +288,21 @@ namespace Umbraco.Core.Persistence.Repositories
 
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too                
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
-                foreach (var entity in entities)
-                {
-                    yield return entity;
-                }
+                return entities;
             }
             else
             {
-                var dtos = _work.Database.Fetch<dynamic>(sql);
-                foreach (var entity in dtos.Select(dto => factory.BuildEntityFromDynamic(dto)))
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
                 {
-                    yield return entity;
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
                 }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();                
             }
         }
 
@@ -231,8 +347,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     }
                 });
 
-                //treat media differently for now 
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too    
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, mediaSql);
                 return entities;
@@ -241,8 +356,15 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData
                 var finalSql = entitySql.Append(GetGroupBy(isContent, false));
-                var dtos = _work.Database.Fetch<dynamic>(finalSql);
-                return dtos.Select(factory.BuildEntityFromDynamic).Cast<IUmbracoEntity>().ToList();
+
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(finalSql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();
             }
         }
 
@@ -278,11 +400,9 @@ namespace Umbraco.Core.Persistence.Repositories
             return GetFullSqlForMedia(entitySql.Append(GetGroupBy(isContent, true, false)), filter);
         }
 
-        private Sql GetFullSqlForMedia(Sql entitySql, Action<Sql> filter = null)
+        private Sql GetPropertySql(string nodeObjectType)
         {
-            //this will add any dataNvarchar property to the output which can be added to the additional properties
-
-            var joinSql = new Sql()
+            var sql = new Sql()
                 .Select("contentNodeId, versionId, dataNvarchar, dataNtext, propertyEditorAlias, alias as propertyTypeAlias")
                 .From<PropertyDataDto>()
                 .InnerJoin<NodeDto>()
@@ -291,7 +411,16 @@ namespace Umbraco.Core.Persistence.Repositories
                 .On<PropertyTypeDto, PropertyDataDto>(dto => dto.Id, dto => dto.PropertyTypeId)
                 .InnerJoin<DataTypeDto>()
                 .On<PropertyTypeDto, DataTypeDto>(dto => dto.DataTypeId, dto => dto.DataTypeId)
-                .Where("umbracoNode.nodeObjectType = @nodeObjectType", new {nodeObjectType = Constants.ObjectTypes.Media});
+                .Where("umbracoNode.nodeObjectType = @nodeObjectType", new { nodeObjectType = nodeObjectType });
+
+            return sql;
+        }
+
+        private Sql GetFullSqlForMedia(Sql entitySql, Action<Sql> filter = null)
+        {
+            //this will add any dataNvarchar property to the output which can be added to the additional properties
+
+            var joinSql = GetPropertySql(Constants.ObjectTypes.Media);
 
             if (filter != null)
             {
@@ -310,34 +439,53 @@ namespace Umbraco.Core.Persistence.Repositories
                 .OrderBy("sortOrder, id");
 
             return wrappedSql;
-        }        
+        }
 
         protected virtual Sql GetBase(bool isContent, bool isMedia, Action<Sql> customFilter)
         {
-            var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate",
-                                  "COUNT(parent.parentID) as children"
-                              };
+            return GetBase(isContent, isMedia, customFilter, false);
+        }
 
-            if (isContent || isMedia)
+        protected virtual Sql GetBase(bool isContent, bool isMedia, Action<Sql> customFilter, bool isCount)
+        {
+            var columns = new List<object>();
+            if (isCount)
             {
-                columns.Add("published.versionId as publishedVersion");
-                columns.Add("latest.versionId as newestVersion");
-                columns.Add("contenttype.alias");
-                columns.Add("contenttype.icon");
-                columns.Add("contenttype.thumbnail");
-                columns.Add("contenttype.isContainer");
+                columns.Add("COUNT(*)");
+            }
+            else
+            {
+                columns.AddRange(new List<object>
+                {
+                    "umbracoNode.id",
+                    "umbracoNode.trashed",
+                    "umbracoNode.parentID",
+                    "umbracoNode.nodeUser",
+                    "umbracoNode.level",
+                    "umbracoNode.path",
+                    "umbracoNode.sortOrder",
+                    "umbracoNode.uniqueID",
+                    "umbracoNode.text",
+                    "umbracoNode.nodeObjectType",
+                    "umbracoNode.createDate",
+                    "COUNT(parent.parentID) as children"
+                });
+
+                if (isContent || isMedia)
+                {
+                    if (isContent)
+                    {
+                        //only content has/needs this info
+                        columns.Add("published.versionId as publishedVersion");
+                        columns.Add("document.versionId as newestVersion");
+                        columns.Add("contentversion.id as versionId");
+                    }
+
+                    columns.Add("contenttype.alias");
+                    columns.Add("contenttype.icon");
+                    columns.Add("contenttype.thumbnail");
+                    columns.Add("contenttype.isContainer");
+                }
             }
 
             //Creates an SQL query to return a single row for the entity
@@ -345,20 +493,28 @@ namespace Umbraco.Core.Persistence.Repositories
             var entitySql = new Sql()
                 .Select(columns.ToArray())
                 .From("umbracoNode umbracoNode");
-                
+
             if (isContent || isMedia)
             {
-                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id")
-                   .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published")
-                   .On("umbracoNode.id = published.nodeId")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest")
-                   .On("umbracoNode.id = latest.nodeId");
+                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id");
+
+                if (isContent)
+                {
+                    //only content has/needs this info
+                    entitySql                        
+                        .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id")
+                        .InnerJoin("cmsContentVersion contentversion").On("contentversion.VersionId = document.versionId")
+                        .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published")
+                        .On("umbracoNode.id = published.nodeId");
+                }
+
+                entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType");                
             }
 
-            entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id");
+            if (isCount == false)
+            {
+                entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id");
+            }
 
             if (customFilter != null)
             {
@@ -372,22 +528,42 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = baseQuery(isContent, isMedia, filter)
                 .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, int id)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.id = @Id", new { Id = id })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.id = @Id", new { Id = id });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, Guid key)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
@@ -396,6 +572,12 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new {Id = id, NodeObjectType = nodeObjectType});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
@@ -404,36 +586,46 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new { UniqueID = key, NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
         protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
         {
             var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate"
+            };
 
             if (isContent || isMedia)
             {
-                columns.Add("published.versionId");
-                columns.Add("latest.versionId");
+                if (isContent)
+                {
+                    columns.Add("published.versionId");
+                    columns.Add("document.versionId");
+                    columns.Add("contentversion.id");
+                }
                 columns.Add("contenttype.alias");
                 columns.Add("contenttype.icon");
                 columns.Add("contenttype.thumbnail");
-                columns.Add("contenttype.isContainer");
+                columns.Add("contenttype.isContainer");                
             }
-            
+
             var sql = new Sql()
                 .GroupBy(columns.ToArray());
 
@@ -444,7 +636,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             return sql;
         }
-        
+
         #endregion
 
         /// <summary>
@@ -470,33 +662,7 @@ namespace Umbraco.Core.Persistence.Repositories
             return _work.Database.ExecuteScalar<int>(sql) > 0;
         }
 
-        #region umbracoNode POCO - Extends NodeDto
-        [TableName("umbracoNode")]
-        [PrimaryKey("id")]
-        [ExplicitColumns]
-        internal class UmbracoEntityDto : NodeDto
-        {
-            [Column("children")]
-            public int Children { get; set; }
-
-            [Column("publishedVersion")]
-            public Guid PublishedVersion { get; set; }
-
-            [Column("newestVersion")]
-            public Guid NewestVersion { get; set; }
-
-            [Column("alias")]
-            public string Alias { get; set; }
-
-            [Column("icon")]
-            public string Icon { get; set; }
-
-            [Column("thumbnail")]
-            public string Thumbnail { get; set; }
-
-            [ResultColumn]
-            public List<UmbracoPropertyDto> UmbracoPropertyDtos { get; set; }
-        }
+        #region private classes
         
         [ExplicitColumns]
         internal class UmbracoPropertyDto
@@ -580,6 +746,99 @@ namespace Umbraco.Core.Persistence.Repositories
                 return prev;
             }
         }
+
+        private class EntityDefinitionCollection : KeyedCollection<int, EntityDefinition>
+        {
+            protected override int GetKeyForItem(EntityDefinition item)
+            {
+                return item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(EntityDefinition item)
+            {
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                EntityDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Id);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+
+                base.Add(item);
+                return true;
+            }
+
+            private bool TryGetValue(int key, out EntityDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        private class EntityDefinition
+        {
+            private readonly UmbracoEntityFactory _factory;
+            private readonly dynamic _entity;
+            private readonly bool _isContent;
+            private readonly bool _isMedia;
+
+            public EntityDefinition(UmbracoEntityFactory factory, dynamic entity, bool isContent, bool isMedia)
+            {
+                _factory = factory;
+                _entity = entity;
+                _isContent = isContent;
+                _isMedia = isMedia;
+            }
+
+            public IUmbracoEntity BuildFromDynamic()
+            {
+                return _factory.BuildEntityFromDynamic(_entity);
+            }
+
+            public int Id
+            {
+                get { return _entity.id; }
+            }
+
+            public int VersionId
+            {
+                get
+                {
+                    if (_isContent || _isMedia)
+                    {
+                        return _entity.versionId;
+                    }
+                    return _entity.id;
+                }
+            }
+        }
         #endregion
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IEntityRepository.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using Umbraco.Core.Models.EntityBase;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Querying;
 
 namespace Umbraco.Core.Persistence.Repositories
@@ -16,6 +17,21 @@ namespace Umbraco.Core.Persistence.Repositories
         IEnumerable<IUmbracoEntity> GetByQuery(IQuery<IUmbracoEntity> query);
         IEnumerable<IUmbracoEntity> GetByQuery(IQuery<IUmbracoEntity> query, Guid objectTypeId);
 
+        /// <summary>
+        /// Gets paged results
+        /// </summary>
+        /// <param name="query">Query to excute</param>
+        /// <param name="objectTypeId"></param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="filter"></param>
+        /// <returns>An Enumerable list of <see cref="IUmbracoEntity"/> objects</returns>
+        IEnumerable<IUmbracoEntity> GetPagedResultsByQuery(IQuery<IUmbracoEntity> query, Guid objectTypeId, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy, Direction orderDirection, IQuery<IUmbracoEntity> filter = null);
+
         /// <summary>
         /// Returns true if the entity exists
         /// </summary>

src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs

@@ -23,7 +23,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="serializer">The serializer to convert TEntity to Xml</param>
         /// <param name="groupSize">Structures will be rebuilt in chunks of this size</param>
         /// <param name="contentTypeIds"></param>
-        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null);
+        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null);
 
         /// <summary>
         /// Get the total count of entities

src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs

@@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMediaFromDto(dto, dto.VersionId, sql);
+            var content = CreateMediaFromDto(dto, sql);
 
             return content;
         }
@@ -94,6 +94,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 .InnerJoin<NodeDto>(SqlSyntax)
                 .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax)
+                //TODO: IF we want to enable querying on content type information this will need to be joined
+                //.InnerJoin<ContentTypeDto>(SqlSyntax)
+                //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
                 .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
             return sql;
         }
@@ -161,25 +164,27 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sqlFull);
-            var content = new IMedia[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMedia, bool>>();
+            var defs = new DocumentDefinitionCollection();
 
             //track the looked up content types, even though the content types are cached
             // they still need to be deep cloned out of the cache and we don't want to add
             // the overhead of deep cloning them on every item in this loop
             var contentTypes = new Dictionary<int, IMediaType>();
 
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
-
                 // if the cache contains the item, use it
                 if (withCache)
                 {
                     var cached = IsolatedCache.GetCacheItem<IMedia>(GetCacheIdKey<IMedia>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, media doesn't 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.VersionId)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMedia, bool>(cached, true));
                         continue;
                     }
                 }
@@ -197,39 +202,34 @@ namespace Umbraco.Core.Persistence.Repositories
                     contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
                     contentTypes[dto.ContentDto.ContentTypeId] = contentType;
                 }
-                
-                content[i] = MediaFactory.BuildEntity(dto, contentType);
 
-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.VersionDate,
-                    dto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    content.Add(new Tuple<IMedia, bool>(MediaFactory.BuildEntity(dto, contentType), false));
+                }
             }
 
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
 
-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         public override IMedia GetByVersion(Guid versionId)
@@ -243,7 +243,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMediaFromDto(dto, versionId, sql);
+            var content = CreateMediaFromDto(dto, sql);
 
             return content;
         }
@@ -263,10 +263,12 @@ namespace Umbraco.Core.Persistence.Repositories
                 // get the next group of nodes
                 var query = GetBaseQuery(false);
                 if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                {
+                    query = query.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
                 query = query
                     .Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
+                    .Where<NodeDto>(x => x.Trashed == false, SqlSyntax)
                     .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
                 var sql = SqlSyntax.SelectTop(query, groupSize);
                 var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
@@ -290,7 +292,38 @@ namespace Umbraco.Core.Persistence.Repositories
                         Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
                     }
                 }
-                baseId = xmlItems.Last().NodeId;
+                baseId = xmlItems[xmlItems.Count - 1].NodeId;
+            }
+
+            //now delete the items that shouldn't be there
+            var allMediaIds = Database.Fetch<int>(GetBaseQuery(BaseQueryType.Ids).Where<NodeDto>(x => x.Trashed == false, SqlSyntax));
+            var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+                
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allMediaIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }
             }
         }
 
@@ -526,20 +559,19 @@ namespace Umbraco.Core.Persistence.Repositories
         /// Private method to create a media object from a ContentDto
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
+        private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql)
         {
             var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
             
             var media = MediaFactory.BuildEntity(dto, contentType);
 
-            var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
+            var docDef = new DocumentDefinition(dto, contentType);
 
             var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });
 
-            media.Properties = properties[dto.NodeId];
+            media.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs

@@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMemberFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateMemberFromDto(dto, sql);
 
             return content;
 
@@ -448,16 +448,16 @@ namespace Umbraco.Core.Persistence.Repositories
             var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
             var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            var member = factory.BuildEntity(dto);
 
-            var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
+            var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) });
 
-            media.Properties = properties[dto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            ((Entity)member).ResetDirtyProperties(false);
+            return member;
 
         }
 
@@ -658,20 +658,21 @@ namespace Umbraco.Core.Persistence.Repositories
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
 
-            var content = new IMember[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMember, bool>>();
+            var defs = new DocumentDefinitionCollection();
 
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
-
                 // if the cache contains the item, use it
                 if (withCache)
                 {
                     var cached = IsolatedCache.GetCacheItem<IMember>(GetCacheIdKey<IMember>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, members dont 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.ContentVersionDto.VersionId)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMember, bool>(cached, true));
                         continue;
                     }
                 }
@@ -679,60 +680,54 @@ namespace Umbraco.Core.Persistence.Repositories
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
                 var contentType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new MemberFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
 
                 // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.ContentVersionDto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType)))
+                {
+                    content.Add(new Tuple<IMember, bool>(MemberFactory.BuildEntity(dto, contentType), false));
+                }
             }
 
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
 
-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity)cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// <summary>
         /// Private method to create a member object from a MemberDto
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IMember CreateMemberFromDto(MemberDto dto, Guid versionId, Sql docSql)
+        private IMember CreateMemberFromDto(MemberDto dto, Sql docSql)
         {
             var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
             var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.ContentVersionDto.NodeId);
             var member = factory.BuildEntity(dto);
 
-            var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType);
+            var docDef = new DocumentDefinition(dto.ContentVersionDto, memberType);
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            member.Properties = properties[dto.ContentVersionDto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs

@@ -38,10 +38,9 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 sql.Where("umbracoAccess.id IN (@ids)", new { ids = ids });
             }
-
-            sql.OrderBy<AccessDto>(x => x.NodeId, SqlSyntax);
-
+            
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -53,6 +52,7 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = translator.Translate();
 
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -63,7 +63,9 @@ namespace Umbraco.Core.Persistence.Repositories
             sql.Select("*")
                 .From<AccessDto>(SqlSyntax)
                 .LeftJoin<AccessRuleDto>(SqlSyntax)
-                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId);
+                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId)
+                //MUST be ordered by this GUID ID for the AccessRulesRelator to work
+                .OrderBy<AccessDto>(dto => dto.Id, SqlSyntax);
 
             return sql;
         }
@@ -104,6 +106,11 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 rule.AccessId = entity.Key;
                 Database.Insert(rule);
+                //update the id so HasEntity is correct
+                var entityRule = entity.Rules.First(x => x.Key == rule.Id);
+                entityRule.Id = entityRule.Key.GetHashCode();
+                //double make sure that this is set since it is possible to add rules via ctor without AddRule
+                entityRule.AccessEntryId = entity.Key;
             }
 
             entity.ResetDirtyProperties();
@@ -124,7 +131,7 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 if (rule.HasIdentity)
                 {
-                    var count = Database.Update(dto.Rules.Single(x => x.Id == rule.Key));
+                    var count = Database.Update(dto.Rules.First(x => x.Id == rule.Key));
                     if (count == 0)
                     {
                         throw new InvalidOperationException("No rows were updated for the access rule");
@@ -150,6 +157,8 @@ namespace Umbraco.Core.Persistence.Repositories
                 Database.Delete<AccessRuleDto>("WHERE id=@Id", new {Id = removedRule});
             }
 
+            entity.ClearRemovedRules();
+
             entity.ResetDirtyProperties();
         }
 
@@ -158,4 +167,4 @@ namespace Umbraco.Core.Persistence.Repositories
             return entity.Key;
         }
     }
-}
+}

src/Umbraco.Core/Persistence/Repositories/RelationRepository.cs

@@ -73,12 +73,16 @@ namespace Umbraco.Core.Persistence.Repositories
             RelationFactory factory = null;
             var relationTypeId = -1;
 
+            // the ToList() here is important because we are using _relationTypeRepository and we
+            // cannot wait until the result is actually enumerated to do so, because that would
+            // mean we kinda capture the current unit of work and reuse it after it's been disposed
+
             return dtos.Select(x =>
             {
                 if (relationTypeId != x.RelationType)
                     factory = new RelationFactory(_relationTypeRepository.Get(relationTypeId = x.RelationType));
                 return DtoToEntity(x, factory);
-            });
+            }).ToList();
         }
 
         private static IRelation DtoToEntity(RelationDto dto, RelationFactory factory)

src/Umbraco.Core/Persistence/Repositories/TemplateRepository.cs

@@ -270,12 +270,11 @@ namespace Umbraco.Core.Persistence.Repositories
 
             // once content has been set, "template on disk" are not "on disk" anymore
             template.Content = content;
+            SetVirtualPath(template);
 
             if (dto.Design == content) return;
             dto.Design = content;
             Database.Update(dto); // though... we don't care about the db value really??!!
-
-            SetVirtualPath(template);
         }
 
         protected override void PersistDeletedItem(ITemplate entity)

src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Data.SqlTypes;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -33,6 +34,11 @@ namespace Umbraco.Core.Persistence.Repositories
     {
         private readonly IContentSection _contentSection;
 
+        /// <summary>
+        /// This is used for unit tests ONLY
+        /// </summary>
+        internal static bool ThrowOnWarning = false;
+
         protected VersionableRepositoryBase(IScopeUnitOfWork work, CacheHelper cache, ILogger logger, ISqlSyntaxProvider sqlSyntax, IContentSection contentSection)
             : base(work, cache, logger, sqlSyntax)
         {
@@ -503,7 +509,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="sql"></param>
         /// <param name="documentDefs"></param>
         /// <returns></returns>
-        protected IDictionary<int, PropertyCollection> GetPropertyCollection(
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
             Sql sql,
             IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
@@ -516,11 +522,11 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="pagingSqlQuery"></param>
         /// <param name="documentDefs"></param>
         /// <returns></returns>
-        protected IDictionary<int, PropertyCollection> GetPropertyCollection(
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
             PagingSqlQuery pagingSqlQuery,
             IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
-            if (documentDefs.Count == 0) return new Dictionary<int, PropertyCollection>();
+            if (documentDefs.Count == 0) return new Dictionary<Guid, PropertyCollection>();
 
             //initialize to the query passed in
             var docSql = pagingSqlQuery.PrePagedSql;
@@ -576,16 +582,16 @@ ON cmsPropertyData.propertytypeid = cmsPropertyType.id
 INNER JOIN 
 	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
 ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-ORDER BY contentNodeId, propertytypeid
+ORDER BY contentNodeId, versionId, propertytypeid
 ", docSql.Arguments);
-
+            
             //This does NOT fetch all data into memory in a list, this will read
             // over the records as a data reader, this is much better for performance and memory,
             // but it means that during the reading of this data set, nothing else can be read
             // from SQL server otherwise we'll get an exception.
             var allPropertyData = Database.Query<PropertyDataDto>(propSql);
 
-            var result = new Dictionary<int, PropertyCollection>();
+            var result = new Dictionary<Guid, PropertyCollection>();
             var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
             //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
             var resolvedCompositionProperties = new Dictionary<int, PropertyType[]>();
@@ -594,11 +600,13 @@ ORDER BY contentNodeId, propertytypeid
             var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
             var hasCurrent = false; // initially there is no enumerator.Current
 
+            var comparer = new DocumentDefinitionComparer(SqlSyntax);
+
             try
             {
                 //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
                 // which allows us to more efficiently iterate over the large data set of property values
-                foreach (var def in documentDefs.OrderBy(x => x.Id))
+                foreach (var def in documentDefs.OrderBy(x => x.Id).ThenBy(x => x.Version, comparer))
                 {
                     // get the resolved properties from our local cache, or resolve them and put them in cache
                     PropertyType[] compositionProperties;
@@ -617,15 +625,17 @@ ORDER BY contentNodeId, propertytypeid
                     var propertyDataDtos = new List<PropertyDataDto>();
                     while (hasCurrent || propertyDataSetEnumerator.MoveNext())
                     {
-                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        //Not checking null on VersionId because it can never be null - no idea why it's set to nullable
+                        // ReSharper disable once PossibleInvalidOperationException
+                        if (propertyDataSetEnumerator.Current.VersionId.Value == def.Version)
                         {
                             hasCurrent = false; // enumerator.Current is not available
                             propertyDataDtos.Add(propertyDataSetEnumerator.Current);
                         }
                         else
                         {
-                            hasCurrent = true; // enumerator.Current is available for another def
-                            break; // no more propertyDataDto for this def
+                            hasCurrent = true;  // enumerator.Current is available for another def
+                            break;              // no more propertyDataDto for this def
                         }
                     }
 
@@ -660,11 +670,19 @@ ORDER BY contentNodeId, propertytypeid
                         }
                     }
 
-                    if (result.ContainsKey(def.Id))
+                    if (result.ContainsKey(def.Version))
                     {
-                        Logger.Warn<VersionableRepositoryBase<TId, TEntity>>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name);
+                        var msg = string.Format("The query returned multiple property sets for document definition {0}, {1}, {2}", def.Id, def.Version, def.Composition.Name);
+                        if (ThrowOnWarning)
+                        {
+                            throw new InvalidOperationException(msg);
+                        }
+                        else
+                        {
+                            Logger.Warn<VersionableRepositoryBase<TId, TEntity>>(msg);
+                        }
                     }
-                    result[def.Id] = new PropertyCollection(properties);
+                    result[def.Version] = new PropertyCollection(properties);
                 }
             }
             finally
@@ -687,8 +705,8 @@ ORDER BY contentNodeId, propertytypeid
                 case "NAME":
                     return "umbracoNode.text";
                 case "PUBLISHED":
-                    return "cmsDocument.published";
                 case "OWNER":
+                    return "cmsDocument.published";
                     //TODO: This isn't going to work very nicely because it's going to order by ID, not by letter
                     return "umbracoNode.nodeUser";
                 // Members only
@@ -772,25 +790,159 @@ ORDER BY contentNodeId, propertytypeid
         /// <returns></returns>
         protected abstract Sql GetBaseQuery(BaseQueryType queryType);
 
+        internal class DocumentDefinitionCollection : KeyedCollection<ValueType, DocumentDefinition>
+        {
+            private readonly bool _includeAllVersions;
+
+            /// <summary>
+            /// Constructor specifying if all versions should be allowed, in that case the key for the collection becomes the versionId (GUID)
+            /// </summary>
+            /// <param name="includeAllVersions"></param>
+            public DocumentDefinitionCollection(bool includeAllVersions = false)
+            {
+                _includeAllVersions = includeAllVersions;
+            }
+
+            protected override ValueType GetKeyForItem(DocumentDefinition item)
+            {
+                return _includeAllVersions ? (ValueType)item.Version : item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(DocumentDefinition item)
+            {
+                //if we are including all versions then just add, we aren't checking for latest
+                if (_includeAllVersions)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                DocumentDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Version);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+                
+                base.Add(item);
+                return true;
+            }
+          
+            public bool TryGetValue(ValueType key, out DocumentDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        /// <summary>
+        /// A custom comparer required for sorting entities by GUIDs to match how the sorting of GUIDs works on SQL server
+        /// </summary>
+        /// <remarks>
+        /// MySql sorts GUIDs as a string, MSSQL sorts based on byte sections, this comparer will allow sorting GUIDs to be the same as how SQL server does
+        /// </remarks>
+        private class DocumentDefinitionComparer : IComparer<Guid>
+        {
+            private readonly ISqlSyntaxProvider _sqlSyntax;
+
+            public DocumentDefinitionComparer(ISqlSyntaxProvider sqlSyntax)
+            {
+                _sqlSyntax = sqlSyntax;
+            }
+
+            public int Compare(Guid x, Guid y)
+            {
+                //MySql sorts on GUIDs as strings (i.e. normal)
+                if (_sqlSyntax is MySqlSyntaxProvider)
+                {
+                    return x.CompareTo(y);
+                }
+
+                //MSSQL doesn't it sorts them on byte sections!
+                return new SqlGuid(x).CompareTo(new SqlGuid(y));
+            }
+        }
+
         internal class DocumentDefinition
         {
             /// <summary>
             /// Initializes a new instance of the <see cref="T:System.Object"/> class.
             /// </summary>
-            public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition)
+            public DocumentDefinition(DocumentDto dto, IContentTypeComposition composition)
             {
-                Id = id;
-                Version = version;
-                VersionDate = versionDate;
-                CreateDate = createDate;
+                DocumentDto = dto;
+                ContentVersionDto = dto.ContentVersionDto;
                 Composition = composition;
             }
 
-            public int Id { get; set; }
-            public Guid Version { get; set; }
-            public DateTime VersionDate { get; set; }
-            public DateTime CreateDate { get; set; }
-            public IContentTypeComposition Composition { get; set; }
+            public DocumentDefinition(ContentVersionDto dto, IContentTypeComposition composition)
+            {
+                ContentVersionDto = dto;
+                Composition = composition;
+            }
+
+            public DocumentDto DocumentDto { get; private set; }
+            public ContentVersionDto ContentVersionDto { get; private set; }
+
+            public int Id
+            {
+                get { return ContentVersionDto.NodeId; }
+            }
+            
+            public Guid Version
+            {
+                get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; }
+            }
+
+            /// <summary>
+            /// This is used to determien which version is the most recent
+            /// </summary>
+            public int VersionId
+            {
+                get { return ContentVersionDto.Id; }
+            }
+
+            public DateTime VersionDate
+            {
+                get { return ContentVersionDto.VersionDate; }
+            }
+
+            public DateTime CreateDate
+            {
+                get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; }
+            }
+
+            public IContentTypeComposition Composition { get; set; }            
+
+            
         }
 
         /// <summary>

src/Umbraco.Core/Properties/AssemblyInfo.cs

@@ -47,5 +47,9 @@ using System.Security.Permissions;
 [assembly: InternalsVisibleTo("Umbraco.Deploy.UI")]
 [assembly: InternalsVisibleTo("Umbraco.Deploy.Cloud")]
 
+[assembly: InternalsVisibleTo("Umbraco.Forms.Core")]
+[assembly: InternalsVisibleTo("Umbraco.Forms.Core.Providers")]
+[assembly: InternalsVisibleTo("Umbraco.Forms.Web")]
+
 //allow this to be mocked in our unit tests
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]

src/Umbraco.Core/PropertyEditors/ValueConverters/CheckboxListValueConverter.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(IEnumerable<string>))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class CheckboxListValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.CheckBoxListAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            var sourceString = (source ?? string.Empty).ToString();
+
+            if (string.IsNullOrEmpty(sourceString))
+                return Enumerable.Empty<string>();
+
+            var values =
+                sourceString.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(v => v.Trim());
+
+            return values;
+        }        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/ColorPickerValueConverter.cs

@@ -1,18 +1,28 @@
-using Umbraco.Core.Models.PublishedContent;
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(string))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class ColorPickerValueConverter : PropertyValueConverterBase
     {
         public override bool IsConverter(PublishedPropertyType propertyType)
         {
-            return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.ColorPickerAlias);
+	        if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+	        {
+				return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.ColorPickerAlias);
+			}
+	        return false;
         }
 
         public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
         {
             // make sure it's a string
-            return source.ToString();
+            return source == null ? string.Empty : source.ToString();
         }
+        
     }
 }

src/Umbraco.Core/PropertyEditors/ValueConverters/DatePickerValueConverter.cs

@@ -6,6 +6,7 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(DateTime))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class DatePickerValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/DecimalValueConverter.cs

@@ -3,6 +3,7 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(decimal))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class DecimalValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/DropdownListMultipleValueConverter.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(IEnumerable<string>))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class DropdownListMultipleValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.DropDownListMultipleAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            var sourceString = (source ?? "").ToString();
+
+            if (string.IsNullOrEmpty(sourceString))
+                return Enumerable.Empty<string>();
+
+            var values =
+                sourceString.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(v => v.Trim());
+
+            return values;
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/DropdownListMultipleWithKeysValueConverter.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(IEnumerable<int>))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class DropdownListMultipleWithKeysValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.DropdownlistMultiplePublishKeysAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            if (source == null)
+                return new int[] { };
+
+            var prevalueIds = source.ToString()
+                                    .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
+                                    .Select(p => p.Trim())
+                                    .Select(int.Parse)
+                                    .ToArray();
+
+            return prevalueIds;
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/DropdownListValueConverter.cs

@@ -0,0 +1,28 @@
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(string))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class DropdownListValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.DropDownListAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            return source == null ? string.Empty : source.ToString();
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/DropdownListWithKeysValueConverter.cs

@@ -0,0 +1,31 @@
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(int))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class DropdownListWithKeysValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.DropdownlistPublishingKeysAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            var intAttempt = source.TryConvertTo<int>();
+            if (intAttempt.Success)
+                return intAttempt.Result;
+
+            return null;
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/EmailAddressValueConverter.cs

@@ -0,0 +1,27 @@
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(string))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class EmailAddressValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.EmailAddressAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            return source.ToString();
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/IntegerValueConverter.cs

@@ -2,6 +2,7 @@
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(int))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class IntegerValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/MarkdownEditorValueConverter.cs

@@ -4,6 +4,7 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(IHtmlString))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class MarkdownEditorValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/MemberGroupPickerValueConverter.cs

@@ -0,0 +1,27 @@
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(string))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class MemberGroupPickerValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.MemberGroupPickerAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            return source == null ? string.Empty : source.ToString();
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/MultipleTextStringValueConverter.cs

@@ -8,6 +8,7 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(IEnumerable<string>))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class MultipleTextStringValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/RadioButtonListValueConverter.cs

@@ -0,0 +1,31 @@
+using System;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(int))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class RadioButtonListValueConverter : PropertyValueConverterBase
+    {
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.RadioButtonListAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            var intAttempt = source.TryConvertTo<int>();
+            if (intAttempt.Success)
+                return intAttempt.Result;
+
+            return null;
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/SliderValueConverter.cs

@@ -0,0 +1,100 @@
+using System;
+using System.Linq;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models;
+using Umbraco.Core.Models.PublishedContent;
+using Umbraco.Core.Services;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    public class SliderValueConverter : PropertyValueConverterBase, IPropertyValueConverterMeta
+    {
+        private readonly IDataTypeService _dataTypeService;
+
+        //TODO: Remove this ctor in v8 since the other one will use IoC
+        public SliderValueConverter()
+            : this(ApplicationContext.Current.Services.DataTypeService)
+        {
+        }
+
+        public SliderValueConverter(IDataTypeService dataTypeService)
+        {
+            if (dataTypeService == null) throw new ArgumentNullException("dataTypeService");
+            _dataTypeService = dataTypeService;
+        }
+
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.SliderAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            if (source == null)
+            {
+                return null;
+            }
+
+            if (IsRangeDataType(propertyType.DataTypeId))
+            {
+                var rangeRawValues = source.ToString().Split(',');
+                var minimumAttempt = rangeRawValues[0].TryConvertTo<decimal>();
+                var maximumAttempt = rangeRawValues[1].TryConvertTo<decimal>();
+
+                if (minimumAttempt.Success && maximumAttempt.Success)
+                {
+                    return new Range<decimal>() { Maximum = maximumAttempt.Result, Minimum = minimumAttempt.Result };
+                }
+            }
+
+            var valueAttempt = source.ToString().TryConvertTo<decimal>();
+            if (valueAttempt.Success)
+            {
+                return valueAttempt.Result;
+            }
+
+            // Something failed in the conversion of the strings to decimals
+            return null;
+
+        }
+
+        public Type GetPropertyValueType(PublishedPropertyType propertyType)
+        {
+            if (IsRangeDataType(propertyType.DataTypeId))
+            {
+                return typeof(Range<decimal>);
+            }
+            return typeof(decimal);
+        }
+
+        public PropertyCacheLevel GetPropertyCacheLevel(PublishedPropertyType propertyType, PropertyCacheValue cacheValue)
+        {
+            return PropertyCacheLevel.Content;
+        }
+
+        /// <summary>
+        /// Discovers if the slider is set to range mode.
+        /// </summary>
+        /// <param name="dataTypeId">
+        /// The data type id.
+        /// </param>
+        /// <returns>
+        /// The <see cref="bool"/>.
+        /// </returns>
+        private bool IsRangeDataType(int dataTypeId)
+        {
+            // ** This must be cached (U4-8862) **            
+            var enableRange =
+                _dataTypeService.GetPreValuesCollectionByDataTypeId(dataTypeId)
+                    .PreValuesAsDictionary.FirstOrDefault(
+                        x => string.Equals(x.Key, "enableRange", StringComparison.InvariantCultureIgnoreCase)).Value;
+
+            return enableRange != null && enableRange.Value.TryConvertTo<bool>().Result;
+        }
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/TagsValueConverter.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+using Umbraco.Core.Services;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(IEnumerable<string>))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class TagsValueConverter : PropertyValueConverterBase
+    {
+        private readonly IDataTypeService _dataTypeService;
+
+        //TODO: Remove this ctor in v8 since the other one will use IoC
+        public TagsValueConverter()
+            : this(ApplicationContext.Current.Services.DataTypeService)
+        {
+        }
+
+        public TagsValueConverter(IDataTypeService dataTypeService)
+        {
+            if (dataTypeService == null) throw new ArgumentNullException("dataTypeService");
+            _dataTypeService = dataTypeService;
+        }
+
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.InvariantEquals(Constants.PropertyEditors.TagsAlias);
+            }
+            return false;
+        }
+
+        public override object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            // if Json storage type deserialzie and return as string array
+            if (JsonStorageType(propertyType.DataTypeId))
+            {
+                var jArray = JsonConvert.DeserializeObject<JArray>(source.ToString());
+                return jArray.ToObject<string[]>();
+            }
+
+            // Otherwise assume CSV storage type and return as string array
+            var csvTags =
+                source.ToString()
+                    .Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries)
+                    .ToArray();
+            return csvTags;
+        }
+
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            if (source == null)
+            {
+                return null;
+            }
+            return (string[]) source;
+        }
+
+        /// <summary>
+        /// Discovers if the tags data type is storing its data in a Json format
+        /// </summary>
+        /// <param name="dataTypeId">
+        /// The data type id.
+        /// </param>
+        /// <returns>
+        /// The <see cref="bool"/>.
+        /// </returns>
+        private bool JsonStorageType(int dataTypeId)
+        {
+            // ** This must be cached (U4-8862) **
+            var storageType =
+                _dataTypeService.GetPreValuesCollectionByDataTypeId(dataTypeId)
+                    .PreValuesAsDictionary.FirstOrDefault(
+                        x => string.Equals(x.Key, "storageType", StringComparison.InvariantCultureIgnoreCase)).Value;
+
+            if (storageType.Value.InvariantEquals("Json"))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/TextStringValueConverter.cs

@@ -6,11 +6,12 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(string))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class TextStringValueConverter : PropertyValueConverterBase
     {
-        private readonly static string[] PropertyTypeAliases =
+        private static readonly string[] PropertyTypeAliases =
         {
             Constants.PropertyEditors.TextboxAlias,
             Constants.PropertyEditors.TextboxMultipleAlias

src/Umbraco.Core/PropertyEditors/ValueConverters/TinyMceValueConverter.cs

@@ -4,10 +4,11 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
-	/// <summary>
-	/// Value converter for the RTE so that it always returns IHtmlString so that Html.Raw doesn't have to be used.
-	/// </summary>
+    /// <summary>
+    /// Value converter for the RTE so that it always returns IHtmlString so that Html.Raw doesn't have to be used.
+    /// </summary>
     // PropertyCacheLevel.Content is ok here because that version of RTE converter does not parse {locallink} nor executes macros
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(IHtmlString))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class TinyMceValueConverter : PropertyValueConverterBase

src/Umbraco.Core/PropertyEditors/ValueConverters/UploadPropertyConverter.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using Umbraco.Core;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Models.PublishedContent;
+using Umbraco.Core.PropertyEditors;
+
+namespace Umbraco.Core.PropertyEditors.ValueConverters
+{
+    /// <summary>
+    /// The upload property value converter.
+    /// </summary>
+    [DefaultPropertyValueConverter]
+    [PropertyValueType(typeof(string))]
+    [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
+    public class UploadPropertyConverter : PropertyValueConverterBase
+    {
+        /// <summary>
+        /// Checks if this converter can convert the property editor and registers if it can.
+        /// </summary>
+        /// <param name="propertyType">
+        /// The published property type.
+        /// </param>
+        /// <returns>
+        /// The <see cref="bool"/>.
+        /// </returns>
+        public override bool IsConverter(PublishedPropertyType propertyType)
+        {
+            if (UmbracoConfig.For.UmbracoSettings().Content.EnablePropertyValueConverters)
+            {
+                return propertyType.PropertyEditorAlias.Equals(Constants.PropertyEditors.UploadFieldAlias);
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Convert the source object to a string
+        /// </summary>
+        /// <param name="propertyType">
+        /// The published property type.
+        /// </param>
+        /// <param name="source">
+        /// The value of the property
+        /// </param>
+        /// <param name="preview">
+        /// The preview.
+        /// </param>
+        /// <returns>
+        /// The <see cref="object"/>.
+        /// </returns>
+        public override object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
+        {
+            return (source ?? "").ToString();
+        }
+        
+    }
+}

src/Umbraco.Core/PropertyEditors/ValueConverters/YesNoValueConverter.cs

@@ -3,6 +3,7 @@ using Umbraco.Core.Models.PublishedContent;
 
 namespace Umbraco.Core.PropertyEditors.ValueConverters
 {
+    [DefaultPropertyValueConverter]
     [PropertyValueType(typeof(bool))]
     [PropertyValueCache(PropertyCacheValue.All, PropertyCacheLevel.Content)]
     public class YesNoValueConverter : PropertyValueConverterBase

src/Umbraco.Core/Scoping/NoScope.cs

@@ -15,6 +15,7 @@ namespace Umbraco.Core.Scoping
         private bool _disposed;
 
         private UmbracoDatabase _database;
+        private EventMessages _messages;
 
         public NoScope(ScopeProvider scopeProvider)
         {
@@ -61,12 +62,42 @@ namespace Umbraco.Core.Scoping
         /// <inheritdoc />
         public EventMessages Messages
         {
-            get { throw new NotSupportedException(); }
+            get
+            {
+                EnsureNotDisposed();
+                if (_messages != null) return _messages;
+
+                // see comments in Scope
+
+                var factory = ScopeLifespanMessagesFactory.Current;
+                if (factory == null)
+                {
+                    _messages = new EventMessages();
+                }
+                else
+                {
+                    _messages = factory.GetFromHttpContext();
+                    if (_messages == null)
+                        factory.Set(_messages = new EventMessages());
+                }
+
+                return _messages;
+            }
         }
 
         public EventMessages MessagesOrNull
         {
-            get { throw new NotSupportedException(); }
+            get
+            {
+                EnsureNotDisposed();
+
+                // see comments in Scope
+
+                if (_messages != null) return _messages;
+
+                var factory = ScopeLifespanMessagesFactory.Current;
+                return factory == null ? null : factory.GetFromHttpContext();
+            }
         }
 
         /// <inheritdoc />

src/Umbraco.Core/Scoping/Scope.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Data;
 using Umbraco.Core.Cache;
+using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.IO;
 using Umbraco.Core.Persistence;
@@ -24,6 +25,7 @@ namespace Umbraco.Core.Scoping
 
         private IsolatedRuntimeCache _isolatedRuntimeCache;
         private UmbracoDatabase _database;
+        private EventMessages _messages;
         private ICompletable _fscope;
         private IEventDispatcher _eventDispatcher;
 
@@ -127,6 +129,7 @@ namespace Umbraco.Core.Scoping
         {
             // steal everything from NoScope
             _database = noScope.DatabaseOrNull;
+            _messages = noScope.MessagesOrNull;
 
             // make sure the NoScope can be replaced ie not in a transaction
             if (_database != null && _database.InTransaction)
@@ -271,11 +274,27 @@ namespace Umbraco.Core.Scoping
             {
                 EnsureNotDisposed();
                 if (ParentScope != null) return ParentScope.Messages;
-                //return _messages ?? (_messages = new EventMessages());
 
-                // ok, this isn't pretty, but it works
-                // TODO kill the message factory and let the scope manage it all
-                return ApplicationContext.Current.Services.EventMessagesFactory.Get();
+                if (_messages != null) return _messages;
+
+                // this is ugly - in v7 for backward compatibility reasons, EventMessages need
+                // to survive way longer that the scopes, and kinda resides on its own in http
+                // context, but must also be in scopes for when we do async and lose http context
+                // TODO refactor in v8
+
+                var factory = ScopeLifespanMessagesFactory.Current;
+                if (factory == null)
+                {
+                    _messages = new EventMessages();
+                }
+                else
+                {
+                    _messages = factory.GetFromHttpContext();
+                    if (_messages == null)
+                        factory.Set(_messages = new EventMessages());
+                }
+
+                return _messages;
             }
         }
 
@@ -284,7 +303,14 @@ namespace Umbraco.Core.Scoping
             get
             {
                 EnsureNotDisposed();
-                return ParentScope == null ? null : ParentScope.MessagesOrNull;
+                if (ParentScope != null) return ParentScope.MessagesOrNull;
+
+                // see comments in Messages
+
+                if (_messages != null) return _messages;
+
+                var factory = ScopeLifespanMessagesFactory.Current;
+                return factory == null ? null : factory.GetFromHttpContext();
             }
         }
 
@@ -316,7 +342,11 @@ namespace Umbraco.Core.Scoping
         {
             // if child did not complete we cannot complete
             if (completed.HasValue == false || completed.Value == false)
+            {
+                if (LogUncompletedScopes)
+                    Logging.LogHelper.Debug<Scope>("Uncompleted Child Scope at\r\n" + Environment.StackTrace);
                 _completed = false;
+            }
         }
 
         private void EnsureNotDisposed()
@@ -343,13 +373,13 @@ namespace Umbraco.Core.Scoping
 #endif
             }
 
+            var parent = ParentScope;
+            _scopeProvider.AmbientScope = parent;
+
 #if DEBUG_SCOPES
             _scopeProvider.Disposed(this);
 #endif
 
-            var parent = ParentScope;
-            _scopeProvider.AmbientScope = parent;
-
             if (parent != null)
                 parent.ChildCompleted(_completed);
             else
@@ -468,5 +498,15 @@ namespace Umbraco.Core.Scoping
                 TryFinally(index + 1, actions);
             }
         }
+
+        // backing field for LogUncompletedScopes
+        private static bool? _logUncompletedScopes;
+
+        // caching config
+        // true if Umbraco.CoreDebug.LogUncompletedScope appSetting is set to "true"
+        private static bool LogUncompletedScopes
+        {
+            get { return (_logUncompletedScopes ?? (_logUncompletedScopes = UmbracoConfig.For.CoreDebug().LogUncompletedScopes)).Value; }
+        }
     }
 }

src/Umbraco.Core/Scoping/ScopeProvider.cs

@@ -3,6 +3,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Data;
 using System.Runtime.Remoting.Messaging;
+using System.Text;
 using System.Web;
 using Umbraco.Core.Events;
 using Umbraco.Core.Persistence;
@@ -99,18 +100,23 @@ namespace Umbraco.Core.Scoping
             where T : class
         {
             var objectKey = CallContext.LogicalGetData(key).AsGuid();
+            if (objectKey == Guid.Empty) return null;
+
             lock (StaticCallContextObjectsLock)
             {
                 object callContextObject;
                 if (StaticCallContextObjects.TryGetValue(objectKey, out callContextObject))
                 {
 #if DEBUG_SCOPES
-                    //Logging.LogHelper.Debug<ScopeProvider>("GotObject " + objectKey.ToString("N").Substring(0, 8));
+                    Logging.LogHelper.Debug<ScopeProvider>("Got " + typeof(T).Name + " Object " + objectKey.ToString("N").Substring(0, 8));
+                    //Logging.LogHelper.Debug<ScopeProvider>("At:\r\n" + Head(Environment.StackTrace, 24));
 #endif
                     return (T) callContextObject;
                 }
+
+                Logging.LogHelper.Warn<ScopeProvider>("Missed " + typeof(T).Name + " Object " + objectKey.ToString("N").Substring(0, 8));
 #if DEBUG_SCOPES
-                //Logging.LogHelper.Debug<ScopeProvider>("MissedObject " + objectKey.ToString("N").Substring(0, 8));
+                //Logging.LogHelper.Debug<ScopeProvider>("At:\r\n" + Head(Environment.StackTrace, 24));
 #endif
                 return null;
             }
@@ -146,7 +152,8 @@ namespace Umbraco.Core.Scoping
                 lock (StaticCallContextObjectsLock)
                 {
 #if DEBUG_SCOPES
-                    //Logging.LogHelper.Debug<ScopeProvider>("RemoveObject " + objectKey.ToString("N").Substring(0, 8));
+                    Logging.LogHelper.Debug<ScopeProvider>("Remove Object " + objectKey.ToString("N").Substring(0, 8));
+                    //Logging.LogHelper.Debug<ScopeProvider>("At:\r\n" + Head(Environment.StackTrace, 24));
 #endif
                     StaticCallContextObjects.Remove(objectKey);
                 }
@@ -160,7 +167,8 @@ namespace Umbraco.Core.Scoping
                 lock (StaticCallContextObjectsLock)
                 {
 #if DEBUG_SCOPES
-                    //Logging.LogHelper.Debug<ScopeProvider>("AddObject " + objectKey.ToString("N").Substring(0, 8));
+                    Logging.LogHelper.Debug<ScopeProvider>("AddObject " + objectKey.ToString("N").Substring(0, 8));
+                    //Logging.LogHelper.Debug<ScopeProvider>("At:\r\n" + Head(Environment.StackTrace, 24));
 #endif
                     StaticCallContextObjects.Add(objectKey, value);
                 }
@@ -506,7 +514,7 @@ namespace Umbraco.Core.Scoping
             lock (StaticScopeInfosLock)
             {
                 if (StaticScopeInfos.ContainsKey(scope)) throw new Exception("oops: already registered.");
-                //Logging.LogHelper.Debug<ScopeProvider>("Register " + scope.InstanceId.ToString("N").Substring(0, 8));
+                Logging.LogHelper.Debug<ScopeProvider>("Register " + scope.InstanceId.ToString("N").Substring(0, 8));
                 StaticScopeInfos[scope] = new ScopeInfo(scope, Environment.StackTrace);
             }
         }
@@ -524,14 +532,35 @@ namespace Umbraco.Core.Scoping
                     if (context == null) return;
                     throw new Exception("oops: unregistered scope.");
                 }
-                //Logging.LogHelper.Debug<ScopeProvider>("Register context " + (context ?? "null") + " for " + scope.InstanceId.ToString("N").Substring(0, 8));
+                var sb = new StringBuilder();
+                var s = scope;
+                while (s != null)
+                {
+                    if (sb.Length > 0) sb.Append(" < ");
+                    sb.Append(s.InstanceId.ToString("N").Substring(0, 8));
+                    var ss = s as IScopeInternal;
+                    s = ss == null ? null : ss.ParentScope;
+                }
+                Logging.LogHelper.Debug<ScopeProvider>("Register " + (context ?? "null") + " context " + sb);
                 if (context == null) info.NullStack = Environment.StackTrace;
-                //if (context == null)
-                //    Logging.LogHelper.Debug<ScopeProvider>("STACK\r\n" + info.NullStack);
+                //Logging.LogHelper.Debug<ScopeProvider>("At:\r\n" + Head(Environment.StackTrace, 16));
                 info.Context = context;
             }
         }
 
+        private static string Head(string s, int count)
+        {
+            var pos = 0;
+            var i = 0;
+            while (i < count && pos >= 0)
+            {
+                pos = s.IndexOf("\r\n", pos + 1, StringComparison.OrdinalIgnoreCase);
+                i++;
+            }
+            if (pos < 0) return s;
+            return s.Substring(0, pos);
+        }
+
         public void Disposed(IScope scope)
         {
             lock (StaticScopeInfosLock)
@@ -541,7 +570,7 @@ namespace Umbraco.Core.Scoping
                     // enable this by default
                     //Console.WriteLine("unregister " + scope.InstanceId.ToString("N").Substring(0, 8));
                     StaticScopeInfos.Remove(scope);
-                    //Logging.LogHelper.Debug<ScopeProvider>("Remove " + scope.InstanceId.ToString("N").Substring(0, 8));
+                    Logging.LogHelper.Debug<ScopeProvider>("Remove " + scope.InstanceId.ToString("N").Substring(0, 8));
 
                     // instead, enable this to keep *all* scopes
                     // beware, there can be a lot of scopes!

src/Umbraco.Core/Services/ContentService.cs

@@ -578,11 +578,9 @@ namespace Umbraco.Core.Services
                 var repository = RepositoryFactory.CreateContentRepository(uow);
 
                 var query = Query<IContent>.Builder;
-                //if the id is System Root, then just get all
-                if (id != Constants.System.Root)
-                {
-                    query.Where(x => x.ParentId == id);
-                }
+                // always check for a parent - else it will also get decendants (and then you should use the GetPagedDescendants method)
+                query.Where(x => x.ParentId == id);
+
                 IQuery<IContent> filterQuery = null;
                 if (filter.IsNullOrWhiteSpace() == false)
                 {
@@ -2245,6 +2243,7 @@ namespace Umbraco.Core.Services
                     //We need to check if children and their publish state to ensure that we 'republish' content that was previously published
                     if (published && previouslyPublished == false && HasChildren(content.Id))
                     {
+                    //TODO: Horrible for performance if there are lots of descendents! We should page if anything but this is crazy
                         var descendants = GetPublishedDescendants(content);
 
                         _publishingStrategy.PublishingFinalized(uow, descendants, false);

src/Umbraco.Core/Services/ContentTypeService.cs

@@ -69,7 +69,7 @@ namespace Umbraco.Core.Services
                     repo.AddOrUpdate(container);
                     uow.Commit();
 
-                    uow.Events.Dispatch(SavedContentTypeContainer, this, new SaveEventArgs<EntityContainer>(container, evtMsgs));
+                    uow.Events.Dispatch(SavedContentTypeContainer, this, new SaveEventArgs<EntityContainer>(container, evtMsgs), "SavedContentTypeContainer");
                     //TODO: Audit trail ?
 
                     return Attempt.Succeed(new OperationStatus<EntityContainer, OperationStatusType>(container, OperationStatusType.Success, evtMsgs));
@@ -106,7 +106,7 @@ namespace Umbraco.Core.Services
                     repo.AddOrUpdate(container);
                     uow.Commit();
 
-                    uow.Events.Dispatch(SavedMediaTypeContainer, this, new SaveEventArgs<EntityContainer>(container, evtMsgs));
+                    uow.Events.Dispatch(SavedMediaTypeContainer, this, new SaveEventArgs<EntityContainer>(container, evtMsgs), "SavedMediaTypeContainer");
                     //TODO: Audit trail ?
 
                     return Attempt.Succeed(new OperationStatus<EntityContainer, OperationStatusType>(container, OperationStatusType.Success, evtMsgs));
@@ -122,14 +122,16 @@ namespace Umbraco.Core.Services
         {
             return SaveContainer(
                 SavingContentTypeContainer, SavedContentTypeContainer,
-                container, Constants.ObjectTypes.DocumentTypeContainerGuid, "document type", userId);
+                container, Constants.ObjectTypes.DocumentTypeContainerGuid, "document type",
+                "SavedContentTypeContainer", userId);
         }
 
         public Attempt<OperationStatus> SaveMediaTypeContainer(EntityContainer container, int userId = 0)
         {
             return SaveContainer(
                 SavingMediaTypeContainer, SavedMediaTypeContainer,
-                container, Constants.ObjectTypes.MediaTypeContainerGuid, "media type", userId);
+                container, Constants.ObjectTypes.MediaTypeContainerGuid, "media type",
+                "SavedMediaTypeContainer", userId);
         }
 
         private Attempt<OperationStatus> SaveContainer(
@@ -137,7 +139,9 @@ namespace Umbraco.Core.Services
             TypedEventHandler<IContentTypeService, SaveEventArgs<EntityContainer>> savedEvent,
             EntityContainer container,
             Guid containerObjectType,
-            string objectTypeName, int userId)
+            string objectTypeName, 
+            string savedEventName,
+            int userId)
         {
             var evtMsgs = EventMessagesFactory.Get();
 
@@ -165,7 +169,7 @@ namespace Umbraco.Core.Services
                 var repo = RepositoryFactory.CreateEntityContainerRepository(uow, containerObjectType);
                 repo.AddOrUpdate(container);
                 uow.Commit();
-                uow.Events.Dispatch(savedEvent, this, new SaveEventArgs<EntityContainer>(container, evtMsgs));
+                uow.Events.Dispatch(savedEvent, this, new SaveEventArgs<EntityContainer>(container, evtMsgs), savedEventName);
             }
 
             //TODO: Audit trail ?

src/Umbraco.Core/Services/EntityService.cs

@@ -9,6 +9,7 @@ using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
 
@@ -91,6 +92,7 @@ namespace Umbraco.Core.Services
                         case UmbracoObjectTypes.Member:
                         case UmbracoObjectTypes.DataType:
                         case UmbracoObjectTypes.DocumentTypeContainer:
+                            uow.Commit();
                             return uow.Database.ExecuteScalar<int?>(new Sql().Select("id").From<NodeDto>().Where<NodeDto>(dto => dto.UniqueId == key));
                         case UmbracoObjectTypes.RecycleBin:
                         case UmbracoObjectTypes.Stylesheet:
@@ -129,6 +131,7 @@ namespace Umbraco.Core.Services
                         case UmbracoObjectTypes.DocumentType:
                         case UmbracoObjectTypes.Member:
                         case UmbracoObjectTypes.DataType:
+                            uow.Commit();
                             return uow.Database.ExecuteScalar<Guid?>(new Sql().Select("uniqueID").From<NodeDto>().Where<NodeDto>(dto => dto.NodeId == id));
                         case UmbracoObjectTypes.RecycleBin:
                         case UmbracoObjectTypes.Stylesheet:
@@ -374,6 +377,115 @@ namespace Umbraco.Core.Services
             }
         }
 
+        /// <summary>
+        /// Returns a paged collection of children
+        /// </summary>
+        /// <param name="parentId">The parent id to return children for</param>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>
+        /// <returns></returns>
+        public IEnumerable<IUmbracoEntity> GetPagedChildren(int parentId, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = "")
+        {
+            var objectTypeId = umbracoObjectType.GetGuid();
+            using (var uow = UowProvider.GetUnitOfWork())
+            {
+                var repository = RepositoryFactory.CreateEntityRepository(uow);
+                var query = Query<IUmbracoEntity>.Builder.Where(x => x.ParentId == parentId && x.Trashed == false);
+
+                IQuery<IUmbracoEntity> filterQuery = null;
+                if (filter.IsNullOrWhiteSpace() == false)
+                {
+                    filterQuery = Query<IUmbracoEntity>.Builder.Where(x => x.Name.Contains(filter));
+                }
+
+                var contents = repository.GetPagedResultsByQuery(query, objectTypeId, pageIndex, pageSize, out totalRecords, orderBy, orderDirection, filterQuery);
+                uow.Commit();
+                return contents;
+            }
+        }
+
+        /// <summary>
+        /// Returns a paged collection of descendants
+        /// </summary>
+        /// <param name="id"></param>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>        
+        /// <returns></returns>
+        public IEnumerable<IUmbracoEntity> GetPagedDescendants(int id, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
+        {
+            var objectTypeId = umbracoObjectType.GetGuid();
+            using (var uow = UowProvider.GetUnitOfWork())
+            {
+                var repository = RepositoryFactory.CreateEntityRepository(uow);
+
+                var query = Query<IUmbracoEntity>.Builder;
+                //if the id is System Root, then just get all
+                if (id != Constants.System.Root)
+                    query.Where(x => x.Path.SqlContains(string.Format(",{0},", id), TextColumnType.NVarchar));                
+
+                IQuery<IUmbracoEntity> filterQuery = null;
+                if (filter.IsNullOrWhiteSpace() == false)
+                {
+                    filterQuery = Query<IUmbracoEntity>.Builder.Where(x => x.Name.Contains(filter));
+                }
+
+                var contents = repository.GetPagedResultsByQuery(query, objectTypeId, pageIndex, pageSize, out totalRecords, orderBy, orderDirection, filterQuery);
+                uow.Commit();
+                return contents;
+            }
+        }
+
+        /// <summary>
+        /// Returns a paged collection of descendants from the root
+        /// </summary>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>
+        /// <param name="includeTrashed">true/false to include trashed objects</param>
+        /// <returns></returns>
+        public IEnumerable<IUmbracoEntity> GetPagedDescendantsFromRoot(UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "", bool includeTrashed = true)
+        {
+            var objectTypeId = umbracoObjectType.GetGuid();
+            using (var uow = UowProvider.GetUnitOfWork())
+            {
+                var repository = RepositoryFactory.CreateEntityRepository(uow);
+
+                var query = Query<IUmbracoEntity>.Builder;
+                //don't include trashed if specfied
+                if (includeTrashed == false)
+                {
+                    query.Where(x => x.Trashed == false);
+                }
+
+                IQuery<IUmbracoEntity> filterQuery = null;
+                if (filter.IsNullOrWhiteSpace() == false)
+                {
+                    filterQuery = Query<IUmbracoEntity>.Builder.Where(x => x.Name.Contains(filter));
+                }
+
+                var contents = repository.GetPagedResultsByQuery(query, objectTypeId, pageIndex, pageSize, out totalRecords, orderBy, orderDirection, filterQuery);
+                uow.Commit();
+                return contents;
+            }
+        }
+
         /// <summary>
         /// Gets a collection of descendents by the parents Id
         /// </summary>

src/Umbraco.Core/Services/IEntityService.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 
 namespace Umbraco.Core.Services
 {
@@ -137,6 +138,51 @@ namespace Umbraco.Core.Services
         /// <returns>An enumerable list of <see cref="IUmbracoEntity"/> objects</returns>
         IEnumerable<IUmbracoEntity> GetChildren(int parentId, UmbracoObjectTypes umbracoObjectType);
 
+        /// <summary>
+        /// Returns a paged collection of children
+        /// </summary>
+        /// <param name="parentId">The parent id to return children for</param>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>
+        /// <returns></returns>
+        IEnumerable<IUmbracoEntity> GetPagedChildren(int parentId, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = "");
+
+        /// <summary>
+        /// Returns a paged collection of descendants
+        /// </summary>
+        /// <param name="id"></param>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>        
+        /// <returns></returns>
+        IEnumerable<IUmbracoEntity> GetPagedDescendants(int id, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
+
+        /// <summary>
+        /// Returns a paged collection of descendants from the root
+        /// </summary>
+        /// <param name="umbracoObjectType"></param>
+        /// <param name="pageIndex"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="totalRecords"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="filter"></param>
+        /// <param name="includeTrashed">true/false to include trashed objects</param>
+        /// <returns></returns>
+        IEnumerable<IUmbracoEntity> GetPagedDescendantsFromRoot(UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "", bool includeTrashed = true);
+
         /// <summary>
         /// Gets a collection of descendents by the parents Id
         /// </summary>

src/Umbraco.Core/Services/IMediaService.cs

@@ -81,6 +81,7 @@ namespace Umbraco.Core.Services
         /// </param>
         void RebuildXmlStructures(params int[] contentTypeIds);
 
+        int CountNotTrashed(string contentTypeAlias = null);
         int Count(string contentTypeAlias = null);
         int CountChildren(int parentId, string contentTypeAlias = null);
         int CountDescendants(int parentId, string contentTypeAlias = null);
@@ -167,6 +168,22 @@ namespace Umbraco.Core.Services
         IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter);
 
+        /// <summary>
+        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
+        /// </summary>
+        /// <param name="id">Id of the Parent to retrieve Children from</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
+        /// <param name="filter">Search text filter</param>
+        /// <param name="contentTypeFilter">A list of content type Ids to filter the list by</param>
+        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
+        IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter);
+
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,

src/Umbraco.Core/Services/MediaService.cs

@@ -261,6 +261,29 @@ namespace Umbraco.Core.Services
             }
         }
 
+        public int CountNotTrashed(string contentTypeAlias = null)
+        {
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMediaRepository(uow))
+            using (var mediaTypeRepository = RepositoryFactory.CreateMediaTypeRepository(uow))
+            {
+                var mediaTypeId = 0;
+                if (contentTypeAlias.IsNullOrWhiteSpace() == false)
+                {
+                    var mediaType = mediaTypeRepository.Get(contentTypeAlias);
+                    if (mediaType == null) return 0;
+                    mediaTypeId = mediaType.Id;
+                }
+
+                var query = Query<IMedia>.Builder.Where(media => media.Trashed == false);
+                if (mediaTypeId > 0)
+                {
+                    query.Where(media => media.ContentTypeId == mediaTypeId);
+                }
+                return repository.Count(query);
+            }
+        }
+
         public int Count(string contentTypeAlias = null)
         {
             using (var uow = UowProvider.GetUnitOfWork(commit: true))
@@ -448,6 +471,25 @@ namespace Umbraco.Core.Services
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         public IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter)
+        {
+            return GetPagedChildren(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter, null);
+        }
+
+        /// <summary>
+        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
+        /// </summary>
+        /// <param name="id">Id of the Parent to retrieve Children from</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalChildren">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
+        /// <param name="filter">Search text filter</param>
+        /// <param name="contentTypeFilter">A list of content type Ids to filter the list by</param>
+        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
+        public IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren,
+            string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter)
         {
             Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
             Mandate.ParameterCondition(pageSize > 0, "pageSize");
@@ -457,11 +499,14 @@ namespace Umbraco.Core.Services
                 var repository = RepositoryFactory.CreateMediaRepository(uow);
 
                 var query = Query<IMedia>.Builder;
-                //if the id is System Root, then just get all
-                if (id != Constants.System.Root)
+                // always check for a parent - else it will also get decendants (and then you should use the GetPagedDescendants method)
+                query.Where(x => x.ParentId == id);
+
+                if (contentTypeFilter != null && contentTypeFilter.Length > 0)
                 {
-                    query.Where(x => x.ParentId == id);
+                    query.Where(x => contentTypeFilter.Contains(x.ContentTypeId));
                 }
+
                 IQuery<IMedia> filterQuery = null;
                 if (filter.IsNullOrWhiteSpace() == false)
                 {

src/Umbraco.Core/StringUdi.cs

@@ -1,10 +1,12 @@
 using System;
+using System.ComponentModel;
 
 namespace Umbraco.Core
 {
     /// <summary>
     /// Represents a string-based entity identifier.
     /// </summary>
+    [TypeConverter(typeof(UdiTypeConverter))]
     public class StringUdi : Udi
     {
         /// <summary>

src/Umbraco.Core/Umbraco.Core.csproj

@@ -10,7 +10,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Umbraco.Core</RootNamespace>
     <AssemblyName>Umbraco.Core</AssemblyName>
-    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
@@ -39,15 +39,19 @@
   <ItemGroup>
     <Reference Include="AutoMapper, Version=3.3.1.0, Culture=neutral, PublicKeyToken=be96cd2c38ef1005, processorArchitecture=MSIL">
       <HintPath>..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="AutoMapper.Net4, Version=3.3.1.0, Culture=neutral, PublicKeyToken=be96cd2c38ef1005, processorArchitecture=MSIL">
       <HintPath>..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll</HintPath>
+      <Private>True</Private>
     </Reference>
-    <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
-      <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>
+    <Reference Include="HtmlAgilityPack, Version=1.4.9.5, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
+      <HintPath>..\packages\HtmlAgilityPack.1.4.9.5\lib\Net45\HtmlAgilityPack.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ImageProcessor, Version=2.5.2.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\ImageProcessor.2.5.2\lib\net45\ImageProcessor.dll</HintPath>
@@ -55,49 +59,64 @@
     </Reference>
     <Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
       <HintPath>..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.AspNet.Identity.Core, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.AspNet.Identity.Core.2.2.1\lib\net45\Microsoft.AspNet.Identity.Core.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.AspNet.Identity.Owin, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.AspNet.Identity.Owin.2.2.1\lib\net45\Microsoft.AspNet.Identity.Owin.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Owin, Version=3.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Owin.Security, Version=3.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Owin.Security.3.0.1\lib\net45\Microsoft.Owin.Security.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Owin.Security.Cookies, Version=3.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Owin.Security.Cookies.3.0.1\lib\net45\Microsoft.Owin.Security.Cookies.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Owin.Security.OAuth, Version=3.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Owin.Security.OAuth.3.0.1\lib\net45\Microsoft.Owin.Security.OAuth.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="MiniProfiler, Version=2.1.0.0, Culture=neutral, PublicKeyToken=b44f9351044011a3, processorArchitecture=MSIL">
       <HintPath>..\packages\MiniProfiler.2.1.0\lib\net40\MiniProfiler.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="MySql.Data, Version=6.9.8.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
       <HintPath>..\packages\MySql.Data.6.9.8\lib\net45\MySql.Data.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Owin, Version=1.0.0.0, Culture=neutral, PublicKeyToken=f0ebd12fd5e55cc5, processorArchitecture=MSIL">
       <HintPath>..\packages\Owin.1.0\lib\net40\Owin.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Semver, Version=2.0.4.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Semver.2.0.4\lib\net452\Semver.dll</HintPath>
+      <HintPath>..\packages\Semver.2.0.4\lib\netstandard1.1\Semver.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.ComponentModel.Composition" />
     <Reference Include="System.ComponentModel.DataAnnotations" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Data.Entity" />
     <Reference Include="System.Data.SqlServerCe, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.Data.SqlServerCe.Entity, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.Entity.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.DirectoryServices.AccountManagement" />
     <Reference Include="System.Drawing" />
@@ -179,6 +198,7 @@
     <Compile Include="Configuration\BaseRest\ExtensionElement.cs" />
     <Compile Include="Configuration\BaseRest\ExtensionElementCollection.cs" />
     <Compile Include="Configuration\BaseRest\MethodElement.cs" />
+    <Compile Include="Configuration\ContentXmlStorage.cs" />
     <Compile Include="Configuration\Dashboard\AccessElement.cs" />
     <Compile Include="Configuration\Dashboard\AccessItem.cs" />
     <Compile Include="Configuration\Dashboard\AccessType.cs" />
@@ -199,6 +219,7 @@
     <Compile Include="Configuration\Dashboard\SectionElement.cs" />
     <Compile Include="Configuration\Dashboard\TabCollection.cs" />
     <Compile Include="Configuration\Dashboard\TabElement.cs" />
+    <Compile Include="Configuration\CoreDebug.cs" />
     <Compile Include="Configuration\FileSystemProviderElement.cs" />
     <Compile Include="Configuration\FileSystemProviderElementCollection.cs" />
     <Compile Include="Configuration\FileSystemProvidersSection.cs" />
@@ -311,10 +332,14 @@
     <Compile Include="Deploy\ArtifactSignature.cs" />
     <Compile Include="Deploy\Difference.cs" />
     <Compile Include="Deploy\Direction.cs" />
-    <Compile Include="Deploy\IFileStore.cs" />
+    <Compile Include="Deploy\IFileType.cs" />
+    <Compile Include="Deploy\IFileTypeCollection.cs" />
     <Compile Include="Events\EventDefinitionFilter.cs" />
     <Compile Include="Events\IDeletingMediaFilesEventArgs.cs" />
     <Compile Include="Events\ScopeEventDispatcherBase.cs" />
+    <Compile Include="Events\ScopeLifespanMessagesFactory.cs" />
+    <Compile Include="IHttpContextAccessor.cs" />
+    <Compile Include="Models\PublishedContent\PublishedContentTypeConverter.cs" />
     <Compile Include="OrderedHashSet.cs" />
     <Compile Include="Events\UninstallPackageEventArgs.cs" />
     <Compile Include="Models\GridValue.cs" />
@@ -326,7 +351,6 @@
     <Compile Include="Deploy\IImageSourceParser.cs" />
     <Compile Include="Deploy\ILocalLinkParser.cs" />
     <Compile Include="Deploy\IMacroParser.cs" />
-    <Compile Include="Deploy\IPrettyArtifact.cs" />
     <Compile Include="Deploy\IPreValueConnector.cs" />
     <Compile Include="Deploy\IServiceConnector.cs" />
     <Compile Include="Deploy\IValueConnector.cs" />
@@ -424,9 +448,11 @@
     <Compile Include="Models\PartialViewType.cs" />
     <Compile Include="Models\PublicAccessEntry.cs" />
     <Compile Include="Models\PublicAccessRule.cs" />
+    <Compile Include="Models\PublishedContent\PublishedContentEnumerable.cs" />
     <Compile Include="Models\PublishedContent\PublishedContentWithKeyExtended.cs" />
     <Compile Include="Models\PublishedContent\PublishedContentWithKeyModel.cs" />
     <Compile Include="Models\PublishedContent\PublishedContentWithKeyWrapped.cs" />
+    <Compile Include="Models\Range.cs" />
     <Compile Include="Models\Rdbms\AccessDto.cs" />
     <Compile Include="Media\Exif\BitConverterEx.cs" />
     <Compile Include="Media\Exif\ExifBitConverter.cs" />
@@ -557,10 +583,21 @@
     <Compile Include="Persistence\UnitOfWork\ScopeUnitOfWorkProvider.cs" />
     <Compile Include="PropertyEditors\DecimalValidator.cs" />
     <Compile Include="PropertyEditors\PropertyEditorValueTypes.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\CheckboxListValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\DropdownListMultipleValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\DropdownListMultipleWithKeysValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\DropdownListValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\DropdownListWithKeysValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\EmailAddressValueConverter.cs" />
     <Compile Include="PropertyEditors\ValueConverters\GridValueConverter.cs" />
     <Compile Include="PropertyEditors\ValueConverters\DecimalValueConverter.cs" />
     <Compile Include="PropertyEditors\ValueConverters\LabelValueConverter.cs" />
     <Compile Include="PropertyEditors\ValueConverters\ImageCropperValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\MemberGroupPickerValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\RadioButtonListValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\SliderValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\TagsValueConverter.cs" />
+    <Compile Include="PropertyEditors\ValueConverters\UploadPropertyConverter.cs" />
     <Compile Include="Publishing\UnPublishedStatusType.cs" />
     <Compile Include="Publishing\UnPublishStatus.cs" />
     <Compile Include="SafeCallContext.cs" />

src/Umbraco.Core/XmlExtensions.cs

@@ -16,32 +16,6 @@ namespace Umbraco.Core
 	/// </summary>
 	internal static class XmlExtensions
 	{
-        /// <summary>
-        /// Saves the xml document async
-        /// </summary>
-        /// <param name="xdoc"></param>
-        /// <param name="filename"></param>
-        /// <returns></returns>
-	    public static async Task SaveAsync(this XmlDocument xdoc, string filename)
-	    {
-            if (xdoc.DocumentElement == null)
-                throw new XmlException("Cannot save xml document, there is no root element");
-
-            using (var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true))
-            using (var xmlWriter = XmlWriter.Create(fs, new XmlWriterSettings
-            {
-                Async = true,
-                Encoding = Encoding.UTF8,
-                Indent = true
-            }))
-            {
-                //NOTE: There are no nice methods to write it async, only flushing it async. We
-                // could implement this ourselves but it'd be a very manual process.
-                xdoc.WriteTo(xmlWriter);
-                await xmlWriter.FlushAsync().ConfigureAwait(false);
-            }
-	    }
-
         public static bool HasAttribute(this XmlAttributeCollection attributes, string attributeName)
         {
             return attributes.Cast<XmlAttribute>().Any(x => x.Name == attributeName);

src/Umbraco.Core/app.config

@@ -32,4 +32,9 @@
       </dependentAssembly>
     </assemblyBinding>
   </runtime>
-<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" /></startup></configuration>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup><system.data>
+    <DbProviderFactories>
+      <remove invariant="MySql.Data.MySqlClient" />
+      <add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.8.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
+    </DbProviderFactories>
+  </system.data></configuration>

src/Umbraco.Core/packages.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
-  <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net45" />
+  <package id="HtmlAgilityPack" version="1.4.9.5" targetFramework="net45" />
   <package id="ImageProcessor" version="2.5.2" targetFramework="net45" />
   <package id="log4net" version="2.0.5" targetFramework="net45" />
   <package id="Microsoft.AspNet.Identity.Core" version="2.2.1" targetFramework="net45" />
@@ -12,9 +12,13 @@
   <package id="Microsoft.Owin.Security.OAuth" version="3.0.1" targetFramework="net45" />
   <package id="MiniProfiler" version="2.1.0" targetFramework="net45" />
   <package id="MySql.Data" version="6.9.8" targetFramework="net45" />
-  <package id="Newtonsoft.Json" version="9.0.1" targetFramework="net462" />
+  <package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
   <package id="Owin" version="1.0" targetFramework="net45" />
-  <package id="Semver" version="2.0.4" targetFramework="net462" />
+  <package id="Semver" version="2.0.4" targetFramework="net45" />
   <package id="SharpZipLib" version="0.86.0" targetFramework="net45" />
   <package id="SqlServerCE" version="4.0.0.1" targetFramework="net45" />
+  <package id="System.Runtime" version="4.1.0" targetFramework="net45" />
+  <package id="System.Runtime.Extensions" version="4.1.0" targetFramework="net45" />
+  <package id="System.Runtime.InteropServices" version="4.1.0" targetFramework="net45" />
+  <package id="System.Text.RegularExpressions" version="4.1.0" targetFramework="net45" />
 </packages>

src/Umbraco.Tests.Benchmarks/App.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" />
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
     </startup>
   <runtime>
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">

src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs

@@ -16,6 +16,7 @@ using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
+using Moq;
 using Umbraco.Core;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
@@ -43,15 +44,18 @@ namespace Umbraco.Tests.Benchmarks
 
         public ModelToSqlExpressionHelperBenchmarks()
         {
-            _contentMapper = new ContentMapper(_syntaxProvider);
-            _contentMapper.BuildMap();
+            var contentMapper = new ContentMapper(_syntaxProvider);
+            contentMapper.BuildMap();
             _cachedExpression = new CachedExpression();
+            var mappingResolver = new Mock<MappingResolver>();
+            mappingResolver.Setup(resolver => resolver.ResolveMapperByType(It.IsAny<Type>())).Returns(contentMapper);
+            _mappingResolver = mappingResolver.Object;
         }
         
         private readonly ISqlSyntaxProvider _syntaxProvider = new SqlCeSyntaxProvider();
-        private readonly BaseMapper _contentMapper;
         private readonly CachedExpression _cachedExpression;
-        
+        private readonly MappingResolver _mappingResolver;
+
         [Benchmark(Baseline = true)]
         public void WithNonCached()
         {
@@ -62,7 +66,7 @@ namespace Umbraco.Tests.Benchmarks
                 Expression<Func<IContent, bool>> predicate = content =>
                 content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b);
 
-                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _contentMapper);
+                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _mappingResolver);
                 var result = modelToSqlExpressionHelper.Visit(predicate);
             }
             
@@ -80,7 +84,7 @@ namespace Umbraco.Tests.Benchmarks
                 Expression<Func<IContent, bool>> predicate = content =>
                 content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b);
                 
-                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _contentMapper);
+                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _mappingResolver);
 
                 //wrap it!
                 _cachedExpression.Wrap(predicate);

src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj

@@ -9,7 +9,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Umbraco.Tests.Benchmarks</RootNamespace>
     <AssemblyName>Umbraco.Tests.Benchmarks</AssemblyName>
-    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <NuGetPackageImportStamp>
     </NuGetPackageImportStamp>
@@ -40,101 +40,61 @@
   <ItemGroup>
     <Reference Include="BenchmarkDotNet, Version=0.9.9.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
       <HintPath>..\packages\BenchmarkDotNet.0.9.9\lib\net45\BenchmarkDotNet.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="BenchmarkDotNet.Core, Version=0.9.9.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
       <HintPath>..\packages\BenchmarkDotNet.Core.0.9.9\lib\net45\BenchmarkDotNet.Core.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="BenchmarkDotNet.Diagnostics.Windows, Version=0.9.9.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
       <HintPath>..\packages\BenchmarkDotNet.Diagnostics.Windows.0.9.9\lib\net45\BenchmarkDotNet.Diagnostics.Windows.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="BenchmarkDotNet.Toolchains.Roslyn, Version=0.9.9.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
       <HintPath>..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="Castle.Core, Version=4.0.0.0, Culture=neutral, PublicKeyToken=407dd0808d44fbdc, processorArchitecture=MSIL">
+      <HintPath>..\packages\Castle.Core.4.0.0\lib\net45\Castle.Core.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.CodeAnalysis, Version=1.3.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.CodeAnalysis.Common.1.3.2\lib\net45\Microsoft.CodeAnalysis.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.CodeAnalysis.CSharp, Version=1.3.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.CodeAnalysis.CSharp.1.3.2\lib\net45\Microsoft.CodeAnalysis.CSharp.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Diagnostics.Tracing.TraceEvent, Version=1.0.41.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.41\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll</HintPath>
-    </Reference>
-    <Reference Include="System" />
-    <Reference Include="System.AppContext, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.AppContext.4.1.0\lib\net46\System.AppContext.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="Moq, Version=4.7.0.0, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
+      <HintPath>..\packages\Moq.4.7.0\lib\net45\Moq.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
     <Reference Include="System.Collections.Immutable, Version=1.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Collections.Immutable.1.2.0\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System.ComponentModel.Composition" />
-    <Reference Include="System.Console, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Console.4.0.0\lib\net46\System.Console.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
     <Reference Include="System.Core" />
     <Reference Include="System.Data.SqlServerCe, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.Data.SqlServerCe.Entity, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
       <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.Entity.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Diagnostics.FileVersionInfo, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Diagnostics.FileVersionInfo.4.0.0\lib\net46\System.Diagnostics.FileVersionInfo.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Diagnostics.StackTrace, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Diagnostics.StackTrace.4.0.1\lib\net46\System.Diagnostics.StackTrace.dll</HintPath>
-    </Reference>
-    <Reference Include="System.IO.FileSystem, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.IO.FileSystem.4.0.1\lib\net46\System.IO.FileSystem.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.IO.FileSystem.Primitives, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.IO.FileSystem.Primitives.4.0.1\lib\net46\System.IO.FileSystem.Primitives.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System.Management" />
     <Reference Include="System.Numerics" />
-    <Reference Include="System.Reflection, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Reflection.4.1.0\lib\net462\System.Reflection.dll</HintPath>
-    </Reference>
     <Reference Include="System.Reflection.Metadata, Version=1.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Reflection.Metadata.1.3.0\lib\portable-net45+win8\System.Reflection.Metadata.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Runtime, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Runtime.4.1.0\lib\net462\System.Runtime.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Runtime.Extensions, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Runtime.Extensions.4.1.0\lib\net462\System.Runtime.Extensions.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Runtime.InteropServices, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Runtime.InteropServices.4.1.0\lib\net462\System.Runtime.InteropServices.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Security.Cryptography.Algorithms, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Security.Cryptography.Algorithms.4.2.0\lib\net461\System.Security.Cryptography.Algorithms.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Security.Cryptography.Encoding, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Security.Cryptography.Encoding.4.0.0\lib\net46\System.Security.Cryptography.Encoding.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Security.Cryptography.Primitives, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Security.Cryptography.Primitives.4.0.0\lib\net46\System.Security.Cryptography.Primitives.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Security.Cryptography.X509Certificates, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Security.Cryptography.X509Certificates.4.1.0\lib\net461\System.Security.Cryptography.X509Certificates.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Text.Encoding.CodePages, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Text.Encoding.CodePages.4.0.1\lib\net46\System.Text.Encoding.CodePages.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System.Threading.Tasks.Extensions, Version=4.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Threading.Tasks.Extensions.4.0.0\lib\portable-net45+win8+wp8+wpa81\System.Threading.Tasks.Extensions.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Threading.Thread, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Threading.Thread.4.0.0\lib\net46\System.Threading.Thread.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System.Xml.Linq" />
@@ -143,18 +103,6 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
-    <Reference Include="System.Xml.XmlDocument, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Xml.XmlDocument.4.0.1\lib\net46\System.Xml.XmlDocument.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Xml.XPath, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Xml.XPath.4.0.1\lib\net46\System.Xml.XPath.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Xml.XPath.XDocument, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\System.Xml.XPath.XDocument.4.0.1\lib\net46\System.Xml.XPath.XDocument.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="LinqCastBenchmarks.cs" />
@@ -168,6 +116,7 @@
   <ItemGroup>
     <None Include="App.config" />
     <None Include="packages.config" />
+    <None Include="_TraceEventProgrammersGuide.docx" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Umbraco.Core\Umbraco.Core.csproj">
@@ -187,6 +136,10 @@
     <Analyzer Include="..\packages\Microsoft.CodeAnalysis.Analyzers.1.1.0\analyzers\dotnet\cs\Microsoft.CodeAnalysis.Analyzers.dll" />
     <Analyzer Include="..\packages\Microsoft.CodeAnalysis.Analyzers.1.1.0\analyzers\dotnet\cs\Microsoft.CodeAnalysis.CSharp.Analyzers.dll" />
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="TraceEvent.ReadMe.txt" />
+    <Content Include="TraceEvent.ReleaseNotes.txt" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <PostBuildEvent>

src/Umbraco.Tests.Benchmarks/packages.config

@@ -4,51 +4,37 @@
   <package id="BenchmarkDotNet.Core" version="0.9.9" targetFramework="net45" />
   <package id="BenchmarkDotNet.Diagnostics.Windows" version="0.9.9" targetFramework="net45" />
   <package id="BenchmarkDotNet.Toolchains.Roslyn" version="0.9.9" targetFramework="net45" />
+  <package id="Castle.Core" version="4.0.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Analyzers" version="1.1.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Common" version="1.3.2" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.CSharp" version="1.3.2" targetFramework="net45" />
   <package id="Microsoft.Diagnostics.Tracing.TraceEvent" version="1.0.41" targetFramework="net45" />
   <package id="Microsoft.SqlServer.Compact" version="4.0.8854.1" targetFramework="net45" />
+  <package id="Moq" version="4.7.0" targetFramework="net45" />
   <package id="SqlServerCE" version="4.0.0.1" targetFramework="net45" />
-  <package id="System.AppContext" version="4.1.0" targetFramework="net462" />
-  <package id="System.Collections" version="4.0.11" targetFramework="net462" />
-  <package id="System.Collections.Concurrent" version="4.0.12" targetFramework="net462" />
-  <package id="System.Collections.Immutable" version="1.2.0" targetFramework="net462" />
-  <package id="System.Console" version="4.0.0" targetFramework="net462" />
-  <package id="System.Diagnostics.Debug" version="4.0.11" targetFramework="net462" />
-  <package id="System.Diagnostics.FileVersionInfo" version="4.0.0" targetFramework="net462" />
-  <package id="System.Diagnostics.StackTrace" version="4.0.1" targetFramework="net462" />
-  <package id="System.Diagnostics.Tools" version="4.0.1" targetFramework="net462" />
-  <package id="System.Dynamic.Runtime" version="4.0.11" targetFramework="net462" />
-  <package id="System.Globalization" version="4.0.11" targetFramework="net462" />
-  <package id="System.IO.FileSystem" version="4.0.1" targetFramework="net462" />
-  <package id="System.IO.FileSystem.Primitives" version="4.0.1" targetFramework="net462" />
-  <package id="System.Linq" version="4.1.0" targetFramework="net462" />
-  <package id="System.Linq.Expressions" version="4.1.0" targetFramework="net462" />
-  <package id="System.Reflection" version="4.1.0" targetFramework="net462" />
-  <package id="System.Reflection.Metadata" version="1.3.0" targetFramework="net462" />
-  <package id="System.Reflection.Primitives" version="4.0.1" targetFramework="net462" />
-  <package id="System.Resources.ResourceManager" version="4.0.1" targetFramework="net462" />
-  <package id="System.Runtime" version="4.1.0" targetFramework="net462" />
-  <package id="System.Runtime.Extensions" version="4.1.0" targetFramework="net462" />
-  <package id="System.Runtime.Handles" version="4.0.1" targetFramework="net462" />
-  <package id="System.Runtime.InteropServices" version="4.1.0" targetFramework="net462" />
-  <package id="System.Runtime.Numerics" version="4.0.1" targetFramework="net462" />
-  <package id="System.Security.Cryptography.Algorithms" version="4.2.0" targetFramework="net462" />
-  <package id="System.Security.Cryptography.Encoding" version="4.0.0" targetFramework="net462" />
-  <package id="System.Security.Cryptography.Primitives" version="4.0.0" targetFramework="net462" />
-  <package id="System.Security.Cryptography.X509Certificates" version="4.1.0" targetFramework="net462" />
-  <package id="System.Text.Encoding" version="4.0.11" targetFramework="net462" />
-  <package id="System.Text.Encoding.CodePages" version="4.0.1" targetFramework="net462" />
-  <package id="System.Text.Encoding.Extensions" version="4.0.11" targetFramework="net462" />
-  <package id="System.Threading" version="4.0.11" targetFramework="net462" />
-  <package id="System.Threading.Tasks" version="4.0.11" targetFramework="net462" />
+  <package id="System.Collections" version="4.0.11" targetFramework="net45" />
+  <package id="System.Collections.Concurrent" version="4.0.12" targetFramework="net45" />
+  <package id="System.Collections.Immutable" version="1.2.0" targetFramework="net45" />
+  <package id="System.Diagnostics.Debug" version="4.0.11" targetFramework="net45" />
+  <package id="System.Diagnostics.Tools" version="4.0.1" targetFramework="net45" />
+  <package id="System.Dynamic.Runtime" version="4.0.11" targetFramework="net45" />
+  <package id="System.Globalization" version="4.0.11" targetFramework="net45" />
+  <package id="System.Linq" version="4.1.0" targetFramework="net45" />
+  <package id="System.Linq.Expressions" version="4.1.0" targetFramework="net45" />
+  <package id="System.Reflection" version="4.1.0" targetFramework="net45" />
+  <package id="System.Reflection.Metadata" version="1.3.0" targetFramework="net45" />
+  <package id="System.Reflection.Primitives" version="4.0.1" targetFramework="net45" />
+  <package id="System.Resources.ResourceManager" version="4.0.1" targetFramework="net45" />
+  <package id="System.Runtime" version="4.1.0" targetFramework="net45" />
+  <package id="System.Runtime.Extensions" version="4.1.0" targetFramework="net45" />
+  <package id="System.Runtime.InteropServices" version="4.1.0" targetFramework="net45" />
+  <package id="System.Runtime.Numerics" version="4.0.1" targetFramework="net45" />
+  <package id="System.Text.Encoding" version="4.0.11" targetFramework="net45" />
+  <package id="System.Text.Encoding.Extensions" version="4.0.11" targetFramework="net45" />
+  <package id="System.Threading" version="4.0.11" targetFramework="net45" />
+  <package id="System.Threading.Tasks" version="4.0.11" targetFramework="net45" />
   <package id="System.Threading.Tasks.Extensions" version="4.0.0" targetFramework="net45" />
-  <package id="System.Threading.Tasks.Parallel" version="4.0.1" targetFramework="net462" />
-  <package id="System.Threading.Thread" version="4.0.0" targetFramework="net462" />
-  <package id="System.Xml.ReaderWriter" version="4.0.11" targetFramework="net462" />
-  <package id="System.Xml.XDocument" version="4.0.11" targetFramework="net462" />
-  <package id="System.Xml.XmlDocument" version="4.0.1" targetFramework="net462" />
-  <package id="System.Xml.XPath" version="4.0.1" targetFramework="net462" />
-  <package id="System.Xml.XPath.XDocument" version="4.0.1" targetFramework="net462" />
+  <package id="System.Threading.Tasks.Parallel" version="4.0.1" targetFramework="net45" />
+  <package id="System.Xml.ReaderWriter" version="4.0.11" targetFramework="net45" />
+  <package id="System.Xml.XDocument" version="4.0.11" targetFramework="net45" />
 </packages>

src/Umbraco.Tests/App.config

@@ -91,7 +91,7 @@
     </siteMap>
   </system.web>
 
-<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" /></startup>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup>
   <runtime>
 
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">