Bump version to 7.5.14

Merge pull request #1880 from marcemarc/temp-U4-9765
2017-05-16 08:33:19 +02:00 · 2017-05-16 08:32:33 +02:00 · 2017-05-15 10:28:26 +02:00 · 2017-05-15 10:19:04 +02:00 · 2017-05-10 21:18:58 +02:00 · 2017-05-10 22:37:47 +08:00
306 changed files with 9421 additions and 3620 deletions
@@ -142,3 +142,5 @@ build/ui-docs.zip
 build/csharp-docs.zip
 build/msbuild.log
 .vs/
+
+build/tools/
@@ -1,6 +1,8 @@
+[![Build status](https://ci.appveyor.com/api/projects/status/ikpgqxiw9v8opltv/branch/dev-v7?svg=true)](https://ci.appveyor.com/project/Umbraco/umbraco-cms-hs8dx/branch/dev-v7)
+
 Umbraco CMS
 ===========
-The friendliest, most flexible and fastest growing ASP.NET CMS used by more than 350,000 websites worldwide: [https://umbraco.com](https://umbraco.com)
+The friendliest, most flexible and fastest growing ASP.NET CMS used by more than 390,000 websites worldwide: [https://umbraco.com](https://umbraco.com)

 [![ScreenShot](vimeo.png)](https://vimeo.com/172382998/)

@@ -49,10 +49,52 @@ REM This is necessary because SETLOCAL is on in InstallGit.cmd so that one might
 REM but the path setting is lost due to SETLOCAL 
 path=C:\Program Files (x86)\Git\cmd;C:\Program Files\Git\cmd;%PATH%

+SET toolsFolder=%CD%\tools\
+IF NOT EXIST "%toolsFolder%" (
+	MD tools
+)
+
+SET nuGetExecutable=%CD%\tools\nuget.exe
+IF NOT EXIST "%nuGetExecutable%" (
+	ECHO Getting NuGet so we can fetch some tools
+	ECHO Downloading https://dist.nuget.org/win-x86-commandline/latest/nuget.exe to %nuGetExecutable%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', '%nuGetExecutable%')"
+)
+
+:: We need 7za.exe for BuildBelle.bat
+IF NOT EXIST "%toolsFolder%7za.exe" (
+	ECHO 7zip not found - fetching now
+	"%nuGetExecutable%" install 7-Zip.CommandLine -OutputDirectory tools -Verbosity quiet
+)
+
+:: We need vswhere.exe for VS2017+
+IF NOT EXIST "%toolsFolder%vswhere.exe" (
+	ECHO vswhere not found - fetching now
+	"%nuGetExecutable%" install vswhere -OutputDirectory tools -Verbosity quiet
+)
+
+:: Put 7za.exe and vswhere.exe in a predictable path (not version specific)
+FOR /f "delims=" %%A in ('dir "%toolsFolder%7-Zip.CommandLine.*" /b') DO SET "sevenZipExePath=%toolsFolder%%%A\"
+MOVE "%sevenZipExePath%tools\7za.exe" "%toolsFolder%7za.exe"
+
+FOR /f "delims=" %%A in ('dir "%toolsFolder%vswhere.*" /b') DO SET "vswhereExePath=%toolsFolder%%%A\"
+MOVE "%vswhereExePath%tools\vswhere.exe" "%toolsFolder%vswhere.exe"
+	
 ECHO.
 ECHO Making sure we have a web.config
 IF NOT EXIST %CD%\..\src\Umbraco.Web.UI\web.config COPY %CD%\..\src\Umbraco.Web.UI\web.Template.config %CD%\..\src\Umbraco.Web.UI\web.config

+for /f "usebackq tokens=1* delims=: " %%i in (`"%CD%\tools\vswhere.exe" -latest -requires Microsoft.Component.MSBuild`) do (
+  if /i "%%i"=="installationPath" set InstallDir=%%j
+)
+
+SET VSWherePath="%InstallDir%\MSBuild"
+ECHO.
+ECHO Visual Studio is installed in: %InstallDir%
+
+SET MSBUILDPATH=C:\Program Files (x86)\MSBuild\14.0\Bin
+SET MSBUILD="%MSBUILDPATH%\MsBuild.exe"
+
 ECHO.
 ECHO.
 ECHO Performing MSBuild and producing Umbraco binaries zip files
@@ -60,7 +102,7 @@ ECHO This takes a few minutes and logging is set to report warnings
 ECHO and errors only so it might seems like nothing is happening for a while. 
 ECHO You can check the msbuild.log file for progress.
 ECHO.
-%windir%\Microsoft.NET\Framework\v4.0.30319\msbuild.exe "Build.proj" /p:BUILD_RELEASE=%release% /p:BUILD_COMMENT=%comment% /p:NugetPackagesDirectory=%nuGetFolder% /consoleloggerparameters:Summary;ErrorsOnly;WarningsOnly /fileLogger
+%MSBUILD% "Build.proj" /p:BUILD_RELEASE=%release% /p:BUILD_COMMENT=%comment% /p:NugetPackagesDirectory=%nuGetFolder% /p:VSWherePath=%VSWherePath%
 IF ERRORLEVEL 1 GOTO :error

 ECHO.
@@ -1,33 +1,59 @@
@ECHO OFF
 SETLOCAL
+	:: SETLOCAL is on, so changes to the path not persist to the actual user's path

-SET release=%1
-ECHO Installing Npm NuGet Package
-
-SET nuGetFolder=%CD%\..\src\packages\
-ECHO Configured packages folder: %nuGetFolder%
+SET toolsFolder=%CD%\tools\
 ECHO Current folder: %CD%

-%CD%\..\src\.nuget\NuGet.exe install Npm.js -OutputDirectory %nuGetFolder%  -Verbosity quiet
+SET nodeFileName=node-v6.9.1-win-x86.7z
+SET nodeExtractFolder=%toolsFolder%node.js.691

-for /f "delims=" %%A in ('dir %nuGetFolder%node.js.* /b') do set "nodePath=%nuGetFolder%%%A\"
-for /f "delims=" %%A in ('dir %nuGetFolder%npm.js.* /b') do set "npmPath=%nuGetFolder%%%A\tools\"
+IF NOT EXIST "%nodeExtractFolder%" (
+	ECHO Downloading http://nodejs.org/dist/v6.9.1/%nodeFileName% to %toolsFolder%%nodeFileName%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('http://nodejs.org/dist/v6.9.1/%nodeFileName%', '%toolsFolder%%nodeFileName%')"
+	ECHO Extracting %nodeFileName% to %nodeExtractFolder%
+	"%toolsFolder%\7za.exe" x "%toolsFolder%\%nodeFileName%" -o"%nodeExtractFolder%" -aos > nul
+)
+FOR /f "delims=" %%A in ('dir "%nodeExtractFolder%\node*" /b') DO SET "nodePath=%nodeExtractFolder%\%%A"

-ECHO Adding Npm and Node to path 
-REM SETLOCAL is on, so changes to the path not persist to the actual user's path
-PATH=%npmPath%;%nodePath%;%PATH%

-SET buildFolder=%CD%
+SET nuGetExecutable=%CD%\tools\nuget.exe
+IF NOT EXIST "%nuGetExecutable%" (
+	ECHO Downloading https://dist.nuget.org/win-x86-commandline/latest/nuget.exe to %nuGetExecutable%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', '%nuGetExecutable%')"
+)

-ECHO Change directory to %CD%\..\src\Umbraco.Web.UI.Client\
-CD %CD%\..\src\Umbraco.Web.UI.Client\
+SET drive=%CD:~0,2%
+SET nuGetFolder=%drive%\packages\
+FOR /f "delims=" %%A in ('dir "%nuGetFolder%npm.*" /b') DO SET "npmPath=%nuGetFolder%%%A\"

-ECHO Do npm install and the grunt build of Belle
-call npm cache clean --quiet
-call npm install --quiet
-call npm install -g grunt-cli --quiet
-call npm install -g bower --quiet
-call grunt build --buildversion=%release%
+IF [%npmPath%] == [] GOTO :installnpm 
+IF NOT [%npmPath%] == [] GOTO :build

-ECHO Move back to the build folder
-CD %buildFolder% 
+:installnpm
+	ECHO Downloading npm
+	ECHO Configured packages folder: %nuGetFolder%	
+	ECHO Installing Npm NuGet Package
+	"%nuGetExecutable%" install Npm -OutputDirectory %nuGetFolder% -Verbosity detailed
+	REM Ensures that we look for the just downloaded NPM, not whatever the user has installed on their machine
+	FOR /f "delims=" %%A in ('dir %nuGetFolder%npm.* /b') DO SET "npmPath=%nuGetFolder%%%A\"
+	GOTO :build
+
+:build
+    ECHO Adding Npm and Node to path 
+    REM SETLOCAL is on, so changes to the path not persist to the actual user's path
+    PATH="%npmPath%";"%nodePath%";%PATH%
+    SET buildFolder=%CD%
+
+    ECHO Change directory to %CD%\..\src\Umbraco.Web.UI.Client\
+    CD %CD%\..\src\Umbraco.Web.UI.Client\
+
+    ECHO Do npm install and the grunt build of Belle
+    call npm cache clean --quiet
+    call npm install --quiet
+    call npm install -g grunt-cli --quiet
+    call npm install -g bower --quiet
+    call grunt build --buildversion=%release%
+
+    ECHO Move back to the build folder
+    CD "%buildFolder%"
@@ -28,17 +28,17 @@
      <dependency id="MySql.Data" version="[6.9.8, 7.0.0)" />
      <dependency id="xmlrpcnet" version="[2.5.0, 3.0.0)" />
      <dependency id="ClientDependency" version="[1.9.2, 2.0.0)" />
-      <dependency id="ClientDependency-Mvc5" version="[1.8.0, 2.0.0)" />
+      <dependency id="ClientDependency-Mvc5" version="[1.8.0.0, 2.0.0)" />
 	  <!-- AutoMapper can not be updated due to: https://github.com/AutoMapper/AutoMapper/issues/373#issuecomment-127644405 -->
      <dependency id="AutoMapper" version="[3.3.1, 4.0.0)" />
      <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Examine" version="[0.1.70, 1.0.0)" />
-      <dependency id="ImageProcessor" version="[2.4.5, 3.0.0)" />
-      <dependency id="ImageProcessor.Web" version="[4.6.6, 5.0.0)" />
+      <dependency id="Examine" version="[0.1.82, 1.0.0)" />
+      <dependency id="ImageProcessor" version="[2.5.2, 3.0.0)" />
+      <dependency id="ImageProcessor.Web" version="[4.8.2, 5.0.0)" />
      <dependency id="semver" version="[1.1.2, 2.0.0)" />
      <dependency id="UrlRewritingNet" version="[2.0.7, 3.0.0)" />	  
 	  <!-- Markdown can not be updated due to: https://github.com/hey-red/markdownsharp/issues/71#issuecomment-233585487 -->
-      <dependency id="Markdown" version="[1.14.4, 2.0.0)" />	  
+      <dependency id="Markdown" version="[1.14.7, 2.0.0)" />	  
    </dependencies>
  </metadata>
  <files>
@@ -103,4 +103,4 @@
    <file src="..\_BuildOutput\bin\UmbracoExamine.pdb" target="lib" />
    <file src="..\..\src\UmbracoExamine\**\*.cs" exclude="..\..\src\**\TemporaryGeneratedFile*.cs" target="src\UmbracoExamine" />
  </files>
-</package>
+</package>
@@ -17,7 +17,8 @@
    <dependencies>
      <dependency id="UmbracoCms.Core" version="[$version$]" />
      <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Umbraco.ModelsBuilder" version="[3.0.5, 4.0.0)" />
+      <dependency id="Umbraco.ModelsBuilder" version="[3.0.7, 4.0.0)" />
+      <dependency id="ImageProcessor.Web.Config" version="[2.3.0, 3.0.0)" />
    </dependencies>
  </metadata>
  <files>
@@ -43,6 +44,8 @@
    <file src="tools\trees.config.install.xdt" target="Content\config\trees.config.install.xdt" />
    <file src="tools\umbracoSettings.config.install.xdt" target="Content\config\umbracoSettings.config.install.xdt" />
    <file src="tools\Views.Web.config.install.xdt" target="Views\Web.config.install.xdt" />
+    <file src="tools\processing.config.install.xdt" target="Content\Config\imageprocessor\processing.config.install.xdt" />
+    <file src="tools\cache.config.install.xdt" target="Content\Config\imageprocessor\cache.config.install.xdt" />
    <file src="build\**" target="build" />
  </files>
 </package>
@@ -41,16 +41,6 @@
        views/dashboard/developer/examinemanagement.html
      </control>
    </tab>
-    <tab caption="Health Check" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/healthcheck.html
-      </control>
-    </tab>
-    <tab caption="Redirect URL Management" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/redirecturls.html
-      </control>
-    </tab>	
  </section>
  
  <section alias="StartupMediaDashboardSection" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
@@ -80,4 +70,26 @@
 	<tab caption="Change Password" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
 	<tab caption="Last Edits" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
  </section>
+
+  <section alias="RedirectUrlManagement" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>content</area>
+    </areas>
+    <tab caption="Redirect URL Management">
+      <control>
+        views/dashboard/developer/redirecturls.html
+      </control>
+    </tab>
+  </section>
+
+  <section alias="UmbracoHealthCheck" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>developer</area>
+    </areas>
+    <tab caption="Health Check">
+      <control>
+        views/dashboard/developer/healthcheck.html
+      </control>
+    </tab>
+  </section>
 </dashBoard>
@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<caching xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform">
+  <caches>
+    <cache name="DiskCache" trimCache="false" xdt:Transform="SetAttributes(trimCache)" xdt:Locator="Match(name)"  />
+  </caches>
+</caching>
@@ -71,6 +71,7 @@ if ($project) {
 		if(Test-Path $umbracoBinFolder\ImageProcessor.dll) { Remove-Item $umbracoBinFolder\ImageProcessor.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\ImageProcessor.Web.dll) { Remove-Item $umbracoBinFolder\ImageProcessor.Web.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\Lucene.Net.dll) { Remove-Item $umbracoBinFolder\Lucene.Net.dll -Force -Confirm:$false }
+		if(Test-Path $umbracoBinFolder\Microsoft.IO.RecyclableMemoryStream.dll) { Remove-Item $umbracoBinFolder\Microsoft.IO.RecyclableMemoryStream.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\Microsoft.AspNet.Identity.Core.dll) { Remove-Item $umbracoBinFolder\Microsoft.AspNet.Identity.Core.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\Microsoft.AspNet.Identity.Owin.dll) { Remove-Item $umbracoBinFolder\Microsoft.AspNet.Identity.Owin.dll -Force -Confirm:$false }
 		if(Test-Path $umbracoBinFolder\Microsoft.CodeAnalysis.CSharp.dll) { Remove-Item $umbracoBinFolder\Microsoft.CodeAnalysis.CSharp.dll -Force -Confirm:$false }
@@ -0,0 +1 @@
+<processing xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform" xdt:Transform="SetAttributes(preserveExifMetaData)" preserveExifMetaData="true" />
@@ -1,2 +1,2 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
-7.5.6
+7.5.14
@@ -49,13 +49,9 @@
    <Reference Include="System.Core" />
    <Reference Include="System.Data.SqlServerCe, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
      <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.dll</HintPath>
-      <SpecificVersion>False</SpecificVersion>
-      <Private>False</Private>
    </Reference>
    <Reference Include="System.Data.SqlServerCe.Entity, Version=4.0.0.1, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
      <HintPath>..\packages\SqlServerCE.4.0.0.1\lib\System.Data.SqlServerCe.Entity.dll</HintPath>
-      <SpecificVersion>False</SpecificVersion>
-      <Private>False</Private>
    </Reference>
    <Reference Include="System.Xml.Linq" />
    <Reference Include="System.Data.DataSetExtensions" />
@@ -2,7 +2,7 @@
 using System.Resources;

 [assembly: AssemblyCompany("Umbraco")]
-[assembly: AssemblyCopyright("Copyright © Umbraco 2016")]
+[assembly: AssemblyCopyright("Copyright © Umbraco 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]

@@ -11,5 +11,5 @@ using System.Resources;

 [assembly: AssemblyVersion("1.0.*")]

-[assembly: AssemblyFileVersion("7.5.6")]
-[assembly: AssemblyInformationalVersion("7.5.6")]
+[assembly: AssemblyFileVersion("7.5.14")]
+[assembly: AssemblyInformationalVersion("7.5.14")]
@@ -64,7 +64,8 @@ namespace Umbraco.Core.Cache
        [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
        public const string ContentTypePropertiesCacheKey = "ContentType_PropertyTypes_Content:";
        
-        [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
+        [Obsolete("No longer used and will be removed in v8")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
        public const string PropertyTypeCacheKey = "UmbracoPropertyTypeCache";

        [Obsolete("This is no longer used and will be removed from the codebase in the future")]
@@ -26,6 +26,9 @@ namespace Umbraco.Core.Cache

        public DeepCloneRuntimeCacheProvider(IRuntimeCacheProvider innerProvider)
        {
+            if (innerProvider.GetType() == typeof(DeepCloneRuntimeCacheProvider))
+                throw new InvalidOperationException("A " + typeof(DeepCloneRuntimeCacheProvider) + " cannot wrap another instance of " + typeof(DeepCloneRuntimeCacheProvider));
+
            InnerProvider = innerProvider;
        }

@@ -105,9 +108,11 @@ namespace Umbraco.Core.Cache
                var value = result.Value; // force evaluation now - this may throw if cacheItem throws, and then nothing goes into cache
                if (value == null) return null; // do not store null values (backward compat)

+                //Clone/reset to go into the cache
                return CheckCloneableAndTracksChanges(value);
            }, timeout, isSliding, priority, removedCallback, dependentFiles);

+            //Clone/reset to go out of the cache
            return CheckCloneableAndTracksChanges(cached);
        }

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Configuration
+{
+    internal enum ContentXmlStorage
+    {
+        Default,
+        AspNetTemp,
+        EnvironmentTemp
+    }
+}
@@ -0,0 +1,27 @@
+using System;
+
+namespace Umbraco.Core.Configuration
+{
+    internal static class CoreDebugExtensions
+    {
+        private static CoreDebug _coreDebug;
+
+        public static CoreDebug CoreDebug(this UmbracoConfig config)
+        {
+            return _coreDebug ?? (_coreDebug = new CoreDebug());
+        }
+    }
+
+    internal class CoreDebug
+    {
+        public CoreDebug()
+        {
+            var appSettings = System.Configuration.ConfigurationManager.AppSettings;
+            DumpOnTimeoutThreadAbort = string.Equals("true", appSettings["Umbraco.CoreDebug.DumpOnTimeoutThreadAbort"], StringComparison.OrdinalIgnoreCase);
+        }
+
+        // when true, the Logger creates a minidump of w3wp in ~/App_Data/MiniDump whenever it logs
+        // an error due to a ThreadAbortException that is due to a timeout.
+        public bool DumpOnTimeoutThreadAbort { get; private set; }
+    }
+}
@@ -520,12 +520,25 @@ namespace Umbraco.Core.Configuration
        }

        internal static bool ContentCacheXmlStoredInCodeGen
+        {
+            get { return ContentCacheXmlStorageLocation == ContentXmlStorage.AspNetTemp; }
+        }
+
+        internal static ContentXmlStorage ContentCacheXmlStorageLocation
        {
            get
            {
-                //defaults to false
-                return ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp") 
-                    && bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]); //default to false
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLStorage"))
+                {
+                    return Enum<ContentXmlStorage>.Parse(ConfigurationManager.AppSettings["umbracoContentXMLStorage"]);
+                }
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp"))
+                {
+                    return bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]) 
+                        ? ContentXmlStorage.AspNetTemp 
+                        : ContentXmlStorage.Default;
+                }
+                return ContentXmlStorage.Default;
            }
        }

@@ -4,8 +4,7 @@ using System.Configuration;

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-
-    internal class ContentElement : ConfigurationElement, IContentSection
+    internal class ContentElement : UmbracoConfigurationElement, IContentSection
    {
        [ConfigurationProperty("imaging")]
        internal ContentImagingElement Imaging
@@ -22,25 +21,13 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        [ConfigurationProperty("ResolveUrlsFromTextString")]
        internal InnerTextConfigurationElement<bool> ResolveUrlsFromTextString
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                       (InnerTextConfigurationElement<bool>)this["ResolveUrlsFromTextString"],
-                        //set the default
-                       false);
-            }
+            get { return GetOptionalTextElement<bool>("ResolveUrlsFromTextString", false); }
        }

        [ConfigurationProperty("UploadAllowDirectories")]
        internal InnerTextConfigurationElement<bool> UploadAllowDirectories
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                       (InnerTextConfigurationElement<bool>)this["UploadAllowDirectories"],
-                    //set the default
-                       true);
-            }
+            get { return GetOptionalTextElement("UploadAllowDirectories", true); }
        }

        public IEnumerable<IContentErrorPage> Error404Collection
@@ -63,121 +50,61 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        [ConfigurationProperty("ensureUniqueNaming")]
        internal InnerTextConfigurationElement<bool> EnsureUniqueNaming
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                      (InnerTextConfigurationElement<bool>)this["ensureUniqueNaming"],
-                    //set the default
-                      true);
-            }
+            get { return GetOptionalTextElement("ensureUniqueNaming", true); }
        }

        [ConfigurationProperty("TidyEditorContent")]
        internal InnerTextConfigurationElement<bool> TidyEditorContent
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                      (InnerTextConfigurationElement<bool>)this["TidyEditorContent"],
-                    //set the default
-                      false);
-            }
+            get { return GetOptionalTextElement("TidyEditorContent", false); }
        }

        [ConfigurationProperty("TidyCharEncoding")]
        internal InnerTextConfigurationElement<string> TidyCharEncoding
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                          (InnerTextConfigurationElement<string>)this["TidyCharEncoding"],
-                            //set the default
-                          "UTF8");
-            }
+            get { return GetOptionalTextElement("TidyCharEncoding", "UTF8"); }
        }

        [ConfigurationProperty("XmlCacheEnabled")]
        internal InnerTextConfigurationElement<bool> XmlCacheEnabled
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["XmlCacheEnabled"],
-                    //set the default
-                          true);
-            }
+            get { return GetOptionalTextElement("XmlCacheEnabled", true); }
        }

        [ConfigurationProperty("ContinouslyUpdateXmlDiskCache")]
        internal InnerTextConfigurationElement<bool> ContinouslyUpdateXmlDiskCache
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["ContinouslyUpdateXmlDiskCache"],
-                    //set the default
-                          true);                
-            }
+            get { return GetOptionalTextElement("ContinouslyUpdateXmlDiskCache", true); }
        }

        [ConfigurationProperty("XmlContentCheckForDiskChanges")]
        internal InnerTextConfigurationElement<bool> XmlContentCheckForDiskChanges
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["XmlContentCheckForDiskChanges"],
-                    //set the default
-                          false); 
-            }
+            get { return GetOptionalTextElement("XmlContentCheckForDiskChanges", false); }
        }

        [ConfigurationProperty("EnableSplashWhileLoading")]
        internal InnerTextConfigurationElement<bool> EnableSplashWhileLoading
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["EnableSplashWhileLoading"],
-                    //set the default
-                          false); 
-            }
+            get { return GetOptionalTextElement("EnableSplashWhileLoading", false); }
        }

        [ConfigurationProperty("PropertyContextHelpOption")]
        internal InnerTextConfigurationElement<string> PropertyContextHelpOption
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                          (InnerTextConfigurationElement<string>)this["PropertyContextHelpOption"],
-                    //set the default
-                          "text"); 
-            }
+            get { return GetOptionalTextElement("PropertyContextHelpOption", "text"); }
        }

        [ConfigurationProperty("UseLegacyXmlSchema")]
        internal InnerTextConfigurationElement<bool> UseLegacyXmlSchema
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["UseLegacyXmlSchema"],
-                    //set the default
-                          false); 
-            }
+            get { return GetOptionalTextElement("UseLegacyXmlSchema", false); }
        }
        
        [ConfigurationProperty("ForceSafeAliases")]
        internal InnerTextConfigurationElement<bool> ForceSafeAliases
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                          (InnerTextConfigurationElement<bool>)this["ForceSafeAliases"],
-                    //set the default
-                          true); 
-            }
+            get { return GetOptionalTextElement("ForceSafeAliases", true); }
        }

        [ConfigurationProperty("PreviewBadge")]
@@ -185,123 +112,63 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        {
            get
            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                          (InnerTextConfigurationElement<string>)this["PreviewBadge"],
-                    //set the default
-                          @"<a id=""umbracoPreviewBadge"" style=""position: absolute; top: 0; right: 0; border: 0; width: 149px; height: 149px; background: url('{1}/preview/previewModeBadge.png') no-repeat;"" href=""{0}/endPreview.aspx?redir={2}""><span style=""display:none;"">In Preview Mode - click to end</span></a>"); 
+                return GetOptionalTextElement("PreviewBadge", @"<a id=""umbracoPreviewBadge"" style=""position: absolute; top: 0; right: 0; border: 0; width: 149px; height: 149px; background: url('{1}/preview/previewModeBadge.png') no-repeat;"" href=""{0}/endPreview.aspx?redir={2}""><span style=""display:none;"">In Preview Mode - click to end</span></a>");                
            }
        }

        [ConfigurationProperty("UmbracoLibraryCacheDuration")]
        internal InnerTextConfigurationElement<int> UmbracoLibraryCacheDuration
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<int>(
-                          (InnerTextConfigurationElement<int>)this["UmbracoLibraryCacheDuration"],
-                    //set the default
-                          1800); 
-                
-            }
+            get { return GetOptionalTextElement("UmbracoLibraryCacheDuration", 1800); }
        }

        [ConfigurationProperty("MacroErrors")]
        internal InnerTextConfigurationElement<MacroErrorBehaviour> MacroErrors
        {
-            get
-            {
-
-                return new OptionalInnerTextConfigurationElement<MacroErrorBehaviour>(
-                          (InnerTextConfigurationElement<MacroErrorBehaviour>)this["MacroErrors"],
-                    //set the default
-                          MacroErrorBehaviour.Inline); 
-            }
+            get { return GetOptionalTextElement("MacroErrors", MacroErrorBehaviour.Inline); }
        }

        [Obsolete("This is here so that if this config element exists we won't get a YSOD, it is not used whatsoever and will be removed in future versions")]
        [ConfigurationProperty("DocumentTypeIconList")]
        internal InnerTextConfigurationElement<IconPickerBehaviour> DocumentTypeIconList
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<IconPickerBehaviour>(
-                          (InnerTextConfigurationElement<IconPickerBehaviour>)this["DocumentTypeIconList"],
-                    //set the default
-                          IconPickerBehaviour.HideFileDuplicates);
-            }
+            get { return GetOptionalTextElement("DocumentTypeIconList", IconPickerBehaviour.HideFileDuplicates); }
        }

        [ConfigurationProperty("disallowedUploadFiles")]
        internal CommaDelimitedConfigurationElement DisallowedUploadFiles
        {
-            get
-            {
-                return new OptionalCommaDelimitedConfigurationElement(
-                       (CommaDelimitedConfigurationElement)this["disallowedUploadFiles"],
-                    //set the default
-                       new[] { "ashx", "aspx", "ascx", "config", "cshtml", "vbhtml", "asmx", "air", "axd" });
-
-            }
+            get { return GetOptionalDelimitedElement("disallowedUploadFiles", new[] {"ashx", "aspx", "ascx", "config", "cshtml", "vbhtml", "asmx", "air", "axd"}); }
        }

        [ConfigurationProperty("cloneXmlContent")]
        internal InnerTextConfigurationElement<bool> CloneXmlContent
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["cloneXmlContent"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("cloneXmlContent", true); }
        }

        [ConfigurationProperty("GlobalPreviewStorageEnabled")]
        internal InnerTextConfigurationElement<bool> GlobalPreviewStorageEnabled
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["GlobalPreviewStorageEnabled"],
-                    //set the default
-                    false);
-            }
+            get { return GetOptionalTextElement("GlobalPreviewStorageEnabled", false); }
        }

        [ConfigurationProperty("defaultDocumentTypeProperty")]
        internal InnerTextConfigurationElement<string> DefaultDocumentTypeProperty
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                    (InnerTextConfigurationElement<string>)this["defaultDocumentTypeProperty"],
-                    //set the default
-                    "Textstring");
-            }
+            get { return GetOptionalTextElement("defaultDocumentTypeProperty", "Textstring"); }
        }

        [ConfigurationProperty("EnableInheritedDocumentTypes")]
        internal InnerTextConfigurationElement<bool> EnableInheritedDocumentTypes
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>) this["EnableInheritedDocumentTypes"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("EnableInheritedDocumentTypes", true); }
        }

        [ConfigurationProperty("EnableInheritedMediaTypes")]
        internal InnerTextConfigurationElement<bool> EnableInheritedMediaTypes
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["EnableInheritedMediaTypes"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("EnableInheritedMediaTypes", true); }
        }

        string IContentSection.NotificationEmailAddress
@@ -3,42 +3,24 @@ using System.Configuration;

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class ContentScriptEditorElement : ConfigurationElement
+    internal class ContentScriptEditorElement : UmbracoConfigurationElement
    {
        [ConfigurationProperty("scriptFolderPath")]
        internal InnerTextConfigurationElement<string> ScriptFolderPath
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                       (InnerTextConfigurationElement<string>)this["scriptFolderPath"],
-                    //set the default
-                       "/scripts");
-            }
+            get { return GetOptionalTextElement("scriptFolderPath", "/scripts"); }
        }

        [ConfigurationProperty("scriptFileTypes")]
        internal OptionalCommaDelimitedConfigurationElement ScriptFileTypes
        {
-            get
-            {
-                return new OptionalCommaDelimitedConfigurationElement(
-                       (OptionalCommaDelimitedConfigurationElement)this["scriptFileTypes"],
-                    //set the default
-                       new[] { "js", "xml" });                
-            }
+            get { return GetOptionalDelimitedElement("scriptFileTypes", new[] {"js", "xml"}); }
        }

        [ConfigurationProperty("scriptDisableEditor")]
        internal InnerTextConfigurationElement<bool> ScriptEditorDisable
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>) this["scriptDisableEditor"],
-                    //set the default
-                    false);
-            }
+            get { return GetOptionalTextElement("scriptDisableEditor", false); }
        }

    }
@@ -8,6 +8,7 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        Guid Id { get;  }
        string RepositoryUrl { get; }
        string WebServiceUrl { get; }
-        bool HasCustomWebServiceUrl { get; }
+        bool HasCustomWebServiceUrl { get; }
+        string RestApiUrl { get; }
    }
 }
@@ -2,7 +2,7 @@

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class ImagingAutoFillUploadFieldElement : ConfigurationElement, IImagingAutoFillUploadField
+    internal class ImagingAutoFillUploadFieldElement : UmbracoConfigurationElement, IImagingAutoFillUploadField
    {
        /// <summary>
        /// Allow setting internally so we can create a default
@@ -17,49 +17,25 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        [ConfigurationProperty("widthFieldAlias")]
        internal InnerTextConfigurationElement<string> WidthFieldAlias
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                       (InnerTextConfigurationElement<string>)this["widthFieldAlias"],
-                    //set the default
-                       "umbracoWidth");
-            }
+            get { return GetOptionalTextElement("widthFieldAlias", "umbracoWidth"); }
        }

        [ConfigurationProperty("heightFieldAlias")]
        internal InnerTextConfigurationElement<string> HeightFieldAlias
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                       (InnerTextConfigurationElement<string>)this["heightFieldAlias"],
-                    //set the default
-                       "umbracoHeight");
-            }
+            get { return GetOptionalTextElement("heightFieldAlias", "umbracoHeight"); }
        }

        [ConfigurationProperty("lengthFieldAlias")]
        internal InnerTextConfigurationElement<string> LengthFieldAlias
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                       (InnerTextConfigurationElement<string>)this["lengthFieldAlias"],
-                    //set the default
-                       "umbracoBytes");
-            }
+            get { return GetOptionalTextElement("lengthFieldAlias", "umbracoBytes"); }
        }

        [ConfigurationProperty("extensionFieldAlias")]
        internal InnerTextConfigurationElement<string> ExtensionFieldAlias
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                       (InnerTextConfigurationElement<string>)this["extensionFieldAlias"],
-                    //set the default
-                       "umbracoExtension");
-            }
+            get { return GetOptionalTextElement("extensionFieldAlias", "umbracoExtension"); }
        }

        string IImagingAutoFillUploadField.Alias
@@ -3,67 +3,37 @@ using System.Configuration;

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class LoggingElement : ConfigurationElement, ILoggingSection
+    internal class LoggingElement : UmbracoConfigurationElement, ILoggingSection
    {
        
        [ConfigurationProperty("autoCleanLogs")]
        internal InnerTextConfigurationElement<bool> AutoCleanLogs
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["autoCleanLogs"],
-                    //set the default
-                    false);            
-            }
+            get { return GetOptionalTextElement("autoCleanLogs", false); }
        }

        [ConfigurationProperty("enableLogging")]
        internal InnerTextConfigurationElement<bool> EnableLogging
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["enableLogging"],
-                    //set the default
-                    true);            
-            }
+            get { return GetOptionalTextElement("enableLogging", true); }
        }

        [ConfigurationProperty("enableAsyncLogging")]
        internal InnerTextConfigurationElement<bool> EnableAsyncLogging
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["enableAsyncLogging"],
-                    //set the default
-                    true);           
-            }
+            get { return GetOptionalTextElement("enableAsyncLogging", true); }
        }

        [ConfigurationProperty("cleaningMiliseconds")]
        internal InnerTextConfigurationElement<int> CleaningMiliseconds
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<int>(
-                    (InnerTextConfigurationElement<int>)this["cleaningMiliseconds"],
-                    //set the default
-                    -1);                      
-            }
+            get { return GetOptionalTextElement("cleaningMiliseconds", -1); }
        }

        [ConfigurationProperty("maxLogAge")]
        internal InnerTextConfigurationElement<int> MaxLogAge
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<int>(
-                    (InnerTextConfigurationElement<int>)this["maxLogAge"],
-                    //set the default
-                    -1);
-            }
+            get { return GetOptionalTextElement("maxLogAge", -1); }
        }

        [ConfigurationCollection(typeof(DisabledLogTypesCollection), AddItemName = "logTypeAlias")]
@@ -2,7 +2,7 @@

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class NotificationsElement : ConfigurationElement
+    internal class NotificationsElement : UmbracoConfigurationElement
    {
        [ConfigurationProperty("email")]
        internal InnerTextConfigurationElement<string> NotificationEmailAddress
@@ -13,13 +13,7 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
        [ConfigurationProperty("disableHtmlEmail")]
        internal InnerTextConfigurationElement<bool> DisableHtmlEmail
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>) this["disableHtmlEmail"], 
-                    //set the default
-                    false);
-            }
+            get { return GetOptionalTextElement("disableHtmlEmail", false); }
        }

    }
@@ -0,0 +1,19 @@
+using System;
+using System.Linq;
+
+namespace Umbraco.Core.Configuration.UmbracoSettings
+{
+    public static class RepositoryConfigExtensions
+    {
+        //Our package repo
+        private static readonly Guid RepoGuid = new Guid("65194810-1f85-11dd-bd0b-0800200c9a66");
+
+        public static IRepository GetDefault(this IRepositoriesSection repos)
+        {
+            var found = repos.Repositories.FirstOrDefault(x => x.Id == RepoGuid);
+            if (found == null)
+                throw new InvalidOperationException("No default package repository found with id " + RepoGuid);
+            return found;
+        }
+    }
+}
@@ -38,9 +38,16 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
            get
            {
                var prop = Properties["webserviceurl"];
-                var repoUrl = this[prop] as ConfigurationElement;
-                return (repoUrl != null && repoUrl.ElementInformation.IsPresent);
+                return (string) prop.DefaultValue != (string) this[prop];                
            }
        }
+
+        [ConfigurationProperty("restapiurl", DefaultValue = "https://our.umbraco.org/webapi/packages/v1")]
+        public string RestApiUrl
+        {
+            get { return (string)base["restapiurl"]; }
+            set { base["restapiurl"] = value; }
+        }
+        
    }
 }
@@ -5,30 +5,18 @@ using System.Collections.Generic;

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class RequestHandlerElement : ConfigurationElement, IRequestHandlerSection
+    internal class RequestHandlerElement : UmbracoConfigurationElement, IRequestHandlerSection
    {
        [ConfigurationProperty("useDomainPrefixes")]
        public InnerTextConfigurationElement<bool> UseDomainPrefixes
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["useDomainPrefixes"],
-                    //set the default
-                    false);  
-            }
+            get { return GetOptionalTextElement("useDomainPrefixes", false); }
        }

        [ConfigurationProperty("addTrailingSlash")]
        public InnerTextConfigurationElement<bool> AddTrailingSlash
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["addTrailingSlash"],
-                    //set the default
-                    true);  
-            }
+            get { return GetOptionalTextElement("addTrailingSlash", true); }
        }

        private UrlReplacingElement _defaultUrlReplacing;
@@ -2,66 +2,36 @@

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class SecurityElement : ConfigurationElement, ISecuritySection
+    internal class SecurityElement : UmbracoConfigurationElement, ISecuritySection
    {
        [ConfigurationProperty("keepUserLoggedIn")]
        internal InnerTextConfigurationElement<bool> KeepUserLoggedIn
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["keepUserLoggedIn"],
-                    //set the default
-                    true);   
-            }
+            get { return GetOptionalTextElement("keepUserLoggedIn", true); }
        }

        [ConfigurationProperty("hideDisabledUsersInBackoffice")]
        internal InnerTextConfigurationElement<bool> HideDisabledUsersInBackoffice
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["hideDisabledUsersInBackoffice"],
-                    //set the default
-                    false);                          
-            }
+            get { return GetOptionalTextElement("hideDisabledUsersInBackoffice", false); }
        }

        [ConfigurationProperty("allowPasswordReset")]
        internal InnerTextConfigurationElement<bool> AllowPasswordReset
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["allowPasswordReset"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("allowPasswordReset", true); }
        }

        [ConfigurationProperty("authCookieName")]
        internal InnerTextConfigurationElement<string> AuthCookieName
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                    (InnerTextConfigurationElement<string>)this["authCookieName"],
-                    //set the default
-                    Constants.Web.AuthCookieName);                
-            }
+            get { return GetOptionalTextElement("authCookieName", Constants.Web.AuthCookieName); }
        }

        [ConfigurationProperty("authCookieDomain")]
        internal InnerTextConfigurationElement<string> AuthCookieDomain
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<string>(
-                    (InnerTextConfigurationElement<string>)this["authCookieDomain"],
-                    //set the default
-                    null);                    
-            }
+            get { return GetOptionalTextElement<string>("authCookieDomain", null); }
        }

        bool ISecuritySection.KeepUserLoggedIn
@@ -3,55 +3,31 @@ using System.Configuration;

 namespace Umbraco.Core.Configuration.UmbracoSettings
 {
-    internal class TemplatesElement : ConfigurationElement, ITemplatesSection
+    internal class TemplatesElement : UmbracoConfigurationElement, ITemplatesSection
    {
        [ConfigurationProperty("useAspNetMasterPages")]
        internal InnerTextConfigurationElement<bool> UseAspNetMasterPages
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["useAspNetMasterPages"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("useAspNetMasterPages", true); }
        }

        [ConfigurationProperty("enableSkinSupport")]
        internal InnerTextConfigurationElement<bool> EnableSkinSupport
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["enableSkinSupport"],
-                    //set the default
-                    true);
-            }
+            get { return GetOptionalTextElement("enableSkinSupport", true); }
        }

        [ConfigurationProperty("defaultRenderingEngine", IsRequired = true)]
        internal InnerTextConfigurationElement<RenderingEngine> DefaultRenderingEngine
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<RenderingEngine>(
-                    (InnerTextConfigurationElement<RenderingEngine>)this["defaultRenderingEngine"],
-                    //set the default
-                    RenderingEngine.Mvc);
-            }
+            get { return GetOptionalTextElement("defaultRenderingEngine", RenderingEngine.Mvc); }
        }

        [Obsolete("This has no affect and will be removed in future versions")]
        [ConfigurationProperty("enableTemplateFolders")]
        internal InnerTextConfigurationElement<bool> EnableTemplateFolders
        {
-            get
-            {
-                return new OptionalInnerTextConfigurationElement<bool>(
-                    (InnerTextConfigurationElement<bool>)this["enableTemplateFolders"],
-                    //set the default
-                    false);
-            }
+            get { return GetOptionalTextElement("enableTemplateFolders", false); }
        }

        bool ITemplatesSection.UseAspNetMasterPages
@@ -0,0 +1,36 @@
+using System.Collections.Concurrent;
+using System.Configuration;
+
+namespace Umbraco.Core.Configuration.UmbracoSettings
+{
+    /// <summary>
+    /// Base class with shared helper methods
+    /// </summary>
+    internal class UmbracoConfigurationElement : ConfigurationElement
+    {
+        /// <summary>
+        /// Used so the RawElement types are not re-created every time they are accessed
+        /// </summary>
+        private readonly ConcurrentDictionary<string, RawXmlConfigurationElement> _rawElements = new ConcurrentDictionary<string, RawXmlConfigurationElement>();
+
+        protected OptionalInnerTextConfigurationElement<T> GetOptionalTextElement<T>(string name, T defaultVal)
+        {
+            return (OptionalInnerTextConfigurationElement<T>) _rawElements.GetOrAdd(
+                name,
+                s => new OptionalInnerTextConfigurationElement<T>(
+                    (InnerTextConfigurationElement<T>) this[s],
+                    //set the default
+                    defaultVal));
+        }
+
+        protected OptionalCommaDelimitedConfigurationElement GetOptionalDelimitedElement(string name, string[] defaultVal)
+        {
+            return (OptionalCommaDelimitedConfigurationElement) _rawElements.GetOrAdd(
+                name,
+                s => new OptionalCommaDelimitedConfigurationElement(
+                    (CommaDelimitedConfigurationElement) this[name],
+                    //set the default
+                    defaultVal));
+        }
+    }
+}
@@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration
 {
    public class UmbracoVersion
    {
-        private static readonly Version Version = new Version("7.5.6");
+        private static readonly Version Version = new Version("7.5.14");

        /// <summary>
        /// Gets the current version of Umbraco.
@@ -415,7 +415,7 @@ namespace Umbraco.Core

            if (currentTry == 5)
            {
-                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but the Umbraco cannot connect to the database.");
+                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but Umbraco cannot connect to the database.");
            }
        }

@@ -34,6 +34,12 @@ namespace Umbraco.Core
        private string _connectionString;
        private string _providerName;
        private DatabaseSchemaResult _result;
+        private DateTime? _connectionLastChecked = null;
+
+        /// <summary>
+        /// The number of minutes to throttle the checks to CanConnect
+        /// </summary>
+        private const int ConnectionCheckMinutes = 1;

        [Obsolete("Use the constructor specifying all dependencies instead")]
        public DatabaseContext(IDatabaseFactory factory)
@@ -110,13 +116,28 @@ namespace Umbraco.Core
        {
            get
            {
-                if (IsDatabaseConfigured == false) return false;
-                var canConnect = DbConnectionExtensions.IsConnectionAvailable(ConnectionString, DatabaseProvider);
-                LogHelper.Info<DatabaseContext>("CanConnect = " + canConnect);
-                return canConnect;
+                if (IsDatabaseConfigured == false)
+                    return false;
+
+                //Don't check again if the timeout period hasn't elapsed
+                //this ensures we don't keep checking the connection too many times in a row like during startup.
+                //Do check if the _connectionLastChecked is null which means we're just initializing or it could 
+                //not connect last time it was checked.
+                if ((_connectionLastChecked.HasValue && (DateTime.Now - _connectionLastChecked.Value).TotalMinutes > ConnectionCheckMinutes)
+                    || _connectionLastChecked.HasValue == false)
+                {
+                    var canConnect = DbConnectionExtensions.IsConnectionAvailable(ConnectionString, DatabaseProvider);
+                    LogHelper.Info<DatabaseContext>("CanConnect = " + canConnect);
+
+                    _connectionLastChecked = canConnect == false ? null : (DateTime?) DateTime.Now;
+                    return canConnect;
+                }
+
+                return _connectionLastChecked.HasValue;
            }
        }

+
        /// <summary>
        /// Gets the configured umbraco db connection string.
        /// </summary>
@@ -182,15 +203,10 @@ namespace Umbraco.Core
            var path = Path.Combine(GlobalSettings.FullpathToRoot, "App_Data", "Umbraco.sdf");
            if (File.Exists(path) == false)
            {
-                var engine = new SqlCeEngine(connectionString);
-                engine.CreateDatabase();
-
-                // SD: Pretty sure this should be in a using clause but i don't want to cause unknown side-effects here
-                // since it's been like this for quite some time
-                //using (var engine = new SqlCeEngine(connectionString))
-                //{
-                //    engine.CreateDatabase();    
-                //}
+                using (var engine = new SqlCeEngine(connectionString))
+                {
+                    engine.CreateDatabase();
+                }
            }

            Initialize(providerName);
@@ -0,0 +1,135 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+using Umbraco.Core.IO;
+
+namespace Umbraco.Core.Diagnostics
+{
+    // taken from https://blogs.msdn.microsoft.com/dondu/2010/10/24/writing-minidumps-in-c/
+    // and https://blogs.msdn.microsoft.com/dondu/2010/10/31/writing-minidumps-from-exceptions-in-c/
+    // which itself got it from http://blog.kalmbach-software.de/2008/12/13/writing-minidumps-in-c/
+
+    internal static class MiniDump
+    {
+        private static readonly object LockO = new object();
+
+        [Flags]
+        public enum Option : uint
+        {
+            // From dbghelp.h:
+            Normal = 0x00000000,
+            WithDataSegs = 0x00000001,
+            WithFullMemory = 0x00000002,
+            WithHandleData = 0x00000004,
+            FilterMemory = 0x00000008,
+            ScanMemory = 0x00000010,
+            WithUnloadedModules = 0x00000020,
+            WithIndirectlyReferencedMemory = 0x00000040,
+            FilterModulePaths = 0x00000080,
+            WithProcessThreadData = 0x00000100,
+            WithPrivateReadWriteMemory = 0x00000200,
+            WithoutOptionalData = 0x00000400,
+            WithFullMemoryInfo = 0x00000800,
+            WithThreadInfo = 0x00001000,
+            WithCodeSegs = 0x00002000,
+            WithoutAuxiliaryState = 0x00004000,
+            WithFullAuxiliaryState = 0x00008000,
+            WithPrivateWriteCopyMemory = 0x00010000,
+            IgnoreInaccessibleMemory = 0x00020000,
+            ValidTypeFlags = 0x0003ffff,
+        }
+
+        //typedef struct _MINIDUMP_EXCEPTION_INFORMATION {
+        //    DWORD ThreadId;
+        //    PEXCEPTION_POINTERS ExceptionPointers;
+        //    BOOL ClientPointers;
+        //} MINIDUMP_EXCEPTION_INFORMATION, *PMINIDUMP_EXCEPTION_INFORMATION;
+        [StructLayout(LayoutKind.Sequential, Pack = 4)]  // Pack=4 is important! So it works also for x64!
+        public struct MiniDumpExceptionInformation
+        {
+            public uint ThreadId;
+            public IntPtr ExceptionPointers;
+            [MarshalAs(UnmanagedType.Bool)]
+            public bool ClientPointers;
+        }
+
+        //BOOL
+        //WINAPI
+        //MiniDumpWriteDump(
+        //    __in HANDLE hProcess,
+        //    __in DWORD ProcessId,
+        //    __in HANDLE hFile,
+        //    __in MINIDUMP_TYPE DumpType,
+        //    __in_opt PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam,
+        //    __in_opt PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam,
+        //    __in_opt PMINIDUMP_CALLBACK_INFORMATION CallbackParam
+        //    );
+
+        // Overload requiring MiniDumpExceptionInformation
+        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
+        private static extern bool MiniDumpWriteDump(IntPtr hProcess, uint processId, SafeHandle hFile, uint dumpType, ref MiniDumpExceptionInformation expParam, IntPtr userStreamParam, IntPtr callbackParam);
+
+        // Overload supporting MiniDumpExceptionInformation == NULL
+        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
+        private static extern bool MiniDumpWriteDump(IntPtr hProcess, uint processId, SafeHandle hFile, uint dumpType, IntPtr expParam, IntPtr userStreamParam, IntPtr callbackParam);
+
+        [DllImport("kernel32.dll", EntryPoint = "GetCurrentThreadId", ExactSpelling = true)]
+        private static extern uint GetCurrentThreadId();
+
+        private static bool Write(SafeHandle fileHandle, Option options, bool withException = false)
+        {
+            var currentProcess = Process.GetCurrentProcess();
+            var currentProcessHandle = currentProcess.Handle;
+            var currentProcessId = (uint)currentProcess.Id;
+
+            MiniDumpExceptionInformation exp;
+
+            exp.ThreadId = GetCurrentThreadId();
+            exp.ClientPointers = false;
+            exp.ExceptionPointers = IntPtr.Zero;
+
+            if (withException)
+                exp.ExceptionPointers = Marshal.GetExceptionPointers();
+
+            var bRet = exp.ExceptionPointers == IntPtr.Zero
+                ? MiniDumpWriteDump(currentProcessHandle, currentProcessId, fileHandle, (uint) options, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero)
+                : MiniDumpWriteDump(currentProcessHandle, currentProcessId, fileHandle, (uint) options, ref exp, IntPtr.Zero, IntPtr.Zero);
+
+            return bRet;
+        }
+
+        public static bool Dump(Option options = Option.WithFullMemory, bool withException = false)
+        {
+            lock (LockO)
+            {
+                // work around "stack trace is not available while minidump debugging",
+                // by making sure a local var (that we can inspect) contains the stack trace.
+                // getting the call stack before it is unwound would require a special exception
+                // filter everywhere in our code = not!
+                var stacktrace = withException ? Environment.StackTrace : string.Empty;
+
+                var filepath = IOHelper.MapPath("~/App_Data/MiniDump");
+                if (Directory.Exists(filepath) == false)
+                    Directory.CreateDirectory(filepath);
+
+                var filename = Path.Combine(filepath, string.Format("{0:yyyyMMddTHHmmss}.{1}.dmp", DateTime.UtcNow, Guid.NewGuid().ToString("N").Substring(0, 4)));
+                using (var stream = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.Write))
+                {
+                    return Write(stream.SafeFileHandle, options, withException);
+                }
+            }
+        }
+
+        public static bool OkToDump()
+        {
+            lock (LockO)
+            {
+                var filepath = IOHelper.MapPath("~/App_Data/MiniDump");
+                if (Directory.Exists(filepath) == false) return true;
+                var count = Directory.GetFiles(filepath, "*.dmp").Length;
+                return count < 8;
+            }
+        }
+    }
+}
@@ -0,0 +1,12 @@
+using System;
+
+namespace Umbraco.Core.Exceptions
+{
+    internal class ConnectionException : Exception
+    {
+        public ConnectionException(string message, Exception innerException) : base(message, innerException)
+        {
+
+        }
+    }
+}
@@ -7,39 +7,144 @@ namespace Umbraco.Core.IO
    //TODO: There is no way to create a directory here without creating a file in a directory and then deleting it
    //TODO: Should probably implement a rename?

-	public interface IFileSystem
+    /// <summary>
+    /// Provides methods allowing the manipulation of files within an Umbraco application.
+    /// </summary>
+    public interface IFileSystem
    {
+        /// <summary>
+        /// Gets all directories matching the given path.
+        /// </summary>
+        /// <param name="path">The path to the directories.</param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched directories.
+        /// </returns>
        IEnumerable<string> GetDirectories(string path);

+        /// <summary>
+        /// Deletes the specified directory.
+        /// </summary>
+        /// <param name="path">The name of the directory to remove.</param>
        void DeleteDirectory(string path);

+        /// <summary>
+        /// Deletes the specified directory and, if indicated, any subdirectories and files in the directory.
+        /// </summary>
+        /// <remarks>Azure blob storage has no real concept of directories so deletion is always recursive.</remarks>
+        /// <param name="path">The name of the directory to remove.</param>
+        /// <param name="recursive">Whether to remove directories, subdirectories, and files in path.</param>
        void DeleteDirectory(string path, bool recursive);

+        /// <summary>
+        /// Determines whether the specified directory exists.
+        /// </summary>
+        /// <param name="path">The directory to check.</param>
+        /// <returns>
+        /// <c>True</c> if the directory exists and the user has permission to view it; otherwise <c>false</c>.
+        /// </returns>
        bool DirectoryExists(string path);
-        
+
+        /// <summary>
+        /// Adds a file to the file system.
+        /// </summary>
+        /// <param name="path">The path to the given file.</param>
+        /// <param name="stream">The <see cref="Stream"/> containing the file contents.</param>
        void AddFile(string path, Stream stream);

+        /// <summary>
+        /// Adds a file to the file system.
+        /// </summary>
+        /// <param name="path">The path to the given file.</param>
+        /// <param name="stream">The <see cref="Stream"/> containing the file contents.</param>
+        /// <param name="overrideIfExists">Whether to override the file if it already exists.</param>
        void AddFile(string path, Stream stream, bool overrideIfExists);

+        /// <summary>
+        /// Gets all files matching the given path.
+        /// </summary>
+        /// <param name="path">The path to the files.</param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched files.
+        /// </returns>
        IEnumerable<string> GetFiles(string path);

+        /// <summary>
+        /// Gets all files matching the given path and filter.
+        /// </summary>
+        /// <param name="path">The path to the files.</param>
+        /// <param name="filter">A filter that allows the querying of file extension. <example>*.jpg</example></param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched files.
+        /// </returns>
        IEnumerable<string> GetFiles(string path, string filter);

+        /// <summary>
+        /// Gets a <see cref="Stream"/> representing the file at the gieven path.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="Stream"/>.
+        /// </returns>
        Stream OpenFile(string path);

+        /// <summary>
+        /// Deletes the specified file.
+        /// </summary>
+        /// <param name="path">The name of the file to remove.</param>
        void DeleteFile(string path);

+        /// <summary>
+        /// Determines whether the specified file exists.
+        /// </summary>
+        /// <param name="path">The file to check.</param>
+        /// <returns>
+        /// <c>True</c> if the file exists and the user has permission to view it; otherwise <c>false</c>.
+        /// </returns>
        bool FileExists(string path);

-
+        /// <summary>
+        /// Returns the application relative path to the file.
+        /// </summary>
+        /// <param name="fullPathOrUrl">The full path or url.</param>
+        /// <returns>
+        /// The <see cref="string"/> representing the relative path.
+        /// </returns>
        string GetRelativePath(string fullPathOrUrl);

+        /// <summary>
+        /// Gets the full qualified path to the file.
+        /// </summary>
+        /// <param name="path">The file to return the full path for.</param>
+        /// <returns>
+        /// The <see cref="string"/> representing the full path.
+        /// </returns>
        string GetFullPath(string path);

+        /// <summary>
+        /// Returns the application relative url to the file.
+        /// </summary>
+        /// <param name="path">The path to return the url for.</param>
+        /// <returns>
+        /// <see cref="string"/> representing the relative url.
+        /// </returns>
        string GetUrl(string path);

+        /// <summary>
+        /// Gets the last modified date/time of the file, expressed as a UTC value.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="DateTimeOffset"/>.
+        /// </returns>
        DateTimeOffset GetLastModified(string path);

+        /// <summary>
+        /// Gets the created date/time of the file, expressed as a UTC value.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="DateTimeOffset"/>.
+        /// </returns>
        DateTimeOffset GetCreated(string path);
    }
 }
@@ -122,7 +122,7 @@ namespace Umbraco.Core.IO
            Directory.CreateDirectory(Path.GetDirectoryName(fullPath)); // ensure it exists

            if (stream.CanSeek)
-                stream.Seek(0, 0);
+                stream.Seek(0, SeekOrigin.Begin);

            using (var destination = (Stream)File.Create(fullPath))
                stream.CopyTo(destination);
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -72,15 +73,28 @@ namespace Umbraco.Core.IO
        {
            get
            {
-                if (GlobalSettings.ContentCacheXmlStoredInCodeGen && SystemUtilities.GetCurrentTrustLevel() == AspNetHostingPermissionLevel.Unrestricted)
-                {
-                    return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                switch (GlobalSettings.ContentCacheXmlStorageLocation)
+                {                    
+                    case ContentXmlStorage.AspNetTemp:
+                        return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                    case ContentXmlStorage.EnvironmentTemp:
+                        var appDomainHash = HttpRuntime.AppDomainAppId.ToSHA1();
+                        var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "UmbracoXml",
+                            //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back
+                            // to worker A again, in theory the %temp%  folder should already be empty but we really want to make sure that its not
+                            // utilizing an old path
+                            appDomainHash);
+                        return Path.Combine(cachePath, "umbraco.config");
+                    case ContentXmlStorage.Default:
+                        return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
+                    default:
+                        throw new ArgumentOutOfRangeException();
                }
-                return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
            }
        }

        [Obsolete("Use GlobalSettings.ContentCacheXmlStoredInCodeGen instead")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
        internal static bool ContentCacheXmlStoredInCodeGen
        {
            get { return GlobalSettings.ContentCacheXmlStoredInCodeGen; }
@@ -1,9 +1,9 @@
-namespace Umbraco.Core.Logging
-{
-    using System;
-    using System.Runtime.CompilerServices;
+using System;
+using System.Runtime.CompilerServices;
+using ImageProcessor.Common.Exceptions;

-    using ImageProcessor.Common.Exceptions;
+namespace Umbraco.Core.Logging
+{   

    /// <summary>
    /// A logger for explicitly logging ImageProcessor exceptions.
@@ -2,10 +2,13 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Web;
 using log4net;
 using log4net.Config;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Diagnostics;

 namespace Umbraco.Core.Logging
 {
@@ -56,18 +59,72 @@ namespace Umbraco.Core.Logging
 		internal ILog LoggerFor(object getTypeFromInstance)
 		{
 			if (getTypeFromInstance == null) throw new ArgumentNullException("getTypeFromInstance");
-			
+
 			return LogManager.GetLogger(getTypeFromInstance.GetType());
 		}
-        
+
 		public void Error(Type callingType, string message, Exception exception)
 		{
-			var logger = LogManager.GetLogger(callingType);
-			if (logger != null)
-				logger.Error((message), exception);
+            var logger = LogManager.GetLogger(callingType);
+		    if (logger == null) return;
+
+		    var dump = false;
+
+		    if (IsTimeoutThreadAbortException(exception))
+		    {
+		        message += "\r\nThe thread has been aborted, because the request has timed out.";
+
+                // dump if configured, or if stacktrace contains Monitor.ReliableEnter
+		        dump = UmbracoConfig.For.CoreDebug().DumpOnTimeoutThreadAbort || IsMonitorEnterThreadAbortException(exception);
+
+                // dump if it is ok to dump (might have a cap on number of dump...)
+		        dump &= MiniDump.OkToDump();
+		    }
+
+            if (dump)
+		    {
+                try
+                {
+                    var dumped = MiniDump.Dump(withException: true);
+                    message += dumped
+                        ? "\r\nA minidump was created in App_Data/MiniDump"
+                        : "\r\nFailed to create a minidump";
+                }
+                catch (Exception e)
+                {
+                    message += string.Format("\r\nFailed to create a minidump ({0}: {1})", e.GetType().FullName, e.Message);
+                }
+            }
+
+            logger.Error(message, exception);
 		}

-		public void Warn(Type callingType, string message, params Func<object>[] formatItems)
+        private static bool IsMonitorEnterThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            var stacktrace = abort.StackTrace;
+            return stacktrace.Contains("System.Threading.Monitor.ReliableEnter");
+        }
+
+        private static bool IsTimeoutThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            if (abort.ExceptionState == null) return false;
+
+            var stateType = abort.ExceptionState.GetType();
+            if (stateType.FullName != "System.Web.HttpApplication+CancelModuleException") return false;
+
+            var timeoutField = stateType.GetField("_timeout", BindingFlags.Instance | BindingFlags.NonPublic);
+            if (timeoutField == null) return false;
+
+            return (bool) timeoutField.GetValue(abort.ExceptionState);
+        }
+
+        public void Warn(Type callingType, string message, params Func<object>[] formatItems)
 		{
 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;
@@ -82,7 +139,7 @@ namespace Umbraco.Core.Logging
 			if (showHttpTrace && HttpContext.Current != null)
 			{
 				HttpContext.Current.Trace.Warn(callingType.Name, string.Format(message, formatItems.Select(x => x.Invoke()).ToArray()));
-			}	
+			}

 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;
@@ -99,7 +156,7 @@ namespace Umbraco.Core.Logging
            var logger = LogManager.GetLogger(callingType);
            if (logger == null || logger.IsWarnEnabled == false) return;
            var executedParams = formatItems.Select(x => x.Invoke()).ToArray();
-            logger.WarnFormat((message) + ". Exception: " + e, executedParams);		
+            logger.WarnFormat((message) + ". Exception: " + e, executedParams);
 		}

 		/// <summary>
@@ -227,35 +227,37 @@ namespace Umbraco.Core.Media
                    }

                    // Set compresion ratio to 90%
-                    var ep = new EncoderParameters();
-                    ep.Param[0] = new EncoderParameter(Encoder.Quality, 90L);
-
-                    // Save the new image using the dimensions of the image
-                    var predictableThumbnailName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", maxWidthHeight.ToString(CultureInfo.InvariantCulture));
-                    var predictableThumbnailNameJpg = predictableThumbnailName.Substring(0, predictableThumbnailName.LastIndexOf(".", StringComparison.Ordinal)) + ".jpg";
-                    using (var ms = new MemoryStream())
+                    string newFileName;
+                    using (var ep = new EncoderParameters())
                    {
-                        bp.Save(ms, codec, ep);
-                        ms.Seek(0, 0);
+                        ep.Param[0] = new EncoderParameter(Encoder.Quality, 90L);

-                        fs.AddFile(predictableThumbnailName, ms);
-                        fs.AddFile(predictableThumbnailNameJpg, ms);
-                    }
+                        // Save the new image using the dimensions of the image
+                        var predictableThumbnailName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", maxWidthHeight.ToString(CultureInfo.InvariantCulture));
+                        var predictableThumbnailNameJpg = predictableThumbnailName.Substring(0, predictableThumbnailName.LastIndexOf(".", StringComparison.Ordinal)) + ".jpg";
+                        using (var ms = new MemoryStream())
+                        {
+                            bp.Save(ms, codec, ep);
+                        ms.Seek(0, SeekOrigin.Begin);

-                    // TODO: Remove this, this is ONLY here for backwards compatibility but it is essentially completely unusable see U4-5385
-                    var newFileName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", string.Format("{0}x{1}", widthTh, heightTh));
-                    using (var ms = new MemoryStream())
-                    {
-                        bp.Save(ms, codec, ep);
-                        ms.Seek(0, 0);
+                            fs.AddFile(predictableThumbnailName, ms);
+                            fs.AddFile(predictableThumbnailNameJpg, ms);
+                        }

-                        fs.AddFile(newFileName, ms);
+                        // TODO: Remove this, this is ONLY here for backwards compatibility but it is essentially completely unusable see U4-5385
+                        newFileName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", string.Format("{0}x{1}", widthTh, heightTh));
+                        using (var ms = new MemoryStream())
+                        {
+                            bp.Save(ms, codec, ep);
+                        ms.Seek(0, SeekOrigin.Begin);
+
+                            fs.AddFile(newFileName, ms);
+                        }
                    }

                    return new ResizedImage(widthTh, heightTh, newFileName);
                }
            }
        }
-
    }
 }
@@ -274,6 +274,9 @@ namespace Umbraco.Core.Models
        /// </summary>
        public bool HasPublishedVersion { get { return PublishedVersionGuid != default(Guid); } }

+        [IgnoreDataMember]
+        internal DateTime PublishedDate { get; set; }
+
        /// <summary>
        /// Changes the Trashed state of the content object
        /// </summary>
@@ -404,8 +404,7 @@ namespace Umbraco.Core.Models
            return content.Properties
                          .Where(property => propertyGroup.PropertyTypes
                                                          .Select(propertyType => propertyType.Id)
-                                                          .Contains(property.PropertyTypeId))
-                          .OrderBy(x => x.PropertyType.SortOrder);
+                                                          .Contains(property.PropertyTypeId));
        }

        /// <summary>
@@ -546,7 +545,7 @@ namespace Umbraco.Core.Models
            if (supportsResizing)
            {
                //get the original image from the original stream
-                if (fileStream.CanSeek) fileStream.Seek(0, 0);
+                if (fileStream.CanSeek) fileStream.Seek(0, SeekOrigin.Begin);
                using (var originalImage = Image.FromStream(fileStream))
                {
                    var additionalSizes = new List<int>();
@@ -4,7 +4,7 @@ namespace Umbraco.Core.Models
 {
    /// <summary>
    /// Defines a ContentType, which Media is based on
-    /// </summary
+    /// </summary>
    public interface IMediaType : IContentTypeComposition
    {

@@ -13,6 +13,7 @@ namespace Umbraco.Core.Models.Identity
        public override void ConfigureMappings(IConfiguration config, ApplicationContext applicationContext)
        {
            config.CreateMap<IUser, BackOfficeIdentityUser>()
+                .ForMember(user => user.LastLoginDateUtc, expression => expression.MapFrom(user => user.LastLoginDate.ToUniversalTime()))
                .ForMember(user => user.Email, expression => expression.MapFrom(user => user.Email))
                .ForMember(user => user.Id, expression => expression.MapFrom(user => user.Id))
                .ForMember(user => user.LockoutEndDateUtc, expression => expression.MapFrom(user => user.IsLockedOut ? DateTime.MaxValue.ToUniversalTime() : (DateTime?)null))                
@@ -24,12 +24,17 @@ namespace Umbraco.Core.Models.Identity
        /// 
        /// </summary>
        public IdentityUser()
-        {
+        {            
            this.Claims = new List<TClaim>();
            this.Roles = new List<TRole>();
            this.Logins = new List<TLogin>();
        }

+        /// <summary>
+        /// Last login date
+        /// </summary>
+        public virtual DateTime? LastLoginDateUtc { get; set; }
+
        /// <summary>
        /// Email
        /// 
@@ -69,6 +69,7 @@ namespace Umbraco.Core.Models
            OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
        }

+        //TODO: Instead of 'new' this should explicitly implement one of the collection interfaces members
        internal new void Add(PropertyType item)
        {
            using (new WriteLock(_addLocker))
@@ -110,6 +110,12 @@ namespace Umbraco.Core.Models
            _ruleCollection.Clear();
        }

+
+        internal void ClearRemovedRules()
+        {
+            _removedRules.Clear();
+        }
+
        [DataMember]
        public int LoginNodeId
        {
@@ -179,7 +179,7 @@ namespace Umbraco.Core.Models.PublishedContent
            }

            if (contentType == null)
-                throw new Exception(string.Format("ContentTypeService failed to find a {0} type with alias \"{1}\".",
+                throw new InvalidOperationException(string.Format("ContentTypeService failed to find a {0} type with alias \"{1}\".",
                    itemType.ToString().ToLower(), alias));

            return new PublishedContentType(contentType);
@@ -106,12 +106,13 @@ namespace Umbraco.Core.Models.PublishedContent
        /// <param name="propertyTypeAlias">The property type alias.</param>
        /// <param name="dataTypeDefinitionId">The datatype definition identifier.</param>
        /// <param name="propertyEditorAlias">The property editor alias.</param>
+        /// <param name="initConverters">Generally used only for testing, in production this will always be true</param>
        /// <remarks>
        /// <para>The new published property type does not belong to a published content type.</para>
        /// <para>The values of <paramref name="dataTypeDefinitionId"/> and <paramref name="propertyEditorAlias"/> are
        /// assumed to be valid and consistent.</para>
        /// </remarks>
-        internal PublishedPropertyType(string propertyTypeAlias, int dataTypeDefinitionId, string propertyEditorAlias)
+        internal PublishedPropertyType(string propertyTypeAlias, int dataTypeDefinitionId, string propertyEditorAlias, bool initConverters = true)
        {
            // ContentType 
            // - in unit tests, to be set by PublishedContentType when creating it
@@ -122,7 +123,8 @@ namespace Umbraco.Core.Models.PublishedContent
            DataTypeId = dataTypeDefinitionId;
            PropertyEditorAlias = propertyEditorAlias;

-            InitializeConverters();
+            if (initConverters)
+                InitializeConverters();
        }

        #endregion
@@ -19,5 +19,8 @@ namespace Umbraco.Core.Models.Rdbms

        [Column("newest")]
        public bool Newest { get; set; }
+
+        [Column("updateDate")]
+        public DateTime VersionDate { get; set; }
    }
 }
@@ -1,15 +1,117 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using System.Web.UI.WebControls;
+using Umbraco.Core.Logging;
 using Umbraco.Core.Models.EntityBase;
+using Umbraco.Core.Models.Rdbms;

 namespace Umbraco.Core.Models
 {
    internal static class UmbracoEntityExtensions
    {
+        /// <summary>
+        /// Does a quick check on the entity's set path to ensure that it's valid and consistent
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <returns></returns>
+        public static void ValidatePathWithException(this NodeDto entity)
+        {
+            //don't validate if it's empty and it has no id
+            if (entity.NodeId == default(int) && entity.Path.IsNullOrWhiteSpace())
+                return;
+
+            if (entity.Path.IsNullOrWhiteSpace())
+                throw new InvalidDataException(string.Format("The content item {0} has an empty path: {1} with parentID: {2}", entity.NodeId, entity.Path, entity.ParentId));
+
+            var pathParts = entity.Path.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+            if (pathParts.Length < 2)
+            {
+                //a path cannot be less than 2 parts, at a minimum it must be root (-1) and it's own id
+                throw new InvalidDataException(string.Format("The content item {0} has an invalid path: {1} with parentID: {2}", entity.NodeId, entity.Path, entity.ParentId));
+            }
+
+            if (entity.ParentId != default(int) && pathParts[pathParts.Length - 2] != entity.ParentId.ToInvariantString())
+            {
+                //the 2nd last id in the path must be it's parent id
+                throw new InvalidDataException(string.Format("The content item {0} has an invalid path: {1} with parentID: {2}", entity.NodeId, entity.Path, entity.ParentId));
+            }
+        }
+
+        /// <summary>
+        /// Does a quick check on the entity's set path to ensure that it's valid and consistent
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <returns></returns>
+        public static bool ValidatePath(this IUmbracoEntity entity)
+        {
+            //don't validate if it's empty and it has no id
+            if (entity.HasIdentity == false && entity.Path.IsNullOrWhiteSpace())
+                return true;
+
+            if (entity.Path.IsNullOrWhiteSpace())
+                return false;
+
+            var pathParts = entity.Path.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+            if (pathParts.Length < 2)
+            {
+                //a path cannot be less than 2 parts, at a minimum it must be root (-1) and it's own id
+                return false;
+            }
+
+            if (entity.ParentId != default(int) && pathParts[pathParts.Length - 2] != entity.ParentId.ToInvariantString())
+            {
+                //the 2nd last id in the path must be it's parent id
+                return false;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// This will validate the entity's path and if it's invalid it will fix it, if fixing is required it will recursively 
+        /// check and fix all ancestors if required.
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <param name="logger"></param>
+        /// <param name="getParent">A callback specified to retrieve the parent entity of the entity</param>
+        /// <param name="update">A callback specified to update a fixed entity</param>
+        public static void EnsureValidPath<T>(this T entity, 
+            ILogger logger, 
+            Func<T, T> getParent,
+            Action<T> update)
+            where T: IUmbracoEntity
+        {
+            if (entity.HasIdentity == false)
+                throw new InvalidOperationException("Could not ensure the entity path, the entity has not been assigned an identity");
+
+            if (entity.ValidatePath() == false)
+            {
+                logger.Warn(typeof(UmbracoEntityExtensions), string.Format("The content item {0} has an invalid path: {1} with parentID: {2}", entity.Id, entity.Path, entity.ParentId));
+                if (entity.ParentId == -1)
+                {
+                    entity.Path = string.Concat("-1,", entity.Id);
+                    //path changed, update it
+                    update(entity);
+                }
+                else
+                {
+                    var parent = getParent(entity);
+                    if (parent == null)
+                        throw new NullReferenceException("Could not ensure path for entity " + entity.Id + " could not resolve it's parent " + entity.ParentId);
+
+                    //the parent must also be valid!
+                    parent.EnsureValidPath(logger, getParent, update);
+
+                    entity.Path = string.Concat(parent.Path, ",", entity.Id);
+                    //path changed, update it
+                    update(entity);
+                }
+            }
+        }

        public static bool HasChildren(this IUmbracoEntity entity)
        {
@@ -22,6 +22,7 @@ namespace Umbraco.Core.ObjectResolution
        private readonly string _httpContextKey;
        private readonly List<Type> _instanceTypes = new List<Type>();
        private IEnumerable<TResolved> _sortedValues;
+        private readonly Func<HttpContextBase> _httpContextGetter;

        private int _defaultPluginWeight = 100;

@@ -42,12 +43,7 @@ namespace Umbraco.Core.ObjectResolution
            if (logger == null) throw new ArgumentNullException("logger");
            CanResolveBeforeFrozen = false;
            if (scope == ObjectLifetimeScope.HttpRequest)
-            {
-                if (HttpContext.Current == null)
-                    throw new InvalidOperationException("Use alternative constructor accepting a HttpContextBase object in order to set the lifetime scope to HttpRequest when HttpContext.Current is null");
-
-                CurrentHttpContext = new HttpContextWrapper(HttpContext.Current);
-            }
+                _httpContextGetter = () => new HttpContextWrapper(HttpContext.Current);

            ServiceProvider = serviceProvider;
            Logger = logger;
@@ -84,7 +80,7 @@ namespace Umbraco.Core.ObjectResolution
            LifetimeScope = ObjectLifetimeScope.HttpRequest;
            _httpContextKey = GetType().FullName;
            ServiceProvider = serviceProvider;
-            CurrentHttpContext = httpContext;
+            _httpContextGetter = () => httpContext;
            _instanceTypes = new List<Type>();

            InitializeAppInstances();
@@ -160,7 +156,16 @@ namespace Umbraco.Core.ObjectResolution
        /// Gets or sets the <see cref="HttpContextBase"/> used to initialize this object, if any.
        /// </summary>
        /// <remarks>If not null, then <c>LifetimeScope</c> will be <c>ObjectLifetimeScope.HttpRequest</c>.</remarks>
-        protected HttpContextBase CurrentHttpContext { get; private set; }
+        protected HttpContextBase CurrentHttpContext
+        {
+            get
+            {
+                var context = _httpContextGetter == null ? null : _httpContextGetter();
+                if (context == null)
+                    throw new InvalidOperationException("Cannot use this resolver with lifetime 'HttpRequest' when there is no current HttpContext. Either use the ctor accepting an HttpContextBase, or use the resolver from within a request exclusively.");
+                return context;
+            }
+        }

        /// <summary>
        /// Returns the service provider used to instantiate objects
@@ -196,7 +201,7 @@ namespace Umbraco.Core.ObjectResolution
        /// <summary>
        /// Gets or sets the default type weight.
        /// </summary>
-        /// <remarks>Determines the weight of types that do not have a <c>WeightAttribute</c> set on 
+        /// <remarks>Determines the weight of types that do not have a <c>WeightAttribute</c> set on
        /// them, when calling <c>GetSortedValues</c>.</remarks>
        protected virtual int DefaultPluginWeight
        {
@@ -276,7 +281,7 @@ namespace Umbraco.Core.ObjectResolution
        /// Removes a type.
        /// </summary>
        /// <param name="value">The type to remove.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or
        /// the type is not a valid type for the resolver.</exception>
        public virtual void RemoveType(Type value)
        {
@@ -296,7 +301,7 @@ namespace Umbraco.Core.ObjectResolution
        /// Removes a type.
        /// </summary>
        /// <typeparam name="T">The type to remove.</typeparam>
-        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or
        /// the type is not a valid type for the resolver.</exception>
        public void RemoveType<T>()
            where T : TResolved
@@ -309,7 +314,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <param name="types">The types to add.</param>
        /// <remarks>The types are appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
        /// a type is not a valid type for the resolver, or a type is already in the collection of types.</exception>
        protected void AddTypes(IEnumerable<Type> types)
        {
@@ -336,7 +341,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <param name="value">The type to add.</param>
        /// <remarks>The type is appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
        /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
        public virtual void AddType(Type value)
        {
@@ -362,7 +367,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <typeparam name="T">The type to add.</typeparam>
        /// <remarks>The type is appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
        /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
        public void AddType<T>()
            where T : TResolved
@@ -404,7 +409,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <param name="index">The zero-based index at which the type should be inserted.</param>
        /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
        /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
        /// <exception cref="ArgumentOutOfRangeException"><paramref name="index"/> is out of range.</exception>
        public virtual void InsertType(int index, Type value)
@@ -430,7 +435,7 @@ namespace Umbraco.Core.ObjectResolution
        /// Inserts a type at the beginning of the list.
        /// </summary>
        /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
        /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
        public virtual void InsertType(Type value)
        {
@@ -464,7 +469,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <param name="existingType">The existing type before which to insert.</param>
        /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
        /// one of the types is not a valid type for the resolver, or the existing type is not in the collection,
        /// or the new type is already in the collection of types.</exception>
        public virtual void InsertTypeBefore(Type existingType, Type value)
@@ -498,7 +503,7 @@ namespace Umbraco.Core.ObjectResolution
        /// </summary>
        /// <typeparam name="TExisting">The existing type before which to insert.</typeparam>
        /// <typeparam name="T">The type to insert.</typeparam>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
        /// one of the types is not a valid type for the resolver, or the existing type is not in the collection,
        /// or the new type is already in the collection of types.</exception>
        public void InsertTypeBefore<TExisting, T>()
@@ -4,9 +4,11 @@ using System.Configuration;
 using System.Data;
 using System.Data.Common;
 using System.Data.SqlClient;
+using System.Data.SqlServerCe;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using MySql.Data.MySqlClient;
 using Umbraco.Core.Logging; 

 namespace Umbraco.Core.Persistence
@@ -69,6 +71,38 @@ namespace Umbraco.Core.Persistence
            }
        }

+        public static string GetConnStringExSecurityInfo(this IDbConnection connection)
+        {
+            try
+            {
+                if (connection is SqlConnection)
+                {
+                    var builder = new SqlConnectionStringBuilder(connection.ConnectionString);
+                    return string.Format("DataSource: {0}, InitialCatalog: {1}", builder.DataSource, builder.InitialCatalog);
+                }
+
+                if (connection is SqlCeConnection)
+                {
+                    var builder = new SqlCeConnectionStringBuilder(connection.ConnectionString);
+                    return string.Format("DataSource: {0}", builder.DataSource);
+                }
+
+                if (connection is MySqlConnection)
+                {
+                    var builder = new MySqlConnectionStringBuilder(connection.ConnectionString);
+                    return string.Format("Server: {0}, Database: {1}", builder.Server, builder.Database);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogHelper.WarnWithException(typeof(DbConnectionExtensions),
+                    "Could not resolve connection string parameters", ex);
+                return "(Could not resolve)";
+            }
+
+            throw new ArgumentException(string.Format("The connection type {0} is not supported", connection.GetType()));
+        }
+
        public static bool IsAvailable(this IDbConnection connection)
        {
            try
@@ -79,7 +113,8 @@ namespace Umbraco.Core.Persistence
            catch (DbException exc)
            {
                // Don't swallow this error, the exception is super handy for knowing "why" its not available
-                LogHelper.WarnWithException<IDbConnection>("Configured database is reporting as not being available!", exc);
+                LogHelper.WarnWithException(typeof(DbConnectionExtensions),
+                    "Configured database is reporting as not being available! {0}", exc, connection.GetConnStringExSecurityInfo);
                return false;
            }

@@ -27,15 +27,28 @@ namespace Umbraco.Core.Persistence.Factories

        #region Implementation of IEntityFactory<IContent,DocumentDto>

-        public IContent BuildEntity(DocumentDto dto)
+        /// <summary>
+        /// Builds a IContent item from the dto(s) and content type
+        /// </summary>
+        /// <param name="dto">
+        /// This DTO can contain all of the information to build an IContent item, however in cases where multiple entities are being built,
+        /// a separate <see cref="DocumentPublishedReadOnlyDto"/> publishedDto entity will be supplied in place of the <see cref="DocumentDto"/>'s own 
+        /// ResultColumn DocumentPublishedReadOnlyDto
+        /// </param>
+        /// <param name="contentType"></param>
+        /// <param name="publishedDto">
+        /// When querying for multiple content items the main DTO will not contain the ResultColumn DocumentPublishedReadOnlyDto and a separate publishedDto instance will be supplied
+        /// </param>
+        /// <returns></returns>
+        public static IContent BuildEntity(DocumentDto dto, IContentType contentType, DocumentPublishedReadOnlyDto publishedDto = null)
        {
-            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, _contentType);
+            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, contentType);

            try
            {
                content.DisableChangeTracking();

-                content.Id = _id;
+                content.Id = dto.NodeId;
                content.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                content.Name = dto.Text;
                content.NodeName = dto.ContentVersionDto.ContentDto.NodeDto.Text;
@@ -49,11 +62,19 @@ namespace Umbraco.Core.Persistence.Factories
                content.Published = dto.Published;
                content.CreateDate = dto.ContentVersionDto.ContentDto.NodeDto.CreateDate;
                content.UpdateDate = dto.ContentVersionDto.VersionDate;
-                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?) null;
-                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?) null;
+                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?)null;
+                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?)null;
                content.Version = dto.ContentVersionDto.VersionId;
+
                content.PublishedState = dto.Published ? PublishedState.Published : PublishedState.Unpublished;
-                content.PublishedVersionGuid = dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId;
+
+                //Check if the publishedDto has been supplied, if not the use the dto's own DocumentPublishedReadOnlyDto value
+                content.PublishedVersionGuid = publishedDto == null
+                    ? (dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId)
+                    : publishedDto.VersionId;
+                content.PublishedDate = publishedDto == null
+                    ? (dto.DocumentPublishedReadOnlyDto == null ? default(DateTime) : dto.DocumentPublishedReadOnlyDto.VersionDate)
+                    : publishedDto.VersionDate;

                //on initial construction we don't want to have dirty properties tracked
                // http://issues.umbraco.org/issue/U4-1946
@@ -64,6 +85,13 @@ namespace Umbraco.Core.Persistence.Factories
            {
                content.EnableChangeTracking();
            }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IContent BuildEntity(DocumentDto dto)
+        {
+            return BuildEntity(dto, _contentType);
        }

        public DocumentDto BuildDto(IContent entity)
@@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories

        #region Implementation of IEntityFactory<IMedia,ContentVersionDto>

-        public IMedia BuildEntity(ContentVersionDto dto)
+        public static IMedia BuildEntity(ContentVersionDto dto, IMediaType contentType)
        {
-            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, _contentType);
+            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, contentType);

            try
            {
                media.DisableChangeTracking();

-                media.Id = _id;
+                media.Id = dto.NodeId;
                media.Key = dto.ContentDto.NodeDto.UniqueId;
                media.Path = dto.ContentDto.NodeDto.Path;
                media.CreatorId = dto.ContentDto.NodeDto.UserId.Value;
@@ -55,6 +55,13 @@ namespace Umbraco.Core.Persistence.Factories
            {
                media.EnableChangeTracking();
            }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMedia BuildEntity(ContentVersionDto dto)
+        {
+            return BuildEntity(dto, _contentType);
        }

        public ContentVersionDto BuildDto(IMedia entity)
@@ -28,17 +28,17 @@ namespace Umbraco.Core.Persistence.Factories

        #region Implementation of IEntityFactory<IMedia,ContentVersionDto>

-        public IMember BuildEntity(MemberDto dto)
+        public static IMember BuildEntity(MemberDto dto, IMemberType contentType)
        {
            var member = new Member(
                dto.ContentVersionDto.ContentDto.NodeDto.Text,
-                dto.Email, dto.LoginName, dto.Password, _contentType);
+                dto.Email, dto.LoginName, dto.Password, contentType);

            try
            {
                member.DisableChangeTracking();

-                member.Id = _id;
+                member.Id = dto.NodeId;
                member.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                member.Path = dto.ContentVersionDto.ContentDto.NodeDto.Path;
                member.CreatorId = dto.ContentVersionDto.ContentDto.NodeDto.UserId.Value;
@@ -62,6 +62,12 @@ namespace Umbraco.Core.Persistence.Factories
            }
        }

+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMember BuildEntity(MemberDto dto)
+        {
+            return BuildEntity(dto, _contentType);
+        }
+
        public MemberDto BuildDto(IMember entity)
        {
            var dto = new MemberDto
@@ -30,11 +30,11 @@ namespace Umbraco.Core.Persistence.Factories
            _updateDate = updateDate;
        }

-        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        public static IEnumerable<Property> BuildEntity(IReadOnlyCollection<PropertyDataDto> dtos, PropertyType[] compositionTypeProperties, DateTime createDate, DateTime updateDate)
        {
            var properties = new List<Property>();

-            foreach (var propertyType in _compositionTypeProperties)
+            foreach (var propertyType in compositionTypeProperties)
            {
                var propertyDataDto = dtos.LastOrDefault(x => x.PropertyTypeId == propertyType.Id);
                var property = propertyDataDto == null
@@ -47,8 +47,8 @@ namespace Umbraco.Core.Persistence.Factories
                    //on initial construction we don't want to have dirty properties tracked
                    property.DisableChangeTracking();

-                    property.CreateDate = _createDate;
-                    property.UpdateDate = _updateDate;
+                    property.CreateDate = createDate;
+                    property.UpdateDate = updateDate;
                    // http://issues.umbraco.org/issue/U4-1946
                    property.ResetDirtyProperties(false);
                    properties.Add(property);
@@ -57,12 +57,18 @@ namespace Umbraco.Core.Persistence.Factories
                {
                    property.EnableChangeTracking();
                }
-                
+
            }

            return properties;
        }

+        [Obsolete("Use the static method instead, there's no reason to allocate one of these classes everytime we want to map values")]
+        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        {
+            return BuildEntity(dtos, _compositionTypeProperties, _createDate, _updateDate);
+        }
+
        public IEnumerable<PropertyDataDto> BuildDto(IEnumerable<Property> properties)
        {
            var propertyDataDtos = new List<PropertyDataDto>();
@@ -6,6 +6,7 @@ using System.Reflection;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Strings;

 namespace Umbraco.Core.Persistence.Factories
 {
@@ -17,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories
            
            //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data
            foreach (var k in originalEntityProperties.Keys
-                .Select(x => new { orig = x, title = x.ConvertCase(StringAliasCaseType.PascalCase) })
+                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase | CleanStringType.Ascii | CleanStringType.ConvertCase) })
                .Where(x => entityProps.InvariantContains(x.title) == false))
            {
                entity.AdditionalData[k.title] = originalEntityProperties[k.orig];
@@ -75,65 +76,6 @@ namespace Umbraco.Core.Persistence.Factories
                entity.EnableChangeTracking();
            }
        }
-
-        public UmbracoEntity BuildEntity(EntityRepository.UmbracoEntityDto dto)
-        {
-            var entity = new UmbracoEntity(dto.Trashed)
-                             {
-                                 CreateDate = dto.CreateDate,
-                                 CreatorId = dto.UserId.Value,
-                                 Id = dto.NodeId,
-                                 Key = dto.UniqueId,
-                                 Level = dto.Level,
-                                 Name = dto.Text,
-                                 NodeObjectTypeId = dto.NodeObjectType.Value,
-                                 ParentId = dto.ParentId,
-                                 Path = dto.Path,
-                                 SortOrder = dto.SortOrder,
-                                 HasChildren = dto.Children > 0,
-                                 ContentTypeAlias = dto.Alias ?? string.Empty,
-                                 ContentTypeIcon = dto.Icon ?? string.Empty,
-                                 ContentTypeThumbnail = dto.Thumbnail ?? string.Empty,
-                             };
-
-            entity.IsPublished = dto.PublishedVersion != default(Guid) || (dto.NewestVersion != default(Guid) && dto.PublishedVersion == dto.NewestVersion);
-            entity.IsDraft = dto.NewestVersion != default(Guid) && (dto.PublishedVersion == default(Guid) || dto.PublishedVersion != dto.NewestVersion);
-            entity.HasPendingChanges = (dto.PublishedVersion != default(Guid) && dto.NewestVersion != default(Guid)) && dto.PublishedVersion != dto.NewestVersion;
-
-            if (dto.UmbracoPropertyDtos != null)
-            {
-                foreach (var propertyDto in dto.UmbracoPropertyDtos)
-                {
-                    entity.AdditionalData[propertyDto.PropertyAlias] = new UmbracoEntity.EntityProperty
-                    {
-                        PropertyEditorAlias = propertyDto.PropertyEditorAlias,
-                        Value = propertyDto.NTextValue.IsNullOrWhiteSpace()
-                            ? propertyDto.NVarcharValue
-                            : propertyDto.NTextValue.ConvertToJsonIfPossible()
-                    };
-                }
-            }
-
-            return entity;
-        }
-
-        public EntityRepository.UmbracoEntityDto BuildDto(UmbracoEntity entity)
-        {
-            var node = new EntityRepository.UmbracoEntityDto
-                           {
-                               CreateDate = entity.CreateDate,
-                               Level = short.Parse(entity.Level.ToString(CultureInfo.InvariantCulture)),
-                               NodeId = entity.Id,
-                               NodeObjectType = entity.NodeObjectTypeId,
-                               ParentId = entity.ParentId,
-                               Path = entity.Path,
-                               SortOrder = entity.SortOrder,
-                               Text = entity.Name,
-                               Trashed = entity.Trashed,
-                               UniqueId = entity.Key,
-                               UserId = entity.CreatorId
-                           };
-            return node;
-        }
+        
    }
 }
@@ -31,7 +31,7 @@ namespace Umbraco.Core.Persistence.Mappers
        /// </summary>
        /// <param name="type"></param>
        /// <returns></returns>
-        internal BaseMapper ResolveMapperByType(Type type)
+        public virtual BaseMapper ResolveMapperByType(Type type)
        {
            return _mapperCache.GetOrAdd(type, type1 =>
                {
@@ -67,7 +67,7 @@ namespace Umbraco.Core.Persistence.Mappers
            return Attempt<BaseMapper>.Succeed(mapper);
        }  

-        internal string GetMapping(Type type, string propertyName)
+        public virtual string GetMapping(Type type, string propertyName)
        {
            var mapper = ResolveMapperByType(type);
            var result = mapper.Map(propertyName);
@@ -23,7 +23,7 @@ namespace Umbraco.Core.Persistence.Migrations
            Logger = logger;
        }

-        internal IMigrationContext Context;
+        public IMigrationContext Context;

        public abstract void Up();
        public abstract void Down();
@@ -27,6 +27,7 @@ namespace Umbraco.Core.Persistence
            return sql.From(sqlSyntax.GetQuotedTableName(tableName));
        }

+        [Obsolete("Use the overload specifying ISqlSyntaxProvider instead")]
        public static Sql Where<T>(this Sql sql, Expression<Func<T, bool>> predicate)
        {
            var expresionist = new PocoToSqlExpressionVisitor<T>();
@@ -34,6 +35,13 @@ namespace Umbraco.Core.Persistence
            return sql.Where(whereExpression, expresionist.GetSqlParameters());
        }

+        public static Sql Where<T>(this Sql sql, Expression<Func<T, bool>> predicate, ISqlSyntaxProvider sqlSyntax)
+        {
+            var expresionist = new PocoToSqlExpressionVisitor<T>(sqlSyntax);
+            var whereExpression = expresionist.Visit(predicate);
+            return sql.Where(whereExpression, expresionist.GetSqlParameters());
+        }
+
        private static string GetFieldName<T>(Expression<Func<T, object>> fieldSelector, ISqlSyntaxProvider sqlSyntax)
        {
            var field = ExpressionHelper.FindProperty(fieldSelector) as PropertyInfo;
@@ -581,6 +581,18 @@ namespace Umbraco.Core.Persistence.Querying
                case "InvariantContains":
                case "InvariantEquals":

+                    //special case, if it is 'Contains' and the argument that Contains is being called on is 
+                    //Enumerable and the methodArgs is the actual member access, then it's an SQL IN clause
+                    if (m.Object == null 
+                        && m.Arguments[0].Type != typeof(string)
+                        && m.Arguments.Count == 2
+                        && methodArgs.Length == 1 
+                        && methodArgs[0].NodeType == ExpressionType.MemberAccess
+                        && TypeHelper.IsTypeAssignableFrom<IEnumerable>(m.Arguments[0].Type))
+                    {
+                        goto case "SqlIn";
+                    }
+
                    string compareValue;

                    if (methodArgs[0].NodeType != ExpressionType.Constant)
@@ -597,13 +609,6 @@ namespace Umbraco.Core.Persistence.Querying
                        compareValue = methodArgs[0].ToString();
                    }

-                    //special case, if it is 'Contains' and the member that Contains is being called on is not a string, then
-                    // we should be doing an 'In' clause - but we currently do not support this
-                    if (methodArgs[0].Type != typeof(string) && TypeHelper.IsTypeAssignableFrom<IEnumerable>(methodArgs[0].Type))
-                    {
-                        throw new NotSupportedException("An array Contains method is not supported");
-                    }
-
                    //default column type
                    var colType = TextColumnType.NVarchar;

@@ -705,29 +710,33 @@ namespace Umbraco.Core.Persistence.Querying
                //    }
                //    return string.Format("{0}{1}", r, s);

-                //case "In":
+                case "SqlIn":

-                //    var member = Expression.Convert(m.Arguments[0], typeof(object));
-                //    var lambda = Expression.Lambda<Func<object>>(member);
-                //    var getter = lambda.Compile();
+                    if (m.Object == null && methodArgs.Length == 1 && methodArgs[0].NodeType == ExpressionType.MemberAccess)
+                    {
+                        var memberAccess = VisitMemberAccess((MemberExpression) methodArgs[0]);
+                        
+                        var member = Expression.Convert(m.Arguments[0], typeof(object));
+                        var lambda = Expression.Lambda<Func<object>>(member);
+                        var getter = lambda.Compile();

-                //    var inArgs = (object[])getter();
+                        var inArgs = (IEnumerable)getter();

-                //    var sIn = new StringBuilder();
-                //    foreach (var e in inArgs)
-                //    {
-                //        SqlParameters.Add(e);
+                        var sIn = new StringBuilder();
+                        foreach (var e in inArgs)
+                        {
+                            SqlParameters.Add(e);

-                //        sIn.AppendFormat("{0}{1}",
-                //                     sIn.Length > 0 ? "," : "",
-                //                                    string.Format("@{0}", SqlParameters.Count - 1));
+                            sIn.AppendFormat("{0}{1}",
+                                sIn.Length > 0 ? "," : "",
+                                string.Format("@{0}", SqlParameters.Count - 1));
+                        }

-                //        //sIn.AppendFormat("{0}{1}",
-                //        //             sIn.Length > 0 ? "," : "",
-                //        //                            GetQuotedValue(e, e.GetType()));
-                //    }
+                        return string.Format("{0} IN ({1})", memberAccess, sIn);
+                    }
+
+                    throw new NotSupportedException("SqlIn must contain the member being accessed");

-                //    return string.Format("{0} {1} ({2})", r, m.Method.Name, sIn.ToString());
                //case "Desc":
                //    return string.Format("{0} DESC", r);
                //case "Alias":
@@ -1,5 +1,6 @@
 using System;
 using System.Linq.Expressions;
+using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Mappers;
 using Umbraco.Core.Persistence.SqlSyntax;

@@ -12,16 +13,19 @@ namespace Umbraco.Core.Persistence.Querying
    /// <remarks>This object is stateful and cannot be re-used to parse an expression.</remarks>
    internal class ModelToSqlExpressionVisitor<T> : ExpressionVisitorBase
    {
+        private readonly MappingResolver _mappingResolver;
        private readonly BaseMapper _mapper;

-        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, BaseMapper mapper)
+        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, MappingResolver mappingResolver)
            : base(sqlSyntax)
        {
-            _mapper = mapper;
+            _mapper = mappingResolver.ResolveMapperByType(typeof(T));
+            _mappingResolver = mappingResolver;
        }

+        [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
        public ModelToSqlExpressionVisitor()
-            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T)))
+            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current)
        { }

        protected override string VisitMemberAccess(MemberExpression m)
@@ -35,7 +39,7 @@ namespace Umbraco.Core.Persistence.Querying
                {
                    var field = _mapper.Map(m.Member.Name, true);
                    if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                    return field;
                }
                //already compiled, return
@@ -49,13 +53,42 @@ namespace Umbraco.Core.Persistence.Querying
                {
                    var field = _mapper.Map(m.Member.Name, true);
                    if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                    return field;
                }
                //already compiled, return
                return string.Empty;
            }

+            if (m.Expression != null 
+                && m.Expression.Type != typeof(T) 
+                && TypeHelper.IsTypeAssignableFrom<IUmbracoEntity>(m.Expression.Type)
+                && EndsWithConstant(m) == false)
+            {
+                //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test";
+                //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column
+
+                //don't execute if compiled
+                if (Visited == false)
+                {
+                    var subMapper = _mappingResolver.ResolveMapperByType(m.Expression.Type);
+                    if (subMapper == null)
+                        throw new NullReferenceException("No mapper found for type " + m.Expression.Type);
+                    var field = subMapper.Map(m.Member.Name, true);
+                    if (field.IsNullOrWhiteSpace())
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
+                    return field;
+                }
+                //already compiled, return
+                return string.Empty;
+            }
+
+            //TODO: When m.Expression.NodeType == ExpressionType.Constant and it's an expression like: content => aliases.Contains(content.ContentType.Alias);
+            // then an SQL parameter will be added for aliases as an array, however in SqlIn on the subclass it will manually add these SqlParameters anyways,
+            // however the query will still execute because the SQL that is written will only contain the correct indexes of SQL parameters, this would be ignored,
+            // I'm just unsure right now due to time constraints how to make it correct. It won't matter right now and has been working already with this bug but I've
+            // only just discovered what it is actually doing.
+
            var member = Expression.Convert(m, typeof(object));
            var lambda = Expression.Lambda<Func<object>>(member);
            var getter = lambda.Compile();
@@ -70,5 +103,24 @@ namespace Umbraco.Core.Persistence.Querying
            return string.Empty;

        }
+
+        /// <summary>
+        /// Determines if the MemberExpression ends in a Constant value
+        /// </summary>
+        /// <param name="m"></param>
+        /// <returns></returns>
+        private bool EndsWithConstant(MemberExpression m)
+        {
+            Expression expr = m;
+            
+            while (expr is MemberExpression)
+            {
+                var memberExpr = expr as MemberExpression;
+                expr = memberExpr.Expression;
+            }
+            
+            var constExpr = expr as ConstantExpression;
+            return constExpr != null;
+        }
    }
 }
@@ -14,12 +14,19 @@ namespace Umbraco.Core.Persistence.Querying
    {
        private readonly Database.PocoData _pd;

-        public PocoToSqlExpressionVisitor()
-            : base(SqlSyntaxContext.SqlSyntaxProvider)
+
+        public PocoToSqlExpressionVisitor(ISqlSyntaxProvider syntaxProvider)
+            : base(syntaxProvider)
        {
            _pd = new Database.PocoData(typeof(T));
        }

+        [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
+        public PocoToSqlExpressionVisitor()
+            : this(SqlSyntaxContext.SqlSyntaxProvider)
+        {            
+        }
+
        protected override string VisitMemberAccess(MemberExpression m)
        {
            if (m.Expression != null &&
@@ -1,4 +1,5 @@
-using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;

 namespace Umbraco.Core.Persistence.Querying
@@ -6,8 +7,13 @@ namespace Umbraco.Core.Persistence.Querying
    /// <summary>
    /// String extension methods used specifically to translate into SQL
    /// </summary>
-    internal static class SqlStringExtensions
+    internal static class SqlExpressionExtensions
    {
+        public static bool SqlIn<T>(this IEnumerable<T> collection, T item)
+        {
+            return collection.Contains(item);
+        }
+
        public static bool SqlWildcard(this string str, string txt, TextColumnType columnType)
        {
            var wildcardmatch = new Regex("^" + Regex.Escape(txt).
@@ -0,0 +1,31 @@
+namespace Umbraco.Core.Persistence.Repositories
+{
+    internal enum BaseQueryType
+    {
+        /// <summary>
+        /// A query to return all information for a single item
+        /// </summary>
+        /// <remarks>
+        /// In some cases this will be the same as <see cref="FullMultiple"/>
+        /// </remarks>
+        FullSingle,
+
+        /// <summary>
+        /// A query to return all information for multiple items
+        /// </summary>
+        /// <remarks>
+        /// In some cases this will be the same as <see cref="FullSingle"/>
+        /// </remarks>
+        FullMultiple,
+
+        /// <summary>
+        /// A query to return the ids for items
+        /// </summary>
+        Ids,
+
+        /// <summary>
+        /// A query to return the count for items 
+        /// </summary>
+        Count
+    }
+}
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Globalization;
 using System.Linq;
 using System.Xml;
@@ -53,9 +54,9 @@ namespace Umbraco.Core.Persistence.Repositories

        protected override IContent PerformGet(int id)
        {
-            var sql = GetBaseQuery(false)
+            var sql = GetBaseQuery(BaseQueryType.FullSingle)
                .Where(GetBaseWhereClause(), new { Id = id })
-                .Where<DocumentDto>(x => x.Newest)
+                .Where<DocumentDto>(x => x.Newest, SqlSyntax)
                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);

            var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(SqlSyntax.SelectTop(sql, 1)).FirstOrDefault();
@@ -63,76 +64,112 @@ namespace Umbraco.Core.Persistence.Repositories
            if (dto == null)
                return null;

-            var content = CreateContentFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateContentFromDto(dto, sql);

            return content;
        }

        protected override IEnumerable<IContent> PerformGetAll(params int[] ids)
        {
-            var sql = GetBaseQuery(false);
-            if (ids.Any())
+            Func<Sql, Sql> translate = s =>
            {
-                sql.Where("umbracoNode.id in (@ids)", new { ids });
-            }
+                if (ids.Any())
+                {
+                    s.Where("umbracoNode.id in (@ids)", new { ids });
+                }
+                //we only want the newest ones with this method
+                s.Where<DocumentDto>(x => x.Newest, SqlSyntax);
+                return s;
+            };
+            
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);

-            //we only want the newest ones with this method
-            sql.Where<DocumentDto>(x => x.Newest);
-
-            return ProcessQuery(sql);
+            return ProcessQuery(translate(sqlBaseFull), new PagingSqlQuery(translate(sqlBaseIds)));
        }

        protected override IEnumerable<IContent> PerformGetByQuery(IQuery<IContent> query)
        {
-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Newest)
-                                .OrderByDescending<ContentVersionDto>(x => x.VersionDate)
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);

-            return ProcessQuery(sql);
+            Func<SqlTranslator<IContent>, Sql> translate = (translator) =>
+            {
+                return translator.Translate()
+                    .Where<DocumentDto>(x => x.Newest, SqlSyntax)
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
+            var translatorFull = new SqlTranslator<IContent>(sqlBaseFull, query);
+            var translatorIds = new SqlTranslator<IContent>(sqlBaseIds, query);
+
+            return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds)));
        }

-        #endregion
-
-        #region Static Queries
-
-        private readonly IQuery<IContent> _publishedQuery = Query<IContent>.Builder.Where(x => x.Published == true);
-
-        #endregion
+        #endregion        

        #region Overrides of PetaPocoRepositoryBase<IContent>

-
-        protected override Sql GetBaseQuery(bool isCount)
+        /// <summary>
+        /// Returns the base query to return Content
+        /// </summary>
+        /// <param name="queryType"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// Content queries will differ depending on what needs to be returned:
+        /// * FullSingle: When querying for a single document, this will include the Outer join to fetch the content item's published version info
+        /// * FullMultiple: When querying for multiple documents, this will exclude the Outer join to fetch the content item's published version info - this info would need to be fetched separately
+        /// * Ids: This would essentially be the same as FullMultiple however the columns specified will only return the Ids for the documents
+        /// * Count: A query to return the count for documents
+        /// </remarks>
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
        {
-            var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsDocument.nodeId" : "*"))
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<DocumentDto, ContentVersionDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+            //TODO: IF we want to enable querying on content type information this will need to be joined
+            //.InnerJoin<ContentTypeDto>(SqlSyntax)
+            //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
+
+            if (queryType == BaseQueryType.FullSingle)
+            {
+                //The only reason we apply this left outer join is to be able to pull back the DocumentPublishedReadOnlyDto
+                //information with the entire data set, so basically this will get both the latest document and also it's published
+                //version if it has one. When performing a count or when retrieving Ids like in paging, this is unecessary 
+                //and causes huge performance overhead for the SQL server, especially when sorting the result. 
+                //We also don't include this outer join when querying for multiple entities since it is much faster to fetch this information
+                //in a separate query. For a single entity this is ok.
+
+                var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
                SqlSyntax.GetQuotedTableName("cmsDocument"),
                SqlSyntax.GetQuotedTableName("cmsDocument2"),
                SqlSyntax.GetQuotedColumnName("nodeId"),
                SqlSyntax.GetQuotedColumnName("published"));

-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<DocumentDto, ContentVersionDto>(left => left.VersionId, right => right.VersionId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-
                // cannot do this because PetaPoco does not know how to alias the table
                //.LeftOuterJoin<DocumentPublishedReadOnlyDto>()
                //.On<DocumentDto, DocumentPublishedReadOnlyDto>(left => left.NodeId, right => right.NodeId)
                // so have to rely on writing our own SQL
-                .Append(sqlx/*, new { @published = true }*/)
+                sql.Append(sqlx /*, new { @published = true }*/);
+            }
+
+            sql.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);

-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
            return sql;
        }

+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
+        }
+
        protected override string GetBaseWhereClause()
        {
            return "umbracoNode.id = @Id";
@@ -179,20 +216,32 @@ namespace Umbraco.Core.Persistence.Repositories
            // not bring much safety - so this reverts to updating each record individually,
            // and it may be slower in the end, but should be more resilient.

-            var baseId = 0;
            var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
+
+            Func<int, Sql, Sql> translate = (bId, sql) =>
+            {
+                if (contentTypeIdsA.Length > 0)
+                {
+                    sql.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
+
+                sql
+                    .Where<NodeDto>(x => x.NodeId > bId && x.Trashed == false, SqlSyntax)
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
+
+                return sql;
+            };
+
+            var baseId = 0;
+            
            while (true)
            {
                // get the next group of nodes
-                var query = GetBaseQuery(false);
-                if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
-                query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId && x.Trashed == false)
-                    .Where<DocumentDto>(x => x.Published)
-                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
-                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
+                var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.FullMultiple));
+                var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids));
+
+                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), new PagingSqlQuery(SqlSyntax.SelectTop(sqlIds, groupSize)))
                    .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
                    .ToList();

@@ -212,30 +261,70 @@ namespace Umbraco.Core.Persistence.Repositories
                        Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
                    }
                }
-                baseId = xmlItems.Last().NodeId;
+                baseId = xmlItems[xmlItems.Count - 1].NodeId;
            }
+
+            //now delete the items that shouldn't be there
+            var sqlAllIds = translate(0, GetBaseQuery(BaseQueryType.Ids));
+            var allContentIds = Database.Fetch<int>(sqlAllIds);
+            var docObjectType = Guid.Parse(Constants.ObjectTypes.Document);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allContentIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }                
+            }
+                
        }

        public override IEnumerable<IContent> GetAllVersions(int id)
        {
-            var sql = GetBaseQuery(false)
-                .Where(GetBaseWhereClause(), new { Id = id })
-                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
-            return ProcessQuery(sql, true);
+            Func<Sql, Sql> translate = s =>
+            {
+                return s.Where(GetBaseWhereClause(), new {Id = id})
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
+            };
+
+            var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple));
+            var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
+            
+            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true,  includeAllVersions:true);
        }

        public override IContent GetByVersion(Guid versionId)
        {
-            var sql = GetBaseQuery(false);
+            var sql = GetBaseQuery(BaseQueryType.FullSingle);
+            //TODO: cmsContentVersion.VersionId has a Unique Index constraint applied, seems silly then to also add OrderByDescending since it would be impossible to return more than one.
            sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);

            var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql).FirstOrDefault();

            if (dto == null)
                return null;

-            var content = CreateContentFromDto(dto, versionId, sql);
+            var content = CreateContentFromDto(dto, sql);

            return content;
        }
@@ -244,10 +333,11 @@ namespace Umbraco.Core.Persistence.Repositories
        {
            var sql = new Sql()
                .Select("*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
-                .Where<ContentVersionDto>(x => x.VersionId == versionId)
-                .Where<DocumentDto>(x => x.Newest != true);
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .Where<ContentVersionDto>(x => x.VersionId == versionId, SqlSyntax)
+                .Where<DocumentDto>(x => x.Newest != true, SqlSyntax);
            var dto = Database.Fetch<DocumentDto, ContentVersionDto>(sql).FirstOrDefault();

            if (dto == null) return;
@@ -265,7 +355,8 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = new Sql()
                .Select("*")
                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentVersionDto>()
+                .On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
                .Where<ContentVersionDto>(x => x.NodeId == id)
                .Where<ContentVersionDto>(x => x.VersionDate < versionDate)
                .Where<DocumentDto>(x => x.Newest != true);
@@ -348,6 +439,7 @@ namespace Umbraco.Core.Persistence.Repositories

            //Update with new correct path
            nodeDto.Path = string.Concat(parent.Path, ",", nodeDto.NodeId);
+            nodeDto.ValidatePathWithException();
            Database.Update(nodeDto);

            //Update entity with correct values
@@ -421,11 +513,13 @@ namespace Umbraco.Core.Persistence.Repositories
                dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                {
                    VersionId = dto.VersionId,
+                    VersionDate = dto.UpdateDate,
                    Newest = true,
                    NodeId = dto.NodeId,
-                    Published = true
+                    Published = true               
                };
-                ((Content)entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedDate = dto.UpdateDate;
            }

            entity.ResetDirtyProperties();
@@ -485,6 +579,7 @@ namespace Umbraco.Core.Persistence.Repositories

            //Updates the (base) node data - umbracoNode
            var nodeDto = dto.ContentVersionDto.ContentDto.NodeDto;
+            nodeDto.ValidatePathWithException();
            var o = Database.Update(nodeDto);

            //Only update this DTO if the contentType has actually changed
@@ -503,6 +598,7 @@ namespace Umbraco.Core.Persistence.Repositories
            //if (((ICanBeDirty)entity).IsPropertyDirty("Published") && (entity.Published || publishedState == PublishedState.Unpublished))
            if (entity.ShouldClearPublishedFlagForPreviousVersions(publishedState, shouldCreateNewVersion))
            {
+                //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
                var publishedDocs = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND published = @IsPublished", new { Id = entity.Id, IsPublished = true });
                foreach (var doc in publishedDocs)
                {
@@ -516,6 +612,7 @@ namespace Umbraco.Core.Persistence.Repositories
            }

            //Look up (newest) entries by id in cmsDocument table to set newest = false
+            //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
            var documentDtos = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND newest = @IsNewest", new { Id = entity.Id, IsNewest = true });
            foreach (var documentDto in documentDtos)
            {
@@ -592,22 +689,26 @@ namespace Umbraco.Core.Persistence.Repositories
                dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                {
                    VersionId = dto.VersionId,
+                    VersionDate = dto.UpdateDate,
                    Newest = true,
                    NodeId = dto.NodeId,
                    Published = true
                };
-                ((Content)entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedDate = dto.UpdateDate;
            }
            else if (publishedStateChanged)
            {
                dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                {
-                    VersionId = default(Guid),
+                    VersionId = default (Guid),
+                    VersionDate = default (DateTime),
                    Newest = false,
                    NodeId = dto.NodeId,
                    Published = false
                };
-                ((Content)entity).PublishedVersionGuid = default(Guid);
+                ((Content) entity).PublishedVersionGuid = default(Guid);
+                ((Content) entity).PublishedDate = default (DateTime);
            }

            entity.ResetDirtyProperties();
@@ -620,21 +721,27 @@ namespace Umbraco.Core.Persistence.Repositories

        public IEnumerable<IContent> GetByPublishedVersion(IQuery<IContent> query)
        {
+            Func<SqlTranslator<IContent>, Sql> translate = t =>
+            {
+                return t.Translate()
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
            // we WANT to return contents in top-down order, ie parents should come before children
            // ideal would be pure xml "document order" which can be achieved with:
            // ORDER BY substring(path, 1, len(path) - charindex(',', reverse(path))), sortOrder
            // but that's probably an overkill - sorting by level,sortOrder should be enough

-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Published)
-                                .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
-                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            var sqlFull = GetBaseQuery(BaseQueryType.FullMultiple);
+            var translatorFull = new SqlTranslator<IContent>(sqlFull, query);
+            var sqlIds = GetBaseQuery(BaseQueryType.Ids);
+            var translatorIds = new SqlTranslator<IContent>(sqlIds, query);

-            return ProcessQuery(sql, true);
+            return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds)), true);
        }
-
+        
        /// <summary>
        /// This builds the Xml document used for the XML cache
        /// </summary>
@@ -666,10 +773,14 @@ namespace Umbraco.Core.Persistence.Repositories
            parent.Attributes.Append(pIdAtt);
            xmlDoc.AppendChild(parent);

-            const string sql = @"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.xml, umbracoNode.level from umbracoNode
+            //Ensure that only nodes that have published versions are selected
+            var sql = string.Format(@"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.{0}, umbracoNode.{1} from umbracoNode
 inner join cmsContentXml on cmsContentXml.nodeId = umbracoNode.id and umbracoNode.nodeObjectType = @type
 where umbracoNode.id in (select cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1)
-order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
+order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
+                SqlSyntax.GetQuotedColumnName("xml"),
+                SqlSyntax.GetQuotedColumnName("level"),
+                SqlSyntax.GetQuotedColumnName("level"));

            XmlElement last = null;

@@ -720,13 +831,8 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";

        public void ClearPublished(IContent content)
        {
-            // race cond!
-            var documentDtos = Database.Fetch<DocumentDto>("WHERE nodeId=@id AND published=@published", new { id = content.Id, published = true });
-            foreach (var documentDto in documentDtos)
-            {
-                documentDto.Published = false;
-                Database.Update(documentDto);
-            }
+            var sql = "UPDATE cmsDocument SET published=0 WHERE nodeId=@id AND published=1";
+            Database.Execute(sql, new {id = content.Id});
        }

        /// <summary>
@@ -806,9 +912,9 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
            
            Func<Tuple<string, object[]>> filterCallback = () => new Tuple<string, object[]>(filterSql.SQL, filterSql.Arguments);

-            return GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<DocumentDto>(query, pageIndex, pageSize, out totalRecords,
                new Tuple<string, string>("cmsDocument", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField,
                filterCallback);

        }
@@ -839,49 +945,110 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
            return base.GetDatabaseFieldNameForOrderBy(orderBy);
        }

-        private IEnumerable<IContent> ProcessQuery(Sql sql, bool withCache = false)
+        /// <summary>
+        /// This is the underlying method that processes most queries for this repository
+        /// </summary>
+        /// <param name="sqlFull">
+        /// The FullMultiple SQL without the outer join to return all data required to create an IContent excluding it's published state data which this will query separately
+        /// </param>
+        /// <param name="pagingSqlQuery">
+        /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
+        /// </param>
+        /// <param name="withCache"></param>
+        /// <param name="includeAllVersions">
+        /// Generally when querying for content we only want to return the most recent version of the content item, however in some cases like when 
+        /// we want to return all versions of a content item, we can't simply return the latest
+        /// </param>
+        /// <returns></returns>
+        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false, bool includeAllVersions = false)
        {
            // fetch returns a list so it's ok to iterate it in this method
-            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql);
+            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
            if (dtos.Count == 0) return Enumerable.Empty<IContent>();
+            
+            //Go and get all of the published version data separately for this data, this is because when we are querying
+            //for multiple content items we don't include the outer join to fetch this data in the same query because
+            //it is insanely slow. Instead we just fetch the published version data separately in one query.

-            var content = new IContent[dtos.Count];
-            var defs = new List<DocumentDefinition>();
-            var templateIds = new List<int>();
-
-            for (var i = 0; i < dtos.Count; i++)
+            //we need to parse the original SQL statement and reduce the columns to just cmsDocument.nodeId so that we can use
+            // the statement to go get the published data for all of the items by using an inner join
+            var parsedOriginalSql = "SELECT cmsDocument.nodeId " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
+            //now remove everything from an Orderby clause and beyond
+            if (parsedOriginalSql.InvariantContains("ORDER BY "))
            {
-                var dto = dtos[i];
+                parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
+            }            
+
+            //order by update date DESC, if there is corrupted published flags we only want the latest!
+            var publishedSql = new Sql(@"SELECT cmsDocument.nodeId, cmsDocument.published, cmsDocument.versionId, cmsDocument.updateDate, cmsDocument.newest
+FROM cmsDocument INNER JOIN cmsContentVersion ON cmsContentVersion.VersionId = cmsDocument.versionId
+WHERE cmsDocument.published = 1 AND cmsDocument.nodeId IN 
+(" + parsedOriginalSql + @")
+ORDER BY cmsContentVersion.id DESC
+", sqlFull.Arguments);
+
+            //go and get the published version data, we do a Query here and not a Fetch so we are
+            //not allocating a whole list to memory just to allocate another list in memory since
+            //we are assigning this data to a keyed collection for fast lookup below
+            var publishedData = Database.Query<DocumentPublishedReadOnlyDto>(publishedSql);
+            var publishedDataCollection = new DocumentPublishedReadOnlyDtoCollection();
+            foreach (var publishedDto in publishedData)
+            {
+                //double check that there's no corrupt db data, there should only be a single published item
+                if (publishedDataCollection.Contains(publishedDto.NodeId) == false)
+                    publishedDataCollection.Add(publishedDto);
+            }
+
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IContent, bool>>();
+            var defs = new DocumentDefinitionCollection(includeAllVersions);
+            var templateIds = new List<int>();
+            
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IContentType>();
+            
+            foreach (var dto in dtos)
+            {
+                DocumentPublishedReadOnlyDto publishedDto;
+                publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto);

                // if the cache contains the published version, use it
                if (withCache)
                {
                    var cached = RuntimeCache.GetCacheItem<IContent>(GetCacheIdKey<IContent>(dto.NodeId));
-                    if (cached != null && cached.Published)
+                    //only use this cached version if the dto returned is also the publish version, they must match and be teh same version
+                    if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published)
                    {
-                        content[i] = cached;
+                        content.Add(new Tuple<IContent, bool>(cached, true));
                        continue;
                    }
                }

                // else, need to fetch from the database
                // content type repository is full-cache so OK to get each one independently
-                var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);

-                // need template
-                if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
-                    templateIds.Add(dto.TemplateId.Value);
+                IContentType contentType;
+                if (contentTypes.ContainsKey(dto.ContentVersionDto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
+                }

-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    // assign template
+                    if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
+                        templateIds.Add(dto.TemplateId.Value);
+
+                    content.Add(new Tuple<IContent, bool>(ContentFactory.BuildEntity(dto, contentType, publishedDto), false));
+                }
            }

            // load all required templates in 1 query
@@ -889,45 +1056,44 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
                .ToDictionary(x => x.Id, x => x);

            // load all properties for all documents from database in 1 query
-            var propertyData = GetPropertyCollection(sql, defs);
+            var propertyData = GetPropertyCollection(pagingSqlQuery, defs);

-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign template and property data
+            foreach (var contentItem in content)
            {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id];

-                // complete the item
-                var cc = content[dtoIndex];
-                var dto = dtos[dtoIndex];
                ITemplate template = null;
-                if (dto.TemplateId.HasValue)
-                    templates.TryGetValue(dto.TemplateId.Value, out template); // else null
+                if (def.DocumentDto.TemplateId.HasValue)
+                    templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null
                cc.Template = template;
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];

                //on initial construction we don't want to have dirty properties tracked
                // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
            }

-            return content;
+            return content.Select(x => x.Item1).ToArray();
        }

        /// <summary>
        /// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion.
        /// </summary>
        /// <param name="dto"></param>
-        /// <param name="versionId"></param>
        /// <param name="docSql"></param>
        /// <returns></returns>
-        private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql)
+        private IContent CreateContentFromDto(DocumentDto dto, Sql docSql)
        {
            var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);

-            var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var content = factory.BuildEntity(dto);
+            var content = ContentFactory.BuildEntity(dto, contentType);

            //Check if template id is set on DocumentDto, and get ITemplate if it is.
            if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
@@ -935,11 +1101,11 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
                content.Template = _templateRepository.Get(dto.TemplateId.Value);
            }

-            var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType);
+            var docDef = new DocumentDefinition(dto, contentType);

            var properties = GetPropertyCollection(docSql, new[] { docDef });

-            content.Properties = properties[dto.NodeId];
+            content.Properties = properties[dto.VersionId];

            //on initial construction we don't want to have dirty properties tracked
            // http://issues.umbraco.org/issue/U4-1946
@@ -978,7 +1144,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";

            return currentName;
        }
-
+        
        /// <summary>
        /// Dispose disposable properties
        /// </summary>
@@ -993,5 +1159,26 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
            _contentPreviewRepository.Dispose();
            _contentXmlRepository.Dispose();
        }
+
+        /// <summary>
+        /// A keyed collection for fast lookup when retrieving a separate list of published version data
+        /// </summary>
+        private class DocumentPublishedReadOnlyDtoCollection : KeyedCollection<int, DocumentPublishedReadOnlyDto>
+        {
+            protected override int GetKeyForItem(DocumentPublishedReadOnlyDto item)
+            {
+                return item.NodeId;
+            }
+
+            public bool TryGetValue(int key, out DocumentPublishedReadOnlyDto val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
    }
 }
@@ -1250,5 +1250,19 @@ WHERE cmsContentType." + aliasColumn + @" LIKE @pattern",
            while (aliases.Contains(test = alias + i)) i++;
            return test;
        }
+
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        public bool HasContainerInPath(string contentPath)
+        {
+            var ids = contentPath.Split(',').Select(int.Parse);
+            var sql = new Sql(@"SELECT COUNT(*) FROM cmsContentType
+INNER JOIN cmsContent ON cmsContentType.nodeId=cmsContent.contentType 
+WHERE cmsContent.nodeId IN (@ids) AND cmsContentType.isContainer=@isContainer", new { ids, isContainer = true });
+            return Database.ExecuteScalar<int>(sql) > 0;
+        }
    }
 }
@@ -2,19 +2,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using Umbraco.Core.Cache;
-using Umbraco.Core.Events;
-using Umbraco.Core.Exceptions;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
-
-using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Persistence.Relators;
 using Umbraco.Core.Persistence.SqlSyntax;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Services;

 namespace Umbraco.Core.Persistence.Repositories
 {
@@ -118,7 +112,7 @@ namespace Umbraco.Core.Persistence.Repositories

            if (objectTypes.Any())
            {
-                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", objectTypes);
+                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", new {objectTypes = objectTypes});
            }

            return Database.Fetch<string>(sql);
@@ -34,7 +34,7 @@ namespace Umbraco.Core.Persistence.Repositories
            : base(work, cache, logger, sqlSyntax)
        {
            _contentTypeRepository = contentTypeRepository;
-            _preValRepository = new DataTypePreValueRepository(work, CacheHelper.CreateDisabledCacheHelper(), logger, sqlSyntax);            
+            _preValRepository = new DataTypePreValueRepository(work, CacheHelper.CreateDisabledCacheHelper(), logger, sqlSyntax);
        }

        #region Overrides of RepositoryBase<int,DataTypeDefinition>
@@ -276,47 +276,33 @@ AND umbracoNode.id <> @id",

        public PreValueCollection GetPreValuesCollectionByDataTypeId(int dataTypeId)
        {
-            var cached = RuntimeCache.GetCacheItemsByKeySearch<PreValueCollection>(GetPrefixedCacheKey(dataTypeId));
-            if (cached != null && cached.Any())
-            {
-                //return from the cache, ensure it's a cloned result
-                return (PreValueCollection)cached.First().DeepClone();
-            }
-
-            return GetAndCachePreValueCollection(dataTypeId);
-        }
-
-        internal static string GetCacheKeyRegex(int preValueId)
-        {
-            return CacheKeys.DataTypePreValuesCacheKey + @"[-\d]+-([\d]*,)*" + preValueId + @"(?!\d)[,\d$]*";
+            var collection = GetCachedPreValueCollection(dataTypeId);
+            return collection;
        }

+        /// <summary>
+        /// Gets a specific PreValue by its Id
+        /// </summary>
+        /// <param name="preValueId">Id of the PreValue to retrieve the value from</param>
+        /// <returns>PreValue as a string</returns>
        public string GetPreValueAsString(int preValueId)
        {
-            //We need to see if we can find the cached PreValueCollection based on the cache key above
+            var collections = RuntimeCache.GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + "_");

-            var cached = RuntimeCache.GetCacheItemsByKeyExpression<PreValueCollection>(GetCacheKeyRegex(preValueId));
-            if (cached != null && cached.Any())
-            {
-                //return from the cache
-                var collection = cached.First();
-				var preVal = collection.FormatAsDictionary().Single(x => x.Value.Id == preValueId);
-				return preVal.Value.Value;
-            }
+            var preValue = collections.SelectMany(x => x.FormatAsDictionary().Values).FirstOrDefault(x => x.Id == preValueId);
+            if (preValue != null)
+                return preValue.Value;

-            //go and find the data type id for the pre val id passed in
-
-            var dto = Database.FirstOrDefault<DataTypePreValueDto>("WHERE id = @preValueId", new { preValueId = preValueId });
+            var dto = Database.FirstOrDefault<DataTypePreValueDto>("WHERE id = @preValueId", new { preValueId });
            if (dto == null)
-            {
                return string.Empty;
-            }
-            // go cache the collection
-            var preVals = GetAndCachePreValueCollection(dto.DataTypeNodeId);

-            //return the single value for this id
-            var pv = preVals.FormatAsDictionary().Single(x => x.Value.Id == preValueId);
-            return pv.Value.Value;
+            var collection = GetCachedPreValueCollection(dto.DataTypeNodeId);
+            if (collection == null)
+                return string.Empty;
+
+            preValue = collection.FormatAsDictionary().Values.FirstOrDefault(x => x.Id == preValueId);
+            return preValue == null ? string.Empty : preValue.Value;
        }

        public void AddOrUpdatePreValues(int dataTypeId, IDictionary<string, PreValue> values)
@@ -441,40 +427,28 @@ AND umbracoNode.id <> @id",

                sortOrder++;
            }
-
        }

-        private string GetPrefixedCacheKey(int dataTypeId)
+        private static string GetPrefixedCacheKey(int dataTypeId)
        {
-            return CacheKeys.DataTypePreValuesCacheKey + dataTypeId + "-";
+            return CacheKeys.DataTypePreValuesCacheKey + "_" + dataTypeId;
        }

-        private PreValueCollection GetAndCachePreValueCollection(int dataTypeId)
+        private PreValueCollection GetCachedPreValueCollection(int datetypeId)
        {
-            //go get the data
-            var dtos = Database.Fetch<DataTypePreValueDto>("WHERE datatypeNodeId = @Id", new { Id = dataTypeId });
-            var list = dtos.Select(x => new Tuple<PreValue, string, int>(new PreValue(x.Id, x.Value, x.SortOrder), x.Alias, x.SortOrder)).ToList();
-            var collection = PreValueConverter.ConvertToPreValuesCollection(list);
-
-            //now create the cache key, this needs to include all pre-value ids so that we can use this cached item in the GetPreValuesAsString method
-            //the key will be: "UmbracoPreValDATATYPEID-CSVOFPREVALIDS
-
-            var key = GetPrefixedCacheKey(dataTypeId)
-                      + string.Join(",", collection.FormatAsDictionary().Select(x => x.Value.Id).ToArray());
-
-            //store into cache
-            RuntimeCache.InsertCacheItem(key, () => collection,
-                //30 mins
-                new TimeSpan(0, 0, 30),
-                //sliding is true
-                true);
-
-            return collection;
+            var key = GetPrefixedCacheKey(datetypeId);
+            return RuntimeCache.GetCacheItem<PreValueCollection>(key, () =>
+            {
+                var dtos = Database.Fetch<DataTypePreValueDto>("WHERE datatypeNodeId = @Id", new { Id = datetypeId });
+                var list = dtos.Select(x => new Tuple<PreValue, string, int>(new PreValue(x.Id, x.Value, x.SortOrder), x.Alias, x.SortOrder)).ToList();
+                var collection = PreValueConverter.ConvertToPreValuesCollection(list);
+                return collection;
+            }, TimeSpan.FromMinutes(20), isSliding: true);
        }

        private string EnsureUniqueNodeName(string nodeName, int id = 0)
        {
-         
+

            var sql = new Sql();
            sql.Select("*")
@@ -576,7 +550,7 @@ AND umbracoNode.id <> @id",
                }

                //NOTE: We used to check that the Alias was unique for the given DataTypeNodeId prevalues list, BUT
-                // in reality there is no need to check the uniqueness of this alias because the only way that this code executes is 
+                // in reality there is no need to check the uniqueness of this alias because the only way that this code executes is
                // based on an IDictionary<string, PreValue> dictionary being passed to this repository and a dictionary
                // must have unique aliases by definition, so there is no need for this additional check

@@ -596,10 +570,10 @@ AND umbracoNode.id <> @id",
                {
                    throw new InvalidOperationException("Cannot update a pre value for a data type that has no identity");
                }
-                
+
                //NOTE: We used to check that the Alias was unique for the given DataTypeNodeId prevalues list, BUT
                // this causes issues when sorting the pre-values (http://issues.umbraco.org/issue/U4-5670) but in reality
-                // there is no need to check the uniqueness of this alias because the only way that this code executes is 
+                // there is no need to check the uniqueness of this alias because the only way that this code executes is
                // based on an IDictionary<string, PreValue> dictionary being passed to this repository and a dictionary
                // must have unique aliases by definition, so there is no need for this additional check

@@ -614,7 +588,7 @@ AND umbracoNode.id <> @id",
                Database.Update(dto);
            }

-            
+
        }

        internal static class PreValueConverter
@@ -62,17 +62,27 @@ namespace Umbraco.Core.Persistence.Repositories

        protected override IEnumerable<EntityContainer> PerformGetAll(params int[] ids)
        {
-            //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
-            return ids.InGroupsOf(2000).SelectMany(@group =>
+            if (ids.Any())
+            {
+                //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
+                return ids.InGroupsOf(2000).SelectMany(@group =>
+                {
+                    var sql = GetBaseQuery(false)
+                        .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId})
+                        .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new {ids = @group});
+
+                    sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
+
+                    return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
+                });
+            }
+            else
            {
                var sql = GetBaseQuery(false)
-                    .Where("nodeObjectType=@umbracoObjectTypeId", new { umbracoObjectTypeId = NodeObjectTypeId })
-                    .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new { ids = @group });
-
+                    .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId});
                sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
-
                return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
-            });
+            }
        }

        protected override IEnumerable<EntityContainer> PerformGetByQuery(IQuery<EntityContainer> query)
@@ -1,18 +1,13 @@
 using System;
 using System.Collections.Generic;
-using System.Dynamic;
-using System.Globalization;
+using System.Collections.ObjectModel;
 using System.Linq;
-using System.Reflection;
-using System.Text;
 using Umbraco.Core.Models;
-using Umbraco.Core;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Strings;

 namespace Umbraco.Core.Persistence.Repositories
 {
@@ -68,11 +63,12 @@ namespace Umbraco.Core.Persistence.Repositories
            bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);

            var sql = GetFullSqlForEntityType(key, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
            if (isMedia)
            {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                    new UmbracoEntityRelator().Map, sql);

@@ -80,14 +76,16 @@ namespace Umbraco.Core.Persistence.Repositories
            }
            else
            {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;

-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;                
            }
            
            
@@ -112,29 +110,29 @@ namespace Umbraco.Core.Persistence.Repositories
            bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);

            var sql = GetFullSqlForEntityType(id, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
            if (isMedia)
            {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                    new UmbracoEntityRelator().Map, sql);

                return entities.FirstOrDefault();
            }
            else
-            {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;
-
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+            {                
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;
            }
-
-            
        }

        public virtual IEnumerable<IUmbracoEntity> GetAll(Guid objectTypeId, params int[] ids)
@@ -171,22 +169,21 @@ namespace Umbraco.Core.Persistence.Repositories

            if (isMedia)
            {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too                
                var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                    new UmbracoEntityRelator().Map, sql);
-                foreach (var entity in entities)
-                {
-                    yield return entity;
-                }
+                return entities;
            }
            else
            {
-                var dtos = _work.Database.Fetch<dynamic>(sql);
-                foreach (var entity in dtos.Select(dto => factory.BuildEntityFromDynamic(dto)))
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
                {
-                    yield return entity;
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
                }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();                
            }
        }

@@ -231,8 +228,7 @@ namespace Umbraco.Core.Persistence.Repositories
                    }
                });

-                //treat media differently for now 
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too    
                var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                    new UmbracoEntityRelator().Map, mediaSql);
                return entities;
@@ -241,8 +237,15 @@ namespace Umbraco.Core.Persistence.Repositories
            {
                //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData
                var finalSql = entitySql.Append(GetGroupBy(isContent, false));
-                var dtos = _work.Database.Fetch<dynamic>(finalSql);
-                return dtos.Select(factory.BuildEntityFromDynamic).Cast<IUmbracoEntity>().ToList();
+
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(finalSql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();
            }
        }

@@ -315,25 +318,31 @@ namespace Umbraco.Core.Persistence.Repositories
        protected virtual Sql GetBase(bool isContent, bool isMedia, Action<Sql> customFilter)
        {
            var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate",
-                                  "COUNT(parent.parentID) as children"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate",
+                "COUNT(parent.parentID) as children"
+            };

            if (isContent || isMedia)
            {
-                columns.Add("published.versionId as publishedVersion");
-                columns.Add("latest.versionId as newestVersion");
+                if (isContent)
+                {
+                    //only content has/needs this info
+                    columns.Add("published.versionId as publishedVersion");
+                    columns.Add("document.versionId as newestVersion");
+                    columns.Add("contentversion.id as versionId");
+                }
+                
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
@@ -345,17 +354,22 @@ namespace Umbraco.Core.Persistence.Repositories
            var entitySql = new Sql()
                .Select(columns.ToArray())
                .From("umbracoNode umbracoNode");
-                
+
            if (isContent || isMedia)
            {
-                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id")
-                   .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published")
-                   .On("umbracoNode.id = published.nodeId")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest")
-                   .On("umbracoNode.id = latest.nodeId");
+                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id");
+
+                if (isContent)
+                {
+                    //only content has/needs this info
+                    entitySql                        
+                        .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id")
+                        .InnerJoin("cmsContentVersion contentversion").On("contentversion.VersionId = document.versionId")
+                        .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published")
+                        .On("umbracoNode.id = published.nodeId");
+                }
+
+                entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType");
            }

            entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id");
@@ -372,22 +386,42 @@ namespace Umbraco.Core.Persistence.Repositories
        {
            var sql = baseQuery(isContent, isMedia, filter)
                .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
            return sql;
        }

        protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, int id)
        {
            var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.id = @Id", new { Id = id })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.id = @Id", new { Id = id });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
            return sql;
        }

        protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, Guid key)
        {
            var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
            return sql;
        }

@@ -396,6 +430,12 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = baseQuery(isContent, isMedia, null)
                .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType",
                       new {Id = id, NodeObjectType = nodeObjectType});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
            return sql;
        }

@@ -404,36 +444,46 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = baseQuery(isContent, isMedia, null)
                .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType",
                       new { UniqueID = key, NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
            return sql;
        }

        protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
        {
            var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate"
+            };

            if (isContent || isMedia)
            {
-                columns.Add("published.versionId");
-                columns.Add("latest.versionId");
+                if (isContent)
+                {
+                    columns.Add("published.versionId");
+                    columns.Add("document.versionId");
+                    columns.Add("contentversion.id");
+                }
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
-                columns.Add("contenttype.isContainer");
+                columns.Add("contenttype.isContainer");                
            }
-            
+
            var sql = new Sql()
                .GroupBy(columns.ToArray());

@@ -444,7 +494,7 @@ namespace Umbraco.Core.Persistence.Repositories

            return sql;
        }
-        
+
        #endregion

        /// <summary>
@@ -458,33 +508,7 @@ namespace Umbraco.Core.Persistence.Repositories
            UnitOfWork.DisposeIfDisposable();
        }

-        #region umbracoNode POCO - Extends NodeDto
-        [TableName("umbracoNode")]
-        [PrimaryKey("id")]
-        [ExplicitColumns]
-        internal class UmbracoEntityDto : NodeDto
-        {
-            [Column("children")]
-            public int Children { get; set; }
-
-            [Column("publishedVersion")]
-            public Guid PublishedVersion { get; set; }
-
-            [Column("newestVersion")]
-            public Guid NewestVersion { get; set; }
-
-            [Column("alias")]
-            public string Alias { get; set; }
-
-            [Column("icon")]
-            public string Icon { get; set; }
-
-            [Column("thumbnail")]
-            public string Thumbnail { get; set; }
-
-            [ResultColumn]
-            public List<UmbracoPropertyDto> UmbracoPropertyDtos { get; set; }
-        }
+        #region private classes
        
        [ExplicitColumns]
        internal class UmbracoPropertyDto
@@ -568,6 +592,99 @@ namespace Umbraco.Core.Persistence.Repositories
                return prev;
            }
        }
+
+        private class EntityDefinitionCollection : KeyedCollection<int, EntityDefinition>
+        {
+            protected override int GetKeyForItem(EntityDefinition item)
+            {
+                return item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(EntityDefinition item)
+            {
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                EntityDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Id);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+
+                base.Add(item);
+                return true;
+            }
+
+            private bool TryGetValue(int key, out EntityDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        private class EntityDefinition
+        {
+            private readonly UmbracoEntityFactory _factory;
+            private readonly dynamic _entity;
+            private readonly bool _isContent;
+            private readonly bool _isMedia;
+
+            public EntityDefinition(UmbracoEntityFactory factory, dynamic entity, bool isContent, bool isMedia)
+            {
+                _factory = factory;
+                _entity = entity;
+                _isContent = isContent;
+                _isMedia = isMedia;
+            }
+
+            public IUmbracoEntity BuildFromDynamic()
+            {
+                return _factory.BuildEntityFromDynamic(_entity);
+            }
+
+            public int Id
+            {
+                get { return _entity.id; }
+            }
+
+            public int VersionId
+            {
+                get
+                {
+                    if (_isContent || _isMedia)
+                    {
+                        return _entity.versionId;
+                    }
+                    return _entity.id;
+                }
+            }
+        }
        #endregion
    }
 }
@@ -96,5 +96,6 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
        IEnumerable<IContent> GetPagedResultsByQuery(IQuery<IContent> query, long pageIndex, int pageSize, out long totalRecords,
            string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter = null);
+        
    }
 }
@@ -8,6 +8,13 @@ namespace Umbraco.Core.Persistence.Repositories
 {
    public interface IContentTypeRepository : IContentTypeCompositionRepository<IContentType>
    {
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        bool HasContainerInPath(string contentPath);
+
        /// <summary>
        /// Gets all entities of the specified <see cref="PropertyType"/> query
        /// </summary>
@@ -38,15 +38,6 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <returns>An Enumerable list of <see cref="IMedia"/> objects</returns>
        IEnumerable<IMedia> GetPagedResultsByQuery(IQuery<IMedia> query, long pageIndex, int pageSize, out long totalRecords,
            string orderBy, Direction orderDirection, bool orderBySystemField, string filter = "");
-
-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords);
+        
    }
 }
@@ -73,6 +73,6 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <param name="content"></param>
        /// <param name="xml"></param>
        void AddOrUpdatePreviewXml(IMember content, Func<IMember, XElement> xml);
-
+        
    }
 }
@@ -20,7 +20,7 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <param name="serializer">The serializer to convert TEntity to Xml</param>
        /// <param name="groupSize">Structures will be rebuilt in chunks of this size</param>
        /// <param name="contentTypeIds"></param>
-        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null);
+        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null);

        /// <summary>
        /// Get the total count of entities
@@ -67,5 +67,16 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <param name="id">Id of the <see cref="TEntity"/> object to delete versions from</param>
        /// <param name="versionDate">Latest version date</param>
        void DeleteVersions(int id, DateTime versionDate);
+
+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords);
    }
 }
@@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories
            if (dto == null)
                return null;

-            var content = CreateMediaFromDto(dto, dto.VersionId, sql);
+            var content = CreateMediaFromDto(dto, sql);

            return content;
        }
@@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories
                sql.Where("umbracoNode.id in (@ids)", new { ids = ids });
            }

-            return ProcessQuery(sql);
+            return ProcessQuery(sql, new PagingSqlQuery(sql));
        }

        protected override IEnumerable<IMedia> PerformGetByQuery(IQuery<IMedia> query)
@@ -76,26 +76,34 @@ namespace Umbraco.Core.Persistence.Repositories
            var sqlClause = GetBaseQuery(false);
            var translator = new SqlTranslator<IMedia>(sqlClause, query);
            var sql = translator.Translate()
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);

-            return ProcessQuery(sql);
+            return ProcessQuery(sql, new PagingSqlQuery(sql));
        }

        #endregion

        #region Overrides of PetaPocoRepositoryBase<int,IMedia>
+        
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
+        {
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsContentVersion.contentId" : "*"))
+                .From<ContentVersionDto>(SqlSyntax)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax)
+                //TODO: IF we want to enable querying on content type information this will need to be joined
+                //.InnerJoin<ContentTypeDto>(SqlSyntax)
+                //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
+            return sql;
+        }

        protected override Sql GetBaseQuery(bool isCount)
        {
-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<ContentVersionDto>()
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
-            return sql;
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
        }

        protected override string GetBaseWhereClause()
@@ -138,93 +146,106 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = GetBaseQuery(false)
                .Where(GetBaseWhereClause(), new { Id = id })
                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
-            return ProcessQuery(sql, true);
+            return ProcessQuery(sql, new PagingSqlQuery(sql), true);
        }

-        private IEnumerable<IMedia> ProcessQuery(Sql sql, bool withCache = false)
+        /// <summary>
+        /// This is the underlying method that processes most queries for this repository
+        /// </summary>
+        /// <param name="sqlFull">
+        /// The full SQL to select all media data 
+        /// </param>
+        /// <param name="pagingSqlQuery">
+        /// The Id SQL to just return all media ids - used to process the properties for the media item
+        /// </param>
+        /// <param name="withCache"></param>
+        /// <returns></returns>
+        private IEnumerable<IMedia> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
        {
            // fetch returns a list so it's ok to iterate it in this method
-            var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sql);
-            var content = new IMedia[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sqlFull);
+            
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMedia, bool>>();
+            var defs = new DocumentDefinitionCollection();

-            for (var i = 0; i < dtos.Count; i++)
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IMediaType>();
+
+            foreach (var dto in dtos)
            {
-                var dto = dtos[i];
-
                // if the cache contains the item, use it
                if (withCache)
                {
                    var cached = RuntimeCache.GetCacheItem<IMedia>(GetCacheIdKey<IMedia>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, media doesn't 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.VersionId)
                    {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMedia, bool>(cached, true));
                        continue;
                    }
                }

                // else, need to fetch from the database
                // content type repository is full-cache so OK to get each one independently
-                var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
-                var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);

-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.VersionDate,
-                    dto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                IMediaType contentType;
+                if (contentTypes.ContainsKey(dto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentDto.ContentTypeId] = contentType;
+                }
+
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    content.Add(new Tuple<IMedia, bool>(MediaFactory.BuildEntity(dto, contentType), false));
+                }
            }

            // load all properties for all documents from database in 1 query
-            var propertyData = GetPropertyCollection(sql, defs);
+            var propertyData = GetPropertyCollection(pagingSqlQuery, defs);

-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
            {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;

-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];

                //on initial construction we don't want to have dirty properties tracked
                // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
            }

-            return content;
+            return content.Select(x => x.Item1).ToArray();
        }

        public override IMedia GetByVersion(Guid versionId)
        {
            var sql = GetBaseQuery(false);
            sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);

            var dto = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sql).FirstOrDefault();

            if (dto == null)
                return null;

-            var mediaType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+            var content = CreateMediaFromDto(dto, sql);

-            var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
-
-            var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) });
-
-            media.Properties = properties[dto.NodeId];
-
-            //on initial construction we don't want to have dirty properties tracked
-            // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            return content;
        }

        public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
@@ -242,12 +263,15 @@ namespace Umbraco.Core.Persistence.Repositories
                // get the next group of nodes
                var query = GetBaseQuery(false);
                if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                {
+                    query = query.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
                query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId)
+                    .Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
+                    .Where<NodeDto>(x => x.Trashed == false, SqlSyntax)
                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
-                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
+                var sql = SqlSyntax.SelectTop(query, groupSize);
+                var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
                    .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
                    .ToList();

@@ -268,7 +292,38 @@ namespace Umbraco.Core.Persistence.Repositories
                        Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
                    }
                }
-                baseId = xmlItems.Last().NodeId;
+                baseId = xmlItems[xmlItems.Count - 1].NodeId;
+            }
+
+            //now delete the items that shouldn't be there
+            var allMediaIds = Database.Fetch<int>(GetBaseQuery(BaseQueryType.Ids).Where<NodeDto>(x => x.Trashed == false, SqlSyntax));
+            var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+                
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allMediaIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }
            }
        }

@@ -324,6 +379,7 @@ namespace Umbraco.Core.Persistence.Repositories

            //Update with new correct path
            nodeDto.Path = string.Concat(parent.Path, ",", nodeDto.NodeId);
+            nodeDto.ValidatePathWithException();
            Database.Update(nodeDto);

            //Update entity with correct values
@@ -397,6 +453,7 @@ namespace Umbraco.Core.Persistence.Repositories

            //Updates the (base) node data - umbracoNode
            var nodeDto = dto.ContentDto.NodeDto;
+            nodeDto.ValidatePathWithException();
            var o = Database.Update(nodeDto);

            //Only update this DTO if the contentType has actually changed
@@ -493,56 +550,30 @@ namespace Umbraco.Core.Persistence.Repositories
                filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
            }

-            return GetPagedResultsByQuery<ContentVersionDto, Models.Media>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<ContentVersionDto>(query, pageIndex, pageSize, out totalRecords,
                new Tuple<string, string>("cmsContentVersion", "contentId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField,
                filterCallback);

        }

-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        public IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords)
-        {
-            Sql query;
-            if (path == "-1")
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE nodeObjectType = @0)", Guid.Parse(Constants.ObjectTypes.Media)).OrderBy("nodeId");
-            }
-            else
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE path LIKE @0)", path.EnsureEndsWith(",%")).OrderBy("nodeId");
-            }
-            var pagedResult = Database.Page<ContentXmlDto>(pageIndex+1, pageSize, query);
-            totalRecords = pagedResult.TotalItems;
-            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
-        }
-
        /// <summary>
        /// Private method to create a media object from a ContentDto
        /// </summary>
-        /// <param name="d"></param>
-        /// <param name="versionId"></param>
+        /// <param name="dto"></param>
        /// <param name="docSql"></param>
        /// <returns></returns>
-        private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
+        private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql)
        {
            var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+            
+            var media = MediaFactory.BuildEntity(dto, contentType);

-            var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            var docDef = new DocumentDefinition(dto, contentType);

-            var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
+            var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });

-            var properties = GetPropertyCollection(docSql, new[] { docDef });
-
-            media.Properties = properties[dto.NodeId];
+            media.Properties = properties[dto.VersionId];

            //on initial construction we don't want to have dirty properties tracked
            // http://issues.umbraco.org/issue/U4-1946
@@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories
            if (dto == null)
                return null;

-            var content = CreateMemberFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateMemberFromDto(dto, sql);

            return content;

@@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories
                sql.Where("umbracoNode.id in (@ids)", new { ids = ids });
            }

-            return ProcessQuery(sql);
+            return ProcessQuery(sql, new PagingSqlQuery(sql));

        }

@@ -90,7 +90,7 @@ namespace Umbraco.Core.Persistence.Repositories
                baseQuery.Append(new Sql("WHERE umbracoNode.id IN (" + sql.SQL + ")", sql.Arguments))
                    .OrderBy<NodeDto>(x => x.SortOrder);

-                return ProcessQuery(baseQuery);
+                return ProcessQuery(baseQuery, new PagingSqlQuery(baseQuery));
            }
            else
            {
@@ -98,7 +98,7 @@ namespace Umbraco.Core.Persistence.Repositories
                var sql = translator.Translate()
                    .OrderBy<NodeDto>(x => x.SortOrder);

-                return ProcessQuery(sql);
+                return ProcessQuery(sql, new PagingSqlQuery(sql));
            }

        }
@@ -107,24 +107,29 @@ namespace Umbraco.Core.Persistence.Repositories

        #region Overrides of PetaPocoRepositoryBase<int,IMembershipUser>

-        protected override Sql GetBaseQuery(bool isCount)
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
        {
            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<MemberDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<ContentVersionDto, MemberDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsMember.nodeId" : "*"))
+                .From<MemberDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, MemberDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                //We're joining the type so we can do a query against the member type - not sure if this adds much overhead or not?
                // the execution plan says it doesn't so we'll go with that and in that case, it might be worth joining the content
                // types by default on the document and media repo's so we can query by content type there too.
-                .InnerJoin<ContentTypeDto>().On<ContentTypeDto, ContentDto>(left => left.NodeId, right => right.ContentTypeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
+                .InnerJoin<ContentTypeDto>(SqlSyntax)
+                .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
            return sql;
+        }

+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
        }

        protected override string GetBaseWhereClause()
@@ -380,8 +385,8 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = GetBaseQuery(false)
                .Where(GetBaseWhereClause(), new { Id = id })
                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
-            return ProcessQuery(sql, true);
-        }
+            return ProcessQuery(sql, new PagingSqlQuery(sql), true);
+        }        

        public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
        {
@@ -403,7 +408,8 @@ namespace Umbraco.Core.Persistence.Repositories
                query = query
                    .Where<NodeDto>(x => x.NodeId > baseId)
                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
-                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
+                var sql = SqlSyntax.SelectTop(query, groupSize);
+                var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
                    .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
                    .ToList();

@@ -442,16 +448,16 @@ namespace Umbraco.Core.Persistence.Repositories
            var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);

            var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            var member = factory.BuildEntity(dto);

-            var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
+            var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) });

-            media.Properties = properties[dto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];

            //on initial construction we don't want to have dirty properties tracked
            // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            ((Entity)member).ResetDirtyProperties(false);
+            return member;

        }

@@ -535,7 +541,7 @@ namespace Umbraco.Core.Persistence.Repositories
                .OrderByDescending<ContentVersionDto>(x => x.VersionDate)
                .OrderBy<NodeDto>(x => x.SortOrder);

-            return ProcessQuery(sql);
+            return ProcessQuery(sql, new PagingSqlQuery(sql));

        }

@@ -617,9 +623,9 @@ namespace Umbraco.Core.Persistence.Repositories
                filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
            }

-            return GetPagedResultsByQuery<MemberDto, Member>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<MemberDto>(query, pageIndex, pageSize, out totalRecords,
                new Tuple<string, string>("cmsMember", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField,
                filterCallback);
        }

@@ -659,25 +665,37 @@ namespace Umbraco.Core.Persistence.Repositories
            return base.GetEntityPropertyNameForOrderBy(orderBy);
        }

-        private IEnumerable<IMember> ProcessQuery(Sql sql, bool withCache = false)
+        /// <summary>
+        /// This is the underlying method that processes most queries for this repository
+        /// </summary>
+        /// <param name="sqlFull">
+        /// The full SQL to select all member data 
+        /// </param>
+        /// <param name="pagingSqlQuery">
+        /// The Id SQL to just return all member ids - used to process the properties for the member item
+        /// </param>
+        /// <param name="withCache"></param>
+        /// <returns></returns>
+        private IEnumerable<IMember> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
        {
            // fetch returns a list so it's ok to iterate it in this method
-            var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sql);
+            var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);

-            var content = new IMember[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMember, bool>>();
+            var defs = new DocumentDefinitionCollection();

-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
            {
-                var dto = dtos[i];
-
                // if the cache contains the item, use it
                if (withCache)
                {
                    var cached = RuntimeCache.GetCacheItem<IMember>(GetCacheIdKey<IMember>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, members dont 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.ContentVersionDto.VersionId)
                    {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMember, bool>(cached, true));
                        continue;
                    }
                }
@@ -685,60 +703,54 @@ namespace Umbraco.Core.Persistence.Repositories
                // else, need to fetch from the database
                // content type repository is full-cache so OK to get each one independently
                var contentType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new MemberFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);

                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.ContentVersionDto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType)))
+                {
+                    content.Add(new Tuple<IMember, bool>(MemberFactory.BuildEntity(dto, contentType), false));
+                }
            }

            // load all properties for all documents from database in 1 query
-            var propertyData = GetPropertyCollection(sql, defs);
+            var propertyData = GetPropertyCollection(pagingSqlQuery, defs);

-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
            {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;

-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];

                //on initial construction we don't want to have dirty properties tracked
                // http://issues.umbraco.org/issue/U4-1946
-                ((Entity)cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
            }

-            return content;
+            return content.Select(x => x.Item1).ToArray();
        }

        /// <summary>
        /// Private method to create a member object from a MemberDto
        /// </summary>
        /// <param name="dto"></param>
-        /// <param name="versionId"></param>
        /// <param name="docSql"></param>
        /// <returns></returns>
-        private IMember CreateMemberFromDto(MemberDto dto, Guid versionId, Sql docSql)
+        private IMember CreateMemberFromDto(MemberDto dto, Sql docSql)
        {
            var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);

            var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.ContentVersionDto.NodeId);
            var member = factory.BuildEntity(dto);

-            var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType);
+            var docDef = new DocumentDefinition(dto.ContentVersionDto, memberType);

            var properties = GetPropertyCollection(docSql, new[] { docDef });

-            member.Properties = properties[dto.ContentVersionDto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];

            //on initial construction we don't want to have dirty properties tracked
            // http://issues.umbraco.org/issue/U4-1946
@@ -44,10 +44,9 @@ namespace Umbraco.Core.Persistence.Repositories
            {
                sql.Where("umbracoAccess.id IN (@ids)", new { ids = ids });
            }
-
-            sql.OrderBy<AccessDto>(x => x.NodeId, SqlSyntax);
-
+            
            var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
            var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
            return dtos.Select(factory.BuildEntity);
        }
@@ -59,6 +58,7 @@ namespace Umbraco.Core.Persistence.Repositories
            var sql = translator.Translate();

            var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
            var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
            return dtos.Select(factory.BuildEntity);
        }
@@ -69,7 +69,9 @@ namespace Umbraco.Core.Persistence.Repositories
            sql.Select("*")
                .From<AccessDto>(SqlSyntax)
                .LeftJoin<AccessRuleDto>(SqlSyntax)
-                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId);
+                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId)
+                //MUST be ordered by this GUID ID for the AccessRulesRelator to work
+                .OrderBy<AccessDto>(dto => dto.Id, SqlSyntax);

            return sql;
        }
@@ -110,6 +112,11 @@ namespace Umbraco.Core.Persistence.Repositories
            {
                rule.AccessId = entity.Key;
                Database.Insert(rule);
+                //update the id so HasEntity is correct
+                var entityRule = entity.Rules.First(x => x.Key == rule.Id);
+                entityRule.Id = entityRule.Key.GetHashCode();
+                //double make sure that this is set since it is possible to add rules via ctor without AddRule
+                entityRule.AccessEntryId = entity.Key;
            }

            entity.ResetDirtyProperties();
@@ -130,7 +137,7 @@ namespace Umbraco.Core.Persistence.Repositories
            {
                if (rule.HasIdentity)
                {
-                    var count = Database.Update(dto.Rules.Single(x => x.Id == rule.Key));
+                    var count = Database.Update(dto.Rules.First(x => x.Id == rule.Key));
                    if (count == 0)
                    {
                        throw new InvalidOperationException("No rows were updated for the access rule");
@@ -156,6 +163,8 @@ namespace Umbraco.Core.Persistence.Repositories
                Database.Delete<AccessRuleDto>("WHERE id=@Id", new {Id = removedRule});
            }

+            entity.ClearRemovedRules();
+
            entity.ResetDirtyProperties();
        }

@@ -164,4 +173,4 @@ namespace Umbraco.Core.Persistence.Repositories
            return entity.Key;
        }
    }
-}
+}
@@ -63,6 +63,7 @@ namespace Umbraco.Core.Persistence.Repositories
                FormatDeleteStatement("cmsContentVersion", "ContentId"),
                FormatDeleteStatement("cmsContentXml", "nodeId"),
                FormatDeleteStatement("cmsContent", "nodeId"),
+                //TODO: Why is this being done? We just delete this exact data in the next line
                "UPDATE umbracoNode SET parentID = '" + RecycleBinId + "' WHERE trashed = '1' AND nodeObjectType = @NodeObjectType",
                "DELETE FROM umbracoNode WHERE trashed = '1' AND nodeObjectType = @NodeObjectType"
            };
@@ -91,14 +92,18 @@ namespace Umbraco.Core.Persistence.Repositories
            }
        }

+        /// <summary>
+        /// A delete statement that will delete anything in the table specified where its PK (keyName) is found in the
+        /// list of umbracoNode.id that have trashed flag set
+        /// </summary>
+        /// <param name="tableName"></param>
+        /// <param name="keyName"></param>
+        /// <returns></returns>
        private string FormatDeleteStatement(string tableName, string keyName)
        {
-            //This query works with sql ce and sql server:
-            //DELETE FROM umbracoUser2NodeNotify WHERE umbracoUser2NodeNotify.nodeId IN 
-            //(SELECT nodeId FROM umbracoUser2NodeNotify as TB1 INNER JOIN umbracoNode as TB2 ON TB1.nodeId = TB2.id WHERE TB2.trashed = '1' AND TB2.nodeObjectType = 'C66BA18E-EAF3-4CFF-8A22-41B16D66A972')
            return
                string.Format(
-                    "DELETE FROM {0} WHERE {0}.{1} IN (SELECT TB1.{1} FROM {0} as TB1 INNER JOIN umbracoNode as TB2 ON TB1.{1} = TB2.id WHERE TB2.trashed = '1' AND TB2.nodeObjectType = @NodeObjectType)",
+                    "DELETE FROM {0} WHERE {0}.{1} IN (SELECT id FROM umbracoNode WHERE trashed = '1' AND nodeObjectType = @NodeObjectType)",
                    tableName, keyName);
        }

@@ -241,7 +241,12 @@ namespace Umbraco.Core.Persistence.Repositories
                {
                    entity.Path = string.Concat(parent.Path, ",", entity.Id);
                }
-
+                else
+                {
+                    //this means that the master template has been removed, so we need to reset the template's
+                    //path to be at the root
+                    entity.Path = string.Concat("-1,", entity.Id);
+                }
            }

            //Get TemplateDto from db to get the Primary key of the entity
@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Data.SqlTypes;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.Logging;
@@ -25,13 +28,16 @@ using Umbraco.Core.IO;

 namespace Umbraco.Core.Persistence.Repositories
 {
-    using SqlSyntax;
-
    internal abstract class VersionableRepositoryBase<TId, TEntity> : PetaPocoRepositoryBase<TId, TEntity>
        where TEntity : class, IAggregateRoot
    {
        private readonly IContentSection _contentSection;

+        /// <summary>
+        /// This is used for unit tests ONLY
+        /// </summary>
+        internal static bool ThrowOnWarning = false;
+
        protected VersionableRepositoryBase(IDatabaseUnitOfWork work, CacheHelper cache, ILogger logger, ISqlSyntaxProvider sqlSyntax, IContentSection contentSection)
            : base(work, cache, logger, sqlSyntax)
        {
@@ -137,6 +143,40 @@ namespace Umbraco.Core.Persistence.Repositories

        #endregion

+        /// <summary>
+        /// Gets paged document descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public virtual IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords)
+        {
+            var query = new Sql().Select(string.Format("umbracoNode.id, cmsContentXml.{0}", SqlSyntax.GetQuotedColumnName("xml")))
+                .From("umbracoNode")
+                .InnerJoin("cmsContentXml").On("cmsContentXml.nodeId = umbracoNode.id");
+
+            if (path == "-1")
+            {
+                query.Where("umbracoNode.nodeObjectType = @type", new { type = NodeObjectTypeId });
+            }
+            else
+            {
+                query.Where(string.Format("umbracoNode.{0} LIKE (@0)", SqlSyntax.GetQuotedColumnName("path")), path.EnsureEndsWith(",%"));
+            }
+
+            //each order by param needs to be in a bracket! see: https://github.com/toptensoftware/PetaPoco/issues/177
+            query.OrderBy(orderBy == null
+                ? "(umbracoNode.id)"
+                : string.Join(",", orderBy.Select(x => string.Format("({0})", SqlSyntax.GetQuotedColumnName(x)))));
+
+            var pagedResult = Database.Page<ContentXmlDto>(pageIndex + 1, pageSize, query);
+            totalRecords = pagedResult.TotalItems;
+            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
+        }
+
        public int CountDescendants(int parentId, string contentTypeAlias = null)
        {
            var pathMatch = parentId == -1
@@ -348,12 +388,8 @@ namespace Umbraco.Core.Persistence.Repositories
                                ON CustomPropData.CustomPropValContentId = umbracoNode.id
                ", sortedInt, sortedDecimal, sortedDate, sortedString, nodeIdSelect.Item2, nodeIdSelect.Item1, versionQuery, sortedSql.Arguments.Length, newestQuery);

-                //insert this just above the first LEFT OUTER JOIN (for cmsDocument) or the last WHERE (everything else)
-                string newSql;
-                if (nodeIdSelect.Item1 == "cmsDocument")
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.IndexOf("LEFT OUTER JOIN"), outerJoinTempTable);
-                else
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);
+                //insert this just above the last WHERE
+                string newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);

                var newArgs = sortedSql.Arguments.ToList();
                newArgs.Add(orderBy);
@@ -370,11 +406,14 @@ namespace Umbraco.Core.Persistence.Repositories
                }
            }

-            //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
-            // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
-            // is empty for many nodes)
-            // see: http://issues.umbraco.org/issue/U4-8831
-            sortedSql.OrderBy("umbracoNode.id");
+            if (orderBySystemField && orderBy != "umbracoNode.id")
+            {
+                //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
+                // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
+                // is empty for many nodes)
+                // see: http://issues.umbraco.org/issue/U4-8831
+                sortedSql.OrderBy("umbracoNode.id");
+            }

            return sortedSql;

@@ -384,7 +423,6 @@ namespace Umbraco.Core.Persistence.Repositories
        /// A helper method for inheritors to get the paged results by query in a way that minimizes queries
        /// </summary>
        /// <typeparam name="TDto">The type of the d.</typeparam>
-        /// <typeparam name="TContentBase">The 'true' entity type (i.e. Content, Member, etc...)</typeparam>
        /// <param name="query">The query.</param>
        /// <param name="pageIndex">Index of the page.</param>
        /// <param name="pageSize">Size of the page.</param>
@@ -397,34 +435,30 @@ namespace Umbraco.Core.Persistence.Repositories
        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
        /// <returns></returns>
        /// <exception cref="System.ArgumentNullException">orderBy</exception>
-        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto, TContentBase>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
+        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
            Tuple<string, string> nodeIdSelect,
-            Func<Sql, IEnumerable<TEntity>> processQuery,
+            Func<Sql, PagingSqlQuery<TDto>, IEnumerable<TEntity>> processQuery,
            string orderBy,
            Direction orderDirection,
            bool orderBySystemField,
            Func<Tuple<string, object[]>> defaultFilter = null)
-            where TContentBase : class, IAggregateRoot, TEntity
        {
            if (orderBy == null) throw new ArgumentNullException("orderBy");

-            // Get base query
-            var sqlBase = GetBaseQuery(false);
+            // Get base query for returning IDs
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
+            // Get base query for returning all data
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);

            if (query == null) query = new Query<TEntity>();
-            var translator = new SqlTranslator<TEntity>(sqlBase, query);
-            var sqlQuery = translator.Translate();
-
-            // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, 
-            // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
-            // So we'll modify the SQL.
-            var sqlNodeIds = new Sql(
-                sqlQuery.SQL.Replace("SELECT *", string.Format("SELECT {0}.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2)),
-                sqlQuery.Arguments);
+            var translatorIds = new SqlTranslator<TEntity>(sqlBaseIds, query);
+            var sqlQueryIds = translatorIds.Translate();
+            var translatorFull = new SqlTranslator<TEntity>(sqlBaseFull, query);
+            var sqlQueryFull = translatorFull.Translate();

            //get sorted and filtered sql
            var sqlNodeIdsWithSort = GetSortedSqlForPagedResults(
-                GetFilteredSqlForPagedResults(sqlNodeIds, defaultFilter),
+                GetFilteredSqlForPagedResults(sqlQueryIds, defaultFilter),
                orderDirection, orderBy, orderBySystemField, nodeIdSelect);

            // Get page of results and total count
@@ -433,38 +467,32 @@ namespace Umbraco.Core.Persistence.Repositories
            totalRecords = Convert.ToInt32(pagedResult.TotalItems);

            //NOTE: We need to check the actual items returned, not the 'totalRecords', that is because if you request a page number
-            // that doesn't actually have any data on it, the totalRecords will still indicate there are records but there are none in 
+            // that doesn't actually have any data on it, the totalRecords will still indicate there are records but there are none in
            // the pageResult, then the GetAll will actually return ALL records in the db.
            if (pagedResult.Items.Any())
            {
-                //Crete the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
+                //Create the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
                var args = sqlNodeIdsWithSort.Arguments;
                string sqlStringCount, sqlStringPage;
                Database.BuildPageQueries<TDto>(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage);

-                //if this is for sql server, the sqlPage will start with a SELECT * but we don't want that, we only want to return the nodeId                
-                sqlStringPage = sqlStringPage
-                    .Replace("SELECT *",
-                        //This ensures we only take the field name of the node id select and not the table name - since the resulting select
-                        // will ony work with the field name.
-                        "SELECT " + nodeIdSelect.Item2);
-
-                //We need to make this an inner join on the paged query
-                var splitQuery = sqlQuery.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
-                var withInnerJoinSql = new Sql(splitQuery[0])
+                //We need to make this FULL query an inner join on the paged ID query
+                var splitQuery = sqlQueryFull.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
+                var fullQueryWithPagedInnerJoin = new Sql(splitQuery[0])
                    .Append("INNER JOIN (")
                    //join the paged query with the paged query arguments
                    .Append(sqlStringPage, args)
                    .Append(") temp ")
                    .Append(string.Format("ON {0}.{1} = temp.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2))
                    //add the original where clause back with the original arguments
-                    .Where(splitQuery[1], sqlQuery.Arguments);
+                    .Where(splitQuery[1], sqlQueryIds.Arguments);

                //get sorted and filtered sql
                var fullQuery = GetSortedSqlForPagedResults(
-                    GetFilteredSqlForPagedResults(withInnerJoinSql, defaultFilter),
+                    GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter),
                    orderDirection, orderBy, orderBySystemField, nodeIdSelect);
-                return processQuery(fullQuery);
+                
+                return processQuery(fullQuery, new PagingSqlQuery<TDto>(Database, sqlNodeIdsWithSort, pageIndex, pageSize));
            }
            else
            {
@@ -474,38 +502,56 @@ namespace Umbraco.Core.Persistence.Repositories
            return result;
        }

-        protected IDictionary<int, PropertyCollection> GetPropertyCollection(
-            Sql docSql,
-            IEnumerable<DocumentDefinition> documentDefs)
+        /// <summary>
+        /// Gets the property collection for a non-paged query
+        /// </summary>
+        /// <param name="sql"></param>
+        /// <param name="documentDefs"></param>
+        /// <returns></returns>
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
+            Sql sql,
+            IReadOnlyCollection<DocumentDefinition> documentDefs)
        {
-            if (documentDefs.Any() == false) return new Dictionary<int, PropertyCollection>();
+            return GetPropertyCollection(new PagingSqlQuery(sql), documentDefs);
+        }

-            //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use 
+        /// <summary>
+        /// Gets the property collection for a query
+        /// </summary>
+        /// <param name="pagingSqlQuery"></param>
+        /// <param name="documentDefs"></param>
+        /// <returns></returns>
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
+            PagingSqlQuery pagingSqlQuery,
+            IReadOnlyCollection<DocumentDefinition> documentDefs)
+        {
+            if (documentDefs.Count == 0) return new Dictionary<Guid, PropertyCollection>();
+
+            //initialize to the query passed in
+            var docSql = pagingSqlQuery.PrePagedSql;
+
+            //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use
            // the statement to go get the property data for all of the items by using an inner join
            var parsedOriginalSql = "SELECT {0} " + docSql.SQL.Substring(docSql.SQL.IndexOf("FROM", StringComparison.Ordinal));
-            //now remove everything from an Orderby clause and beyond
-            if (parsedOriginalSql.InvariantContains("ORDER BY "))
+            
+            if (pagingSqlQuery.HasPaging)
            {
+                //if this is a paged query, build the paged query with the custom column substitution, then re-assign
+                docSql = pagingSqlQuery.BuildPagedQuery("{0}");
+                parsedOriginalSql = docSql.SQL;
+            }
+            else if (parsedOriginalSql.InvariantContains("ORDER BY "))
+            {
+                //now remove everything from an Orderby clause and beyond if this is unpaged data
                parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
            }

-            var propSql = new Sql(@"SELECT cmsPropertyData.*
-FROM cmsPropertyData
-INNER JOIN cmsPropertyType
-ON cmsPropertyData.propertytypeid = cmsPropertyType.id
-INNER JOIN 
-	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
-ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-LEFT OUTER JOIN cmsDataTypePreValues
-ON cmsPropertyType.dataTypeId = cmsDataTypePreValues.datatypeNodeId", docSql.Arguments);
-
-            var allPropertyData = Database.Fetch<PropertyDataDto>(propSql);
-
-            //This is a lazy access call to get all prevalue data for the data types that make up all of these properties which we use
-            // below if any property requires tag support
-            var allPreValues = new Lazy<IEnumerable<DataTypePreValueDto>>(() =>
-            {
-                var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
+            //This retrieves all pre-values for all data types that are referenced for all property types
+            // that exist in the data set.
+            //Benchmarks show that eagerly loading these so that we can lazily read the property data
+            // below (with the use of Query intead of Fetch) go about 30% faster, so we'll eagerly load
+            // this now since we cannot execute another reader inside of reading the property data.
+            var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
 FROM cmsDataTypePreValues a
 WHERE EXISTS(
    SELECT DISTINCT b.id as preValIdInner
@@ -513,29 +559,86 @@ WHERE EXISTS(
 	INNER JOIN cmsPropertyType
 	ON b.datatypeNodeId = cmsPropertyType.dataTypeId
    INNER JOIN 
-	    (" + string.Format(parsedOriginalSql, "DISTINCT cmsContent.contentType") + @") as docData
+	    (" + string.Format(parsedOriginalSql, "cmsContent.contentType") + @") as docData
    ON cmsPropertyType.contentTypeId = docData.contentType
    WHERE a.id = b.id)", docSql.Arguments);

-                return Database.Fetch<DataTypePreValueDto>(preValsSql);
-            });
+            var allPreValues = Database.Fetch<DataTypePreValueDto>(preValsSql);

-            var result = new Dictionary<int, PropertyCollection>();
+            //It's Important with the sort order here! We require this to be sorted by node id,
+            // this is required because this data set can be huge depending on the page size. Due
+            // to it's size we need to be smart about iterating over the property values to build
+            // the document. Before we used to use Linq to get the property data for a given content node
+            // and perform a Distinct() call. This kills performance because that would mean if we had 7000 nodes
+            // and on each iteration we will perform a lookup on potentially 100,000 property rows against the node
+            // id which turns out to be a crazy amount of iterations. Instead we know it's sorted by this value we'll
+            // keep an index stored of the rows being read so we never have to re-iterate the entire data set
+            // on each document iteration.
+            var propSql = new Sql(@"SELECT cmsPropertyData.*
+FROM cmsPropertyData
+INNER JOIN cmsPropertyType
+ON cmsPropertyData.propertytypeid = cmsPropertyType.id
+INNER JOIN 
+	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
+ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
+ORDER BY contentNodeId, versionId, propertytypeid
+", docSql.Arguments);
+            
+            //This does NOT fetch all data into memory in a list, this will read
+            // over the records as a data reader, this is much better for performance and memory,
+            // but it means that during the reading of this data set, nothing else can be read
+            // from SQL server otherwise we'll get an exception.
+            var allPropertyData = Database.Query<PropertyDataDto>(propSql);

+            var result = new Dictionary<Guid, PropertyCollection>();
            var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
+            //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
+            var resolvedCompositionProperties = new Dictionary<int, PropertyType[]>();

-            //iterate each definition grouped by it's content type - this will mean less property type iterations while building 
-            // up the property collections
-            foreach (var compositionGroup in documentDefs.GroupBy(x => x.Composition))
+            //keep track of the current property data item being enumerated
+            var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
+            var hasCurrent = false; // initially there is no enumerator.Current
+
+            var comparer = new DocumentDefinitionComparer(SqlSyntax);
+
+            try
            {
-                var compositionProperties = compositionGroup.Key.CompositionPropertyTypes.ToArray();
-
-                foreach (var def in compositionGroup)
+                //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
+                // which allows us to more efficiently iterate over the large data set of property values
+                foreach (var def in documentDefs.OrderBy(x => x.Id).ThenBy(x => x.Version, comparer))
                {
-                    var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct();
+                    // get the resolved properties from our local cache, or resolve them and put them in cache
+                    PropertyType[] compositionProperties;
+                    if (resolvedCompositionProperties.ContainsKey(def.Composition.Id))
+                    {
+                        compositionProperties = resolvedCompositionProperties[def.Composition.Id];
+                    }
+                    else
+                    {
+                        compositionProperties = def.Composition.CompositionPropertyTypes.ToArray();
+                        resolvedCompositionProperties[def.Composition.Id] = compositionProperties;
+                    }

-                    var propertyFactory = new PropertyFactory(compositionProperties, def.Version, def.Id, def.CreateDate, def.VersionDate);
-                    var properties = propertyFactory.BuildEntity(propertyDataDtos.ToArray()).ToArray();
+                    // assemble the dtos for this def
+                    // use the available enumerator.Current if any else move to next
+                    var propertyDataDtos = new List<PropertyDataDto>();
+                    while (hasCurrent || propertyDataSetEnumerator.MoveNext())
+                    {
+                        //Not checking null on VersionId because it can never be null - no idea why it's set to nullable
+                        // ReSharper disable once PossibleInvalidOperationException
+                        if (propertyDataSetEnumerator.Current.VersionId.Value == def.Version)
+                        {
+                            hasCurrent = false; // enumerator.Current is not available
+                            propertyDataDtos.Add(propertyDataSetEnumerator.Current);
+                        }
+                        else
+                        {
+                            hasCurrent = true;  // enumerator.Current is available for another def
+                            break;              // no more propertyDataDto for this def
+                        }
+                    }
+
+                    var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray();

                    foreach (var property in properties)
                    {
@@ -552,7 +655,7 @@ WHERE EXISTS(
                            propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport;

                            //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up
-                            var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
+                            var preValData = allPreValues.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
                                .Distinct()
                                .ToArray();

@@ -568,38 +671,30 @@ WHERE EXISTS(
                        }
                    }

-                    if (result.ContainsKey(def.Id))
+                    if (result.ContainsKey(def.Version))
                    {
-                        Logger.Warn<VersionableRepositoryBase<TId, TEntity>>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name);
+                        var msg = string.Format("The query returned multiple property sets for document definition {0}, {1}, {2}", def.Id, def.Version, def.Composition.Name);
+                        if (ThrowOnWarning)
+                        {
+                            throw new InvalidOperationException(msg);
+                        }
+                        else
+                        {
+                            Logger.Warn<VersionableRepositoryBase<TId, TEntity>>(msg);
+                        }
                    }
-                    result[def.Id] = new PropertyCollection(properties);
+                    result[def.Version] = new PropertyCollection(properties);
                }
            }
+            finally
+            {
+                propertyDataSetEnumerator.Dispose();
+            }

            return result;
+
        }
-
-        public class DocumentDefinition
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="T:System.Object"/> class.
-            /// </summary>
-            public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition)
-            {
-                Id = id;
-                Version = version;
-                VersionDate = versionDate;
-                CreateDate = createDate;
-                Composition = composition;
-            }
-
-            public int Id { get; set; }
-            public Guid Version { get; set; }
-            public DateTime VersionDate { get; set; }
-            public DateTime CreateDate { get; set; }
-            public IContentTypeComposition Composition { get; set; }
-        }
-
+        
        protected virtual string GetDatabaseFieldNameForOrderBy(string orderBy)
        {
            // Translate the passed order by field (which were originally defined for in-memory object sorting
@@ -611,8 +706,8 @@ WHERE EXISTS(
                case "NAME":
                    return "umbracoNode.text";
                case "PUBLISHED":
-                    return "cmsDocument.published";
                case "OWNER":
+                    return "cmsDocument.published";
                    //TODO: This isn't going to work very nicely because it's going to order by ID, not by letter
                    return "umbracoNode.nodeUser";
                // Members only
@@ -688,5 +783,233 @@ WHERE EXISTS(

            return allsuccess;
        }
+
+        /// <summary>
+        /// For Paging, repositories must support returning different query for the query type specified
+        /// </summary>
+        /// <param name="queryType"></param>
+        /// <returns></returns>
+        protected abstract Sql GetBaseQuery(BaseQueryType queryType);
+
+        internal class DocumentDefinitionCollection : KeyedCollection<ValueType, DocumentDefinition>
+        {
+            private readonly bool _includeAllVersions;
+
+            /// <summary>
+            /// Constructor specifying if all versions should be allowed, in that case the key for the collection becomes the versionId (GUID)
+            /// </summary>
+            /// <param name="includeAllVersions"></param>
+            public DocumentDefinitionCollection(bool includeAllVersions = false)
+            {
+                _includeAllVersions = includeAllVersions;
+            }
+
+            protected override ValueType GetKeyForItem(DocumentDefinition item)
+            {
+                return _includeAllVersions ? (ValueType)item.Version : item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(DocumentDefinition item)
+            {
+                //if we are including all versions then just add, we aren't checking for latest
+                if (_includeAllVersions)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                DocumentDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Version);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+                
+                base.Add(item);
+                return true;
+            }
+          
+            public bool TryGetValue(ValueType key, out DocumentDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        /// <summary>
+        /// A custom comparer required for sorting entities by GUIDs to match how the sorting of GUIDs works on SQL server
+        /// </summary>
+        /// <remarks>
+        /// MySql sorts GUIDs as a string, MSSQL sorts based on byte sections, this comparer will allow sorting GUIDs to be the same as how SQL server does
+        /// </remarks>
+        private class DocumentDefinitionComparer : IComparer<Guid>
+        {
+            private readonly ISqlSyntaxProvider _sqlSyntax;
+
+            public DocumentDefinitionComparer(ISqlSyntaxProvider sqlSyntax)
+            {
+                _sqlSyntax = sqlSyntax;
+            }
+
+            public int Compare(Guid x, Guid y)
+            {
+                //MySql sorts on GUIDs as strings (i.e. normal)
+                if (_sqlSyntax is MySqlSyntaxProvider)
+                {
+                    return x.CompareTo(y);
+                }
+
+                //MSSQL doesn't it sorts them on byte sections!
+                return new SqlGuid(x).CompareTo(new SqlGuid(y));
+            }
+        }
+
+        internal class DocumentDefinition
+        {
+            /// <summary>
+            /// Initializes a new instance of the <see cref="T:System.Object"/> class.
+            /// </summary>
+            public DocumentDefinition(DocumentDto dto, IContentTypeComposition composition)
+            {
+                DocumentDto = dto;
+                ContentVersionDto = dto.ContentVersionDto;
+                Composition = composition;
+            }
+
+            public DocumentDefinition(ContentVersionDto dto, IContentTypeComposition composition)
+            {
+                ContentVersionDto = dto;
+                Composition = composition;
+            }
+
+            public DocumentDto DocumentDto { get; private set; }
+            public ContentVersionDto ContentVersionDto { get; private set; }
+
+            public int Id
+            {
+                get { return ContentVersionDto.NodeId; }
+            }
+            
+            public Guid Version
+            {
+                get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; }
+            }
+
+            /// <summary>
+            /// This is used to determien which version is the most recent
+            /// </summary>
+            public int VersionId
+            {
+                get { return ContentVersionDto.Id; }
+            }
+
+            public DateTime VersionDate
+            {
+                get { return ContentVersionDto.VersionDate; }
+            }
+
+            public DateTime CreateDate
+            {
+                get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; }
+            }
+
+            public IContentTypeComposition Composition { get; set; }            
+
+            
+        }
+
+        /// <summary>
+        /// An object representing a query that may contain paging information
+        /// </summary>
+        internal class PagingSqlQuery
+        {
+            public Sql PrePagedSql { get; private set; }
+
+            public PagingSqlQuery(Sql prePagedSql)
+            {
+                PrePagedSql = prePagedSql;
+            }
+
+            public virtual bool HasPaging
+            {
+                get { return false; }
+            }
+
+            public virtual Sql BuildPagedQuery(string selectColumns)
+            {
+                throw new InvalidOperationException("This query has no paging information");
+            }
+        }
+
+        /// <summary>
+        /// An object representing a query that contains paging information
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        internal class PagingSqlQuery<T> : PagingSqlQuery
+        {
+            private readonly Database _db;
+            private readonly long _pageIndex;
+            private readonly int _pageSize;
+
+            public PagingSqlQuery(Database db, Sql prePagedSql, long pageIndex, int pageSize) : base(prePagedSql)
+            {
+                _db = db;
+                _pageIndex = pageIndex;
+                _pageSize = pageSize;                
+            }
+
+            public override bool HasPaging
+            {
+                get { return _pageSize > 0; }
+            }
+
+            /// <summary>
+            /// Creates a paged query based on the original query and subtitutes the selectColumns specified
+            /// </summary>
+            /// <param name="selectColumns"></param>
+            /// <returns></returns>
+            public override Sql BuildPagedQuery(string selectColumns)
+            {
+                if (HasPaging == false) throw new InvalidOperationException("This query has no paging information");
+
+                var resultSql = string.Format("SELECT {0} {1}", selectColumns, PrePagedSql.SQL.Substring(PrePagedSql.SQL.IndexOf("FROM", StringComparison.Ordinal)));
+
+                //this query is meant to be paged so we need to generate the paging syntax
+                //Create the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
+                var args = PrePagedSql.Arguments;
+                string sqlStringCount, sqlStringPage;
+                _db.BuildPageQueries<T>(_pageIndex * _pageSize, _pageSize, resultSql, ref args, out sqlStringCount, out sqlStringPage);
+
+                return new Sql(sqlStringPage, args);
+            }
+        }
    }
 }
@@ -48,7 +48,12 @@ namespace Umbraco.Core.Persistence
                _cacheHelper.IsolatedRuntimeCache.CacheFactory = type =>
                {
                    var cache = origFactory(type);
-                    return new DeepCloneRuntimeCacheProvider(cache);
+
+                    //if the result is already a DeepCloneRuntimeCacheProvider then return it, otherwise
+                    //wrap the result with a DeepCloneRuntimeCacheProvider
+                    return cache is DeepCloneRuntimeCacheProvider 
+                        ? cache 
+                        : new DeepCloneRuntimeCacheProvider(cache);
                };
            }

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using Umbraco.Core.Events;
 using Umbraco.Core.Models;
 using Umbraco.Core.Services;
@@ -12,7 +13,8 @@ namespace Umbraco.Core.Publishing
    {
        public PublishStatus(IContent content, PublishStatusType statusType, EventMessages eventMessages)
            : base(content, statusType, eventMessages)
-        {            
+        {
+            InvalidProperties = Enumerable.Empty<Property>();
        }

        /// <summary>
@@ -20,8 +22,7 @@ namespace Umbraco.Core.Publishing
        /// </summary>
        public PublishStatus(IContent content, EventMessages eventMessages)
            : this(content, PublishStatusType.Success, eventMessages)
-        {
-        }
+        { }

        public IContent ContentItem
        {
@@ -30,7 +30,7 @@ namespace Umbraco.Core.Publishing
        /// Publishes a single piece of Content
        /// </summary>
        /// <param name="content"><see cref="IContent"/> to publish</param>
-        /// <param name="userId">Id of the User issueing the publish operation</param>        
+        /// <param name="userId">Id of the User issueing the publish operation</param>
        internal Attempt<PublishStatus> PublishInternal(IContent content, int userId)
        {
            var evtMsgs = _eventMessagesFactory.Get();
@@ -99,26 +99,26 @@ namespace Umbraco.Core.Publishing
        /// By default this is set to true which means that it will publish any content item in the list that is completely unpublished and
        /// not visible on the front-end. If set to false, this will only publish content that is live on the front-end but has new versions
        /// that have yet to be published.
-        /// </param>        
+        /// </param>
        /// <returns></returns>
        /// <remarks>
-        /// 
+        ///
        /// This method becomes complex once we start to be able to cancel events or stop publishing a content item in any way because if a
        /// content item is not published then it's children shouldn't be published either. This rule will apply for the following conditions:
        /// * If a document fails to be published, do not proceed to publish it's children if:
        /// ** The document does not have a publish version
        /// ** The document does have a published version but the includeUnpublishedDocuments = false
-        /// 
+        ///
        /// In order to do this, we will order the content by level and begin by publishing each item at that level, then proceed to the next
-        /// level and so on. If we detect that the above rule applies when the document publishing is cancelled we'll add it to the list of 
+        /// level and so on. If we detect that the above rule applies when the document publishing is cancelled we'll add it to the list of
        /// parentsIdsCancelled so that it's children don't get published.
-        /// 
+        ///
        /// Its important to note that all 'root' documents included in the list *will* be published regardless of the rules mentioned
-        /// above (unless it is invalid)!! By 'root' documents we are referring to documents in the list with the minimum value for their 'level'. 
-        /// In most cases the 'root' documents will only be one document since under normal circumstance we only publish one document and 
+        /// above (unless it is invalid)!! By 'root' documents we are referring to documents in the list with the minimum value for their 'level'.
+        /// In most cases the 'root' documents will only be one document since under normal circumstance we only publish one document and
        /// its children. The reason we have to do this is because if a user is publishing a document and it's children, it is implied that
        /// the user definitely wants to publish it even if it has never been published before.
-        /// 
+        ///
        /// </remarks>
        internal IEnumerable<Attempt<PublishStatus>> PublishWithChildrenInternal(
            IEnumerable<IContent> content, int userId, bool includeUnpublishedDocuments = true)
@@ -132,7 +132,7 @@ namespace Umbraco.Core.Publishing

            //group by levels and iterate over the sorted ascending level.
            //TODO: This will cause all queries to execute, they will not be lazy but I'm not really sure being lazy actually made
-            // much difference because we iterate over them all anyways?? Morten?       
+            // much difference because we iterate over them all anyways?? Morten?
            // Because we're grouping I think this will execute all the queries anyways so need to fetch it all first.
            var fetchedContent = content.ToArray();

@@ -149,7 +149,7 @@ namespace Umbraco.Core.Publishing
            var levelGroups = fetchedContent.GroupBy(x => x.Level);
            foreach (var level in levelGroups.OrderBy(x => x.Key))
            {
-                //set the first level flag, used to ensure that all documents at the first level will 
+                //set the first level flag, used to ensure that all documents at the first level will
                //be published regardless of the rules mentioned in the remarks.
                if (!firstLevel.HasValue)
                {
@@ -200,7 +200,10 @@ namespace Umbraco.Core.Publishing
                        _logger.Info<PublishingStrategy>(
                            string.Format("Content '{0}' with Id '{1}' will not be published because some of it's content is not passing validation rules.",
                                          item.Name, item.Id));
-                        statuses.Add(Attempt.Fail(new PublishStatus(item, PublishStatusType.FailedContentInvalid, evtMsgs)));
+                        statuses.Add(Attempt.Fail(new PublishStatus(item, PublishStatusType.FailedContentInvalid, evtMsgs)
+                        {
+                            InvalidProperties = ((ContentBase)item).LastInvalidProperties
+                        }));

                        //Does this document apply to our rule to cancel it's children being published?
                        CheckCancellingOfChildPublishing(item, parentsIdsCancelled, includeUnpublishedDocuments);
@@ -272,11 +275,11 @@ namespace Umbraco.Core.Publishing
        /// <param name="includeUnpublishedDocuments"></param>
        /// <remarks>
        /// See remarks on method: PublishWithChildrenInternal
-        /// </remarks> 
+        /// </remarks>
        private void CheckCancellingOfChildPublishing(IContent content, List<int> parentsIdsCancelled, bool includeUnpublishedDocuments)
        {
            //Does this document apply to our rule to cancel it's children being published?
-            //TODO: We're going back to the service layer here... not sure how to avoid this? And this will add extra overhead to 
+            //TODO: We're going back to the service layer here... not sure how to avoid this? And this will add extra overhead to
            // any document that fails to publish...
            var hasPublishedVersion = ApplicationContext.Current.Services.ContentService.HasPublishedVersion(content.Id);

@@ -306,8 +309,8 @@ namespace Umbraco.Core.Publishing
            //NOTE: This previously always returned true so I've left it that way. It returned true because (from Morten)...
            // ... if one item couldn't be published it wouldn't be correct to return false.
            // in retrospect it should have returned a list of with Ids and Publish Status
-            // come to think of it ... the cache would still be updated for a failed item or at least tried updated. 
-            // It would call the Published event for the entire list, but if the Published property isn't set to True it 
+            // come to think of it ... the cache would still be updated for a failed item or at least tried updated.
+            // It would call the Published event for the entire list, but if the Published property isn't set to True it
            // wouldn't actually update the cache for that item. But not really ideal nevertheless...
            return true;
        }
@@ -345,7 +348,7 @@ namespace Umbraco.Core.Publishing
            var evtMsgs = _eventMessagesFactory.Get();

            //Fire UnPublishing event
-            if (UnPublishing.IsRaisedEventCancelled(                
+            if (UnPublishing.IsRaisedEventCancelled(
                new PublishEventArgs<IContent>(content, evtMsgs), this))
            {
                _logger.Info<PublishingStrategy>(
@@ -388,8 +391,8 @@ namespace Umbraco.Core.Publishing
            //NOTE: This previously always returned true so I've left it that way. It returned true because (from Morten)...
            // ... if one item couldn't be published it wouldn't be correct to return false.
            // in retrospect it should have returned a list of with Ids and Publish Status
-            // come to think of it ... the cache would still be updated for a failed item or at least tried updated. 
-            // It would call the Published event for the entire list, but if the Published property isn't set to True it 
+            // come to think of it ... the cache would still be updated for a failed item or at least tried updated.
+            // It would call the Published event for the entire list, but if the Published property isn't set to True it
            // wouldn't actually update the cache for that item. But not really ideal nevertheless...
            return true;
        }
@@ -405,7 +408,7 @@ namespace Umbraco.Core.Publishing
        public override void PublishingFinalized(IContent content)
        {
            var evtMsgs = _eventMessagesFactory.Get();
-            Published.RaiseEvent(                
+            Published.RaiseEvent(
                new PublishEventArgs<IContent>(content, false, false, evtMsgs), this);
        }

@@ -114,7 +114,7 @@ namespace Umbraco.Core.Security
                    AllowRefresh = true,
                    IssuedUtc = nowUtc,
                    ExpiresUtc = nowUtc.AddMinutes(GlobalSettings.TimeOutInMinutes)
-                }, userIdentity, rememberBrowserIdentity);
+                }, userIdentity, rememberBrowserIdentity);                
            }
            else
            {
@@ -127,6 +127,11 @@ namespace Umbraco.Core.Security
                }, userIdentity);
            }

+            //track the last login date
+            user.LastLoginDateUtc = DateTime.UtcNow;
+            user.AccessFailedCount = 0;
+            await UserManager.UpdateAsync(user);
+
            _logger.WriteCore(TraceEventType.Information, 0,
                string.Format(
                    "Login attempt succeeded for username {0} from IP address {1}",
@@ -134,4 +139,4 @@ namespace Umbraco.Core.Security
                    _request.RemoteIpAddress), null, null);
        }
    }
-}
+}
@@ -625,6 +625,12 @@ namespace Umbraco.Core.Security
            var anythingChanged = false;
            //don't assign anything if nothing has changed as this will trigger
            //the track changes of the model
+            if ((user.LastLoginDate != default(DateTime) && identityUser.LastLoginDateUtc.HasValue == false)
+                || identityUser.LastLoginDateUtc.HasValue && user.LastLoginDate.ToUniversalTime() != identityUser.LastLoginDateUtc.Value)
+            {
+                anythingChanged = true;
+                user.LastLoginDate = identityUser.LastLoginDateUtc.Value.ToLocalTime();
+            }
            if (user.Name != identityUser.Name && identityUser.Name.IsNullOrWhiteSpace() == false)
            {
                anythingChanged = true;
@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Xml;
@@ -11,6 +12,7 @@ using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
+using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence;
@@ -58,7 +60,7 @@ namespace Umbraco.Core.Services

        #region Static Queries

-        private readonly IQuery<IContent> _notTrashedQuery = Query<IContent>.Builder.Where(x => x.Trashed == false);
+        private IQuery<IContent> _notTrashedQuery;

        #endregion

@@ -583,7 +585,7 @@ namespace Umbraco.Core.Services

        [Obsolete("Use the overload with 'long' parameter types instead")]
        [EditorBrowsable(EditorBrowsableState.Never)]
-        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
        {
            long total;
            var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter);
@@ -602,7 +604,7 @@ namespace Umbraco.Core.Services
        /// <param name="orderDirection">Direction to order by</param>
        /// <param name="filter">Search text filter</param>
        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns> 
-        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
        {
            return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter);
        }
@@ -714,7 +716,12 @@ namespace Umbraco.Core.Services
        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
        public IEnumerable<IContent> GetDescendants(IContent content)
        {
-            using (var repository = RepositoryFactory.CreateContentRepository(UowProvider.GetUnitOfWork()))
+            //This is a check to ensure that the path is correct for this entity to avoid problems like: http://issues.umbraco.org/issue/U4-9336 due to data corruption
+            if (content.ValidatePath() == false)
+                throw new InvalidDataException(string.Format("The content item {0} has an invalid path: {1} with parentID: {2}", content.Id, content.Path, content.ParentId));
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateContentRepository(uow))
            {
                var pathMatch = content.Path + ",";
                var query = Query<IContent>.Builder.Where(x => x.Path.StartsWith(pathMatch) && x.Id != content.Id);
@@ -793,6 +800,12 @@ namespace Umbraco.Core.Services
        /// <returns></returns>
        internal IEnumerable<IContent> GetAllPublished()
        {
+            //create it once if it is needed (no need for locking here)
+            if (_notTrashedQuery == null)
+            {
+                _notTrashedQuery = Query<IContent>.Builder.Where(x => x.Trashed == false);
+            }
+
            using (var repository = RepositoryFactory.CreateContentRepository(UowProvider.GetUnitOfWork()))
            {
                return repository.GetByPublishedVersion(_notTrashedQuery);
@@ -990,6 +1003,10 @@ namespace Umbraco.Core.Services

            using (new WriteLock(Locker))
            {
+                //Hack: this ensures that the entity's path is valid and if not it fixes/persists it
+                //see: http://issues.umbraco.org/issue/U4-9336
+                content.EnsureValidPath(Logger, entity => GetById(entity.ParentId), QuickUpdate);
+
                var originalPath = content.Path;

                if (Trashing.IsRaisedEventCancelled(
@@ -1677,16 +1694,16 @@ namespace Umbraco.Core.Services
        /// <returns>True if sorting succeeded, otherwise False</returns>
        public bool Sort(IEnumerable<IContent> items, int userId = 0, bool raiseEvents = true)
        {
+            var asArray = items.ToArray();
            if (raiseEvents)
            {
-                if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IContent>(items), this))
+                if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IContent>(asArray), this))
                    return false;
            }

            var shouldBePublished = new List<IContent>();
            var shouldBeSaved = new List<IContent>();
-
-            var asArray = items.ToArray();
+            
            using (new WriteLock(Locker))
            {
                var uow = UowProvider.GetUnitOfWork();
@@ -1747,6 +1764,31 @@ namespace Umbraco.Core.Services
            return true;
        }

+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords)
+        {
+            Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
+            Mandate.ParameterCondition(pageSize > 0, "pageSize");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateContentRepository(uow))
+            {
+                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize,
+                    //This order by is VERY important! This allows us to figure out what is implicitly not published, see ContentRepository.BuildXmlCache and
+                    // UmbracoContentIndexer.PerformIndexAll which uses the logic based on this sort order
+                    new[] {"level", "parentID", "sortOrder"},
+                    out totalRecords);
+                return contents;
+            }
+        }
+
        /// <summary>
        /// This builds the Xml document used for the XML cache
        /// </summary>
@@ -1806,6 +1848,23 @@ namespace Umbraco.Core.Services

        #region Private Methods

+        /// <summary>
+        /// Hack: This is used to fix some data if an entity's properties are invalid/corrupt
+        /// </summary>
+        /// <param name="content"></param>
+        private void QuickUpdate(IContent content)
+        {
+            if (content == null) throw new ArgumentNullException("content");
+            if (content.HasIdentity == false) throw new InvalidOperationException("Cannot update an entity without an Identity");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateContentRepository(uow))
+            {
+                repository.AddOrUpdate(content);                
+                uow.Commit();
+            }
+        }
+
        private void Audit(AuditType type, string message, int userId, int objectId)
        {
            var uow = UowProvider.GetUnitOfWork();
@@ -1913,6 +1972,10 @@ namespace Umbraco.Core.Services

            using (new WriteLock(Locker))
            {
+                //Hack: this ensures that the entity's path is valid and if not it fixes/persists it
+                //see: http://issues.umbraco.org/issue/U4-9336
+                content.EnsureValidPath(Logger, entity => GetById(entity.ParentId), QuickUpdate);
+
                var result = new List<Attempt<PublishStatus>>();

                //Check if parent is published (although not if its a root node) - if parent isn't published this Content cannot be published
@@ -2122,6 +2185,7 @@ namespace Umbraco.Core.Services
                //We need to check if children and their publish state to ensure that we 'republish' content that was previously published
                if (published && previouslyPublished == false && HasChildren(content.Id))
                {
+                    //TODO: Horrible for performance if there are lots of descendents! We should page if anything but this is crazy
                    var descendants = GetPublishedDescendants(content);

                    _publishingStrategy.PublishingFinalized(descendants, false);
@@ -2154,6 +2218,11 @@ namespace Umbraco.Core.Services
                }
            }

+            if (string.IsNullOrWhiteSpace(content.Name))
+            {
+                throw new ArgumentException("Cannot save content with empty name.");
+            }
+
            using (new WriteLock(Locker))
            {
                var uow = UowProvider.GetUnitOfWork();
@@ -1,22 +1,15 @@
 using System;
 using System.Collections.Generic;
-using System.Data;
-using System.Diagnostics;
 using System.Linq;
 using System.Text;
-using System.Xml.Linq;
 using System.Threading;
-using AutoMapper;
-using Umbraco.Core.Auditing;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.Exceptions;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
-using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Persistence.UnitOfWork;

 namespace Umbraco.Core.Services
@@ -26,14 +19,15 @@ namespace Umbraco.Core.Services
    /// </summary>
    public class ContentTypeService : ContentTypeServiceBase, IContentTypeService
    {
-	    private readonly IContentService _contentService;
+        private readonly IContentService _contentService;
        private readonly IMediaService _mediaService;

        //Support recursive locks because some of the methods that require locking call other methods that require locking.
        //for example, the Move method needs to be locked but this calls the Save method which also needs to be locked.
        private static readonly ReaderWriterLockSlim Locker = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);

-        public ContentTypeService(IDatabaseUnitOfWorkProvider provider, RepositoryFactory repositoryFactory, ILogger logger, IEventMessagesFactory eventMessagesFactory, IContentService contentService, IMediaService mediaService)
+        public ContentTypeService(IDatabaseUnitOfWorkProvider provider, RepositoryFactory repositoryFactory, ILogger logger, IEventMessagesFactory eventMessagesFactory, IContentService contentService,
+            IMediaService mediaService)
            : base(provider, repositoryFactory, logger, eventMessagesFactory)
        {
            if (contentService == null) throw new ArgumentNullException("contentService");
@@ -154,8 +148,8 @@ namespace Umbraco.Core.Services
            }

            if (savingEvent.IsRaisedEventCancelled(
-                        new SaveEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                new SaveEventArgs<EntityContainer>(container, evtMsgs),
+                this))
            {
                return OperationStatus.Cancelled(evtMsgs);
            }
@@ -214,7 +208,7 @@ namespace Umbraco.Core.Services

        public IEnumerable<EntityContainer> GetMediaTypeContainers(IMediaType mediaType)
        {
-            var ancestorIds = mediaType.Path.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
+            var ancestorIds = mediaType.Path.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries)
                .Select(x =>
                {
                    var asInt = x.TryConvertTo<int>();
@@ -290,8 +284,8 @@ namespace Umbraco.Core.Services
                if (container == null) return OperationStatus.NoOperation(evtMsgs);

                if (DeletingContentTypeContainer.IsRaisedEventCancelled(
-                        new DeleteEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                    new DeleteEventArgs<EntityContainer>(container, evtMsgs),
+                    this))
                {
                    return Attempt.Fail(new OperationStatus(OperationStatusType.FailedCancelledByEvent, evtMsgs));
                }
@@ -316,8 +310,8 @@ namespace Umbraco.Core.Services
                if (container == null) return OperationStatus.NoOperation(evtMsgs);

                if (DeletingMediaTypeContainer.IsRaisedEventCancelled(
-                        new DeleteEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                    new DeleteEventArgs<EntityContainer>(container, evtMsgs),
+                    this))
                {
                    return Attempt.Fail(new OperationStatus(OperationStatusType.FailedCancelledByEvent, evtMsgs));
                }
@@ -421,7 +415,7 @@ namespace Umbraco.Core.Services

            clone.Name = name;

-            var compositionAliases = clone.CompositionAliases().Except(new[] { alias }).ToList();
+            var compositionAliases = clone.CompositionAliases().Except(new[] {alias}).ToList();
            //remove all composition that is not it's current alias
            foreach (var a in compositionAliases)
            {
@@ -680,7 +674,7 @@ namespace Umbraco.Core.Services
            var comparer = new DelegateEqualityComparer<IContentTypeComposition>((x, y) => x.Id == y.Id, x => x.Id);
            var dependencies = new HashSet<IContentTypeComposition>(compositions, comparer);
            var stack = new Stack<IContentTypeComposition>();
-            indirectReferences.ForEach(stack.Push);//Push indirect references to a stack, so we can add recursively
+            indirectReferences.ForEach(stack.Push); //Push indirect references to a stack, so we can add recursively
            while (stack.Count > 0)
            {
                var indirectReference = stack.Pop();
@@ -719,8 +713,13 @@ namespace Umbraco.Core.Services
        /// <param name="userId">Optional id of the user saving the ContentType</param>
        public void Save(IContentType contentType, int userId = 0)
        {
-	        if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
-				return;
+            if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
+                return;
+
+            if (string.IsNullOrWhiteSpace(contentType.Name))
+            {
+                throw new ArgumentException("Cannot save content type with empty name.");
+            }

            using (new WriteLock(Locker))
            {
@@ -729,6 +728,8 @@ namespace Umbraco.Core.Services
                {
                    ValidateLocked(contentType); // throws if invalid
                    contentType.CreatorId = userId;
+                    if (contentType.Description == string.Empty)
+                        contentType.Description = null;
                    repository.AddOrUpdate(contentType);

                    uow.Commit();
@@ -737,7 +738,7 @@ namespace Umbraco.Core.Services
                UpdateContentXmlStructure(contentType);
            }
            SavedContentType.RaiseEvent(new SaveEventArgs<IContentType>(contentType, false), this);
-	        Audit(AuditType.Save, string.Format("Save ContentType performed by user"), userId, contentType.Id);
+            Audit(AuditType.Save, string.Format("Save ContentType performed by user"), userId, contentType.Id);
        }

        /// <summary>
@@ -750,7 +751,7 @@ namespace Umbraco.Core.Services
            var asArray = contentTypes.ToArray();

            if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(asArray), this))
-				return;
+                return;

            using (new WriteLock(Locker))
            {
@@ -765,6 +766,8 @@ namespace Umbraco.Core.Services
                    foreach (var contentType in asArray)
                    {
                        contentType.CreatorId = userId;
+                        if (contentType.Description == string.Empty)
+                            contentType.Description = null;
                        repository.AddOrUpdate(contentType);
                    }

@@ -775,7 +778,7 @@ namespace Umbraco.Core.Services
                UpdateContentXmlStructure(asArray.Cast<IContentTypeBase>().ToArray());
            }
            SavedContentType.RaiseEvent(new SaveEventArgs<IContentType>(asArray, false), this);
-	        Audit(AuditType.Save, string.Format("Save ContentTypes performed by user"), userId, -1);
+            Audit(AuditType.Save, string.Format("Save ContentTypes performed by user"), userId, -1);
        }

        /// <summary>
@@ -786,8 +789,8 @@ namespace Umbraco.Core.Services
        /// <remarks>Deleting a <see cref="IContentType"/> will delete all the <see cref="IContent"/> objects based on this <see cref="IContentType"/></remarks>
        public void Delete(IContentType contentType, int userId = 0)
        {
-	        if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(contentType), this))
-				return;
+            if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(contentType), this))
+                return;

            using (new WriteLock(Locker))
            {
@@ -831,7 +834,7 @@ namespace Umbraco.Core.Services
            var asArray = contentTypes.ToArray();

            if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(asArray), this))
-				return;
+                return;

            using (new WriteLock(Locker))
            {
@@ -999,8 +1002,8 @@ namespace Umbraco.Core.Services
            var evtMsgs = EventMessagesFactory.Get();

            if (MovingMediaType.IsRaisedEventCancelled(
-                  new MoveEventArgs<IMediaType>(evtMsgs, new MoveEventInfo<IMediaType>(toMove, toMove.Path, containerId)),
-                  this))
+                new MoveEventArgs<IMediaType>(evtMsgs, new MoveEventInfo<IMediaType>(toMove, toMove.Path, containerId)),
+                this))
            {
                return Attempt.Fail(
                    new OperationStatus<MoveOperationStatusType>(
@@ -1042,8 +1045,8 @@ namespace Umbraco.Core.Services
            var evtMsgs = EventMessagesFactory.Get();

            if (MovingContentType.IsRaisedEventCancelled(
-                  new MoveEventArgs<IContentType>(evtMsgs, new MoveEventInfo<IContentType>(toMove, toMove.Path, containerId)),
-                  this))
+                new MoveEventArgs<IContentType>(evtMsgs, new MoveEventInfo<IContentType>(toMove, toMove.Path, containerId)),
+                this))
            {
                return Attempt.Fail(
                    new OperationStatus<MoveOperationStatusType>(
@@ -1173,8 +1176,8 @@ namespace Umbraco.Core.Services
        /// <param name="userId">Optional Id of the user saving the MediaType</param>
        public void Save(IMediaType mediaType, int userId = 0)
        {
-	        if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(mediaType), this))
-				return;
+            if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(mediaType), this))
+                return;

            using (new WriteLock(Locker))
            {
@@ -1183,6 +1186,8 @@ namespace Umbraco.Core.Services
                {
                    ValidateLocked(mediaType); // throws if invalid
                    mediaType.CreatorId = userId;
+                    if (mediaType.Description == string.Empty)
+                        mediaType.Description = null;
                    repository.AddOrUpdate(mediaType);
                    uow.Commit();

@@ -1192,7 +1197,7 @@ namespace Umbraco.Core.Services
            }

            SavedMediaType.RaiseEvent(new SaveEventArgs<IMediaType>(mediaType, false), this);
-	        Audit(AuditType.Save, string.Format("Save MediaType performed by user"), userId, mediaType.Id);
+            Audit(AuditType.Save, string.Format("Save MediaType performed by user"), userId, mediaType.Id);
        }

        /// <summary>
@@ -1205,7 +1210,7 @@ namespace Umbraco.Core.Services
            var asArray = mediaTypes.ToArray();

            if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(asArray), this))
-				return;
+                return;

            using (new WriteLock(Locker))
            {
@@ -1220,6 +1225,8 @@ namespace Umbraco.Core.Services
                    foreach (var mediaType in asArray)
                    {
                        mediaType.CreatorId = userId;
+                        if (mediaType.Description == string.Empty)
+                            mediaType.Description = null;
                        repository.AddOrUpdate(mediaType);
                    }

@@ -1231,7 +1238,7 @@ namespace Umbraco.Core.Services
            }

            SavedMediaType.RaiseEvent(new SaveEventArgs<IMediaType>(asArray, false), this);
-			Audit(AuditType.Save, string.Format("Save MediaTypes performed by user"), userId, -1);
+            Audit(AuditType.Save, string.Format("Save MediaTypes performed by user"), userId, -1);
        }

        /// <summary>
@@ -1242,8 +1249,8 @@ namespace Umbraco.Core.Services
        /// <remarks>Deleting a <see cref="IMediaType"/> will delete all the <see cref="IMedia"/> objects based on this <see cref="IMediaType"/></remarks>
        public void Delete(IMediaType mediaType, int userId = 0)
        {
-	        if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(mediaType), this))
-				return;
+            if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(mediaType), this))
+                return;
            using (new WriteLock(Locker))
            {
                _mediaService.DeleteMediaOfType(mediaType.Id, userId);
@@ -1275,7 +1282,7 @@ namespace Umbraco.Core.Services
            var asArray = mediaTypes.ToArray();

            if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(asArray), this))
-				return;
+                return;
            using (new WriteLock(Locker))
            {
                foreach (var mediaType in asArray)
@@ -1385,40 +1392,40 @@ namespace Umbraco.Core.Services
        /// </summary>
        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletingContentType;

-		/// <summary>
-		/// Occurs after Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletedContentType;
+        /// <summary>
+        /// Occurs after Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletedContentType;

-		/// <summary>
-		/// Occurs before Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletingMediaType;
+        /// <summary>
+        /// Occurs before Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletingMediaType;

-		/// <summary>
-		/// Occurs after Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletedMediaType;
+        /// <summary>
+        /// Occurs after Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletedMediaType;

        /// <summary>
        /// Occurs before Save
        /// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavingContentType;
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavingContentType;

        /// <summary>
        /// Occurs after Save
        /// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavedContentType;
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavedContentType;

-		/// <summary>
-		/// Occurs before Save
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavingMediaType;
+        /// <summary>
+        /// Occurs before Save
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavingMediaType;

-		/// <summary>
-		/// Occurs after Save
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavedMediaType;
+        /// <summary>
+        /// Occurs after Save
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavedMediaType;

        /// <summary>
        /// Occurs before Move
@@ -5,6 +5,7 @@ using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Persistence.UnitOfWork;

 namespace Umbraco.Core.Services
@@ -70,5 +71,20 @@ namespace Umbraco.Core.Services
            return toUpdate;

        }
+
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        public bool HasContainerInPath(string contentPath)
+        {
+            using (var uow = UowProvider.GetUnitOfWork())
+            {
+                // can use same repo for both content and media
+                var repository = RepositoryFactory.CreateContentTypeRepository(uow);
+                return repository.HasContainerInPath(contentPath);
+            }
+        }
    }
 }
@@ -361,8 +361,13 @@ namespace Umbraco.Core.Services
        /// <param name="userId">Id of the user issueing the save</param>
        public void Save(IDataTypeDefinition dataTypeDefinition, int userId = 0)
        {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
-				return;
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
+                return;
+
+            if (string.IsNullOrWhiteSpace(dataTypeDefinition.Name))
+            {
+                throw new ArgumentException("Cannot save datatype with empty name.");
+            }

            var uow = UowProvider.GetUnitOfWork();
            using (var repository = RepositoryFactory.CreateDataTypeDefinitionRepository(uow))
@@ -64,7 +64,7 @@ namespace Umbraco.Core.Services

        #region Static Queries

-        private readonly IQuery<IUmbracoEntity> _rootEntityQuery = Query<IUmbracoEntity>.Builder.Where(x => x.ParentId == -1);
+        private IQuery<IUmbracoEntity> _rootEntityQuery;

        #endregion

@@ -392,6 +392,12 @@ namespace Umbraco.Core.Services
        /// <returns>An enumerable list of <see cref="IUmbracoEntity"/> objects</returns>
        public virtual IEnumerable<IUmbracoEntity> GetRootEntities(UmbracoObjectTypes umbracoObjectType)
        {
+            //create it once if it is needed (no need for locking here)
+            if (_rootEntityQuery == null)
+            {
+                _rootEntityQuery = Query<IUmbracoEntity>.Builder.Where(x => x.ParentId == -1);
+            }
+
            var objectTypeId = umbracoObjectType.GetGuid();
            using (var repository = RepositoryFactory.CreateEntityRepository(UowProvider.GetUnitOfWork()))
            {
@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Xml;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -95,6 +96,19 @@ namespace Umbraco.Core.Services
    /// </summary>
    public interface IContentService : IService
    {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table based on the given path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        /// <remarks>
+        /// If -1 is passed, then this will return all content xml entries, otherwise will return all descendents from the path
+        /// </remarks>
+        IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords);
+
        /// <summary>
        /// This builds the Xml document used for the XML cache
        /// </summary>
@@ -245,7 +259,7 @@ namespace Umbraco.Core.Services
        [Obsolete("Use the overload with 'long' parameter types instead")]
        [EditorBrowsable(EditorBrowsableState.Never)]
        IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");

        /// <summary>
        /// Gets a collection of <see cref="IContent"/> objects by Parent Id
@@ -259,7 +273,7 @@ namespace Umbraco.Core.Services
        /// <param name="filter">Search text filter</param>
        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
        IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");

        /// <summary>
        /// Gets a collection of <see cref="IContent"/> objects by Parent Id
@@ -11,6 +11,13 @@ namespace Umbraco.Core.Services
    /// </summary>
    public interface IContentTypeService : IService
    {
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        bool HasContainerInPath(string contentPath);
+
        int CountContentTypes();
        int CountMediaTypes();

@@ -80,6 +80,7 @@ namespace Umbraco.Core.Services
        /// </param>
        void RebuildXmlStructures(params int[] contentTypeIds);

+        int CountNotTrashed(string contentTypeAlias = null);
        int Count(string contentTypeAlias = null);
        int CountChildren(int parentId, string contentTypeAlias = null);
        int CountDescendants(int parentId, string contentTypeAlias = null);
@@ -166,10 +167,26 @@ namespace Umbraco.Core.Services
        IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
            string orderBy, Direction orderDirection, bool orderBySystemField, string filter);

+        /// <summary>
+        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
+        /// </summary>
+        /// <param name="id">Id of the Parent to retrieve Children from</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
+        /// <param name="filter">Search text filter</param>
+        /// <param name="contentTypeFilter">A list of content type Ids to filter the list by</param>
+        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
+        IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter);
+
        [Obsolete("Use the overload with 'long' parameter types instead")]
        [EditorBrowsable(EditorBrowsableState.Never)]
        IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");

        /// <summary>
        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
@@ -183,7 +200,7 @@ namespace Umbraco.Core.Services
        /// <param name="filter">Search text filter</param>
        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
        IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");

        /// <summary>
        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -13,6 +14,15 @@ namespace Umbraco.Core.Services
    /// </summary>
    public interface IMemberService : IMembershipMemberService
    {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table
+        /// </summary>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>        
+        IEnumerable<XElement> GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords);
+
        /// <summary>
        /// Rebuilds all xml content in the cmsContentXml table for all documents
        /// </summary>
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Xml.Linq;
 using Umbraco.Core.Models;

@@ -6,6 +7,15 @@ namespace Umbraco.Core.Services
 {
    public interface IPackagingService : IService
    {
+        /// <summary>
+        /// This will fetch an Umbraco package file from the package repository and return the relative file path to the downloaded package file
+        /// </summary>
+        /// <param name="packageId"></param>
+        /// <param name="umbracoVersion"></param>
+        /// <param name="userId">The current user id performing the operation</param>
+        /// <returns></returns>
+        string FetchPackageFile(Guid packageId, Version umbracoVersion, int userId);
+
        /// <summary>
        /// Imports and saves package xml as <see cref="IContent"/>
        /// </summary>
@@ -142,19 +142,24 @@ namespace Umbraco.Core.Services
        /// <param name="userId">Optional Id of the user deleting the macro</param>
        public void Save(IMacro macro, int userId = 0)
        {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this)) 
-				return;
-	        
-			var uow = UowProvider.GetUnitOfWork();
-	        using (var repository = RepositoryFactory.CreateMacroRepository(uow))
-	        {
-		        repository.AddOrUpdate(macro);
-		        uow.Commit();
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this))
+                return;

-		        Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
-	        }
+            if (string.IsNullOrWhiteSpace(macro.Name))
+            {
+                throw new ArgumentException("Cannot save macro with empty name.");
+            }

-	        Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMacroRepository(uow))
+            {
+                repository.AddOrUpdate(macro);
+                uow.Commit();
+
+                Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
+            }
+
+            Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
        }

        ///// <summary>
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`<processing xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform" xdt:Transform="SetAttributes(preserveExifMetaData)" preserveExifMetaData="true" />`