Bump version to 7.5.14

debug logging when there is a distributed calls config issue

.gitignore

@@ -142,3 +142,5 @@ build/ui-docs.zip
 build/csharp-docs.zip
 build/msbuild.log
 .vs/
+
+build/tools/

README.md

@@ -1,3 +1,5 @@
+[![Build status](https://ci.appveyor.com/api/projects/status/ikpgqxiw9v8opltv/branch/dev-v7?svg=true)](https://ci.appveyor.com/project/Umbraco/umbraco-cms-hs8dx/branch/dev-v7)
+
 Umbraco CMS
 ===========
 The friendliest, most flexible and fastest growing ASP.NET CMS used by more than 390,000 websites worldwide: [https://umbraco.com](https://umbraco.com)

build/Build.bat

@@ -49,10 +49,52 @@ REM This is necessary because SETLOCAL is on in InstallGit.cmd so that one might
 REM but the path setting is lost due to SETLOCAL 
 path=C:\Program Files (x86)\Git\cmd;C:\Program Files\Git\cmd;%PATH%
 
+SET toolsFolder=%CD%\tools\
+IF NOT EXIST "%toolsFolder%" (
+	MD tools
+)
+
+SET nuGetExecutable=%CD%\tools\nuget.exe
+IF NOT EXIST "%nuGetExecutable%" (
+	ECHO Getting NuGet so we can fetch some tools
+	ECHO Downloading https://dist.nuget.org/win-x86-commandline/latest/nuget.exe to %nuGetExecutable%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', '%nuGetExecutable%')"
+)
+
+:: We need 7za.exe for BuildBelle.bat
+IF NOT EXIST "%toolsFolder%7za.exe" (
+	ECHO 7zip not found - fetching now
+	"%nuGetExecutable%" install 7-Zip.CommandLine -OutputDirectory tools -Verbosity quiet
+)
+
+:: We need vswhere.exe for VS2017+
+IF NOT EXIST "%toolsFolder%vswhere.exe" (
+	ECHO vswhere not found - fetching now
+	"%nuGetExecutable%" install vswhere -OutputDirectory tools -Verbosity quiet
+)
+
+:: Put 7za.exe and vswhere.exe in a predictable path (not version specific)
+FOR /f "delims=" %%A in ('dir "%toolsFolder%7-Zip.CommandLine.*" /b') DO SET "sevenZipExePath=%toolsFolder%%%A\"
+MOVE "%sevenZipExePath%tools\7za.exe" "%toolsFolder%7za.exe"
+
+FOR /f "delims=" %%A in ('dir "%toolsFolder%vswhere.*" /b') DO SET "vswhereExePath=%toolsFolder%%%A\"
+MOVE "%vswhereExePath%tools\vswhere.exe" "%toolsFolder%vswhere.exe"
+	
 ECHO.
 ECHO Making sure we have a web.config
 IF NOT EXIST %CD%\..\src\Umbraco.Web.UI\web.config COPY %CD%\..\src\Umbraco.Web.UI\web.Template.config %CD%\..\src\Umbraco.Web.UI\web.config
 
+for /f "usebackq tokens=1* delims=: " %%i in (`"%CD%\tools\vswhere.exe" -latest -requires Microsoft.Component.MSBuild`) do (
+  if /i "%%i"=="installationPath" set InstallDir=%%j
+)
+
+SET VSWherePath="%InstallDir%\MSBuild"
+ECHO.
+ECHO Visual Studio is installed in: %InstallDir%
+
+SET MSBUILDPATH=C:\Program Files (x86)\MSBuild\14.0\Bin
+SET MSBUILD="%MSBUILDPATH%\MsBuild.exe"
+
 ECHO.
 ECHO.
 ECHO Performing MSBuild and producing Umbraco binaries zip files
@@ -60,7 +102,7 @@ ECHO This takes a few minutes and logging is set to report warnings
 ECHO and errors only so it might seems like nothing is happening for a while. 
 ECHO You can check the msbuild.log file for progress.
 ECHO.
-%windir%\Microsoft.NET\Framework\v4.0.30319\msbuild.exe "Build.proj" /p:BUILD_RELEASE=%release% /p:BUILD_COMMENT=%comment% /p:NugetPackagesDirectory=%nuGetFolder% /consoleloggerparameters:Summary;ErrorsOnly;WarningsOnly /fileLogger
+%MSBUILD% "Build.proj" /p:BUILD_RELEASE=%release% /p:BUILD_COMMENT=%comment% /p:NugetPackagesDirectory=%nuGetFolder% /p:VSWherePath=%VSWherePath%
 IF ERRORLEVEL 1 GOTO :error
 
 ECHO.

build/BuildBelle.bat

@@ -1,33 +1,59 @@
 @ECHO OFF
 SETLOCAL
+	:: SETLOCAL is on, so changes to the path not persist to the actual user's path
 
-SET release=%1
-ECHO Installing Npm NuGet Package
-
-SET nuGetFolder=%CD%\..\src\packages\
-ECHO Configured packages folder: %nuGetFolder%
+SET toolsFolder=%CD%\tools\
 ECHO Current folder: %CD%
 
-%CD%\..\src\.nuget\NuGet.exe install Npm.js -OutputDirectory %nuGetFolder%  -Verbosity quiet
+SET nodeFileName=node-v6.9.1-win-x86.7z
+SET nodeExtractFolder=%toolsFolder%node.js.691
 
-for /f "delims=" %%A in ('dir %nuGetFolder%node.js.* /b') do set "nodePath=%nuGetFolder%%%A\"
-for /f "delims=" %%A in ('dir %nuGetFolder%npm.js.* /b') do set "npmPath=%nuGetFolder%%%A\tools\"
+IF NOT EXIST "%nodeExtractFolder%" (
+	ECHO Downloading http://nodejs.org/dist/v6.9.1/%nodeFileName% to %toolsFolder%%nodeFileName%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('http://nodejs.org/dist/v6.9.1/%nodeFileName%', '%toolsFolder%%nodeFileName%')"
+	ECHO Extracting %nodeFileName% to %nodeExtractFolder%
+	"%toolsFolder%\7za.exe" x "%toolsFolder%\%nodeFileName%" -o"%nodeExtractFolder%" -aos > nul
+)
+FOR /f "delims=" %%A in ('dir "%nodeExtractFolder%\node*" /b') DO SET "nodePath=%nodeExtractFolder%\%%A"
 
-ECHO Adding Npm and Node to path 
-REM SETLOCAL is on, so changes to the path not persist to the actual user's path
-PATH=%npmPath%;%nodePath%;%PATH%
 
-SET buildFolder=%CD%
+SET nuGetExecutable=%CD%\tools\nuget.exe
+IF NOT EXIST "%nuGetExecutable%" (
+	ECHO Downloading https://dist.nuget.org/win-x86-commandline/latest/nuget.exe to %nuGetExecutable%
+	powershell -Command "(New-Object Net.WebClient).DownloadFile('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', '%nuGetExecutable%')"
+)
 
-ECHO Change directory to %CD%\..\src\Umbraco.Web.UI.Client\
-CD %CD%\..\src\Umbraco.Web.UI.Client\
+SET drive=%CD:~0,2%
+SET nuGetFolder=%drive%\packages\
+FOR /f "delims=" %%A in ('dir "%nuGetFolder%npm.*" /b') DO SET "npmPath=%nuGetFolder%%%A\"
 
-ECHO Do npm install and the grunt build of Belle
-call npm cache clean --quiet
-call npm install --quiet
-call npm install -g grunt-cli --quiet
-call npm install -g bower --quiet
-call grunt build --buildversion=%release%
+IF [%npmPath%] == [] GOTO :installnpm 
+IF NOT [%npmPath%] == [] GOTO :build
 
-ECHO Move back to the build folder
-CD %buildFolder% 
+:installnpm
+	ECHO Downloading npm
+	ECHO Configured packages folder: %nuGetFolder%	
+	ECHO Installing Npm NuGet Package
+	"%nuGetExecutable%" install Npm -OutputDirectory %nuGetFolder% -Verbosity detailed
+	REM Ensures that we look for the just downloaded NPM, not whatever the user has installed on their machine
+	FOR /f "delims=" %%A in ('dir %nuGetFolder%npm.* /b') DO SET "npmPath=%nuGetFolder%%%A\"
+	GOTO :build
+
+:build
+    ECHO Adding Npm and Node to path 
+    REM SETLOCAL is on, so changes to the path not persist to the actual user's path
+    PATH="%npmPath%";"%nodePath%";%PATH%
+    SET buildFolder=%CD%
+
+    ECHO Change directory to %CD%\..\src\Umbraco.Web.UI.Client\
+    CD %CD%\..\src\Umbraco.Web.UI.Client\
+
+    ECHO Do npm install and the grunt build of Belle
+    call npm cache clean --quiet
+    call npm install --quiet
+    call npm install -g grunt-cli --quiet
+    call npm install -g bower --quiet
+    call grunt build --buildversion=%release%
+
+    ECHO Move back to the build folder
+    CD "%buildFolder%"

build/NuSpecs/UmbracoCms.Core.nuspec

@@ -32,7 +32,7 @@
 	  <!-- AutoMapper can not be updated due to: https://github.com/AutoMapper/AutoMapper/issues/373#issuecomment-127644405 -->
       <dependency id="AutoMapper" version="[3.3.1, 4.0.0)" />
       <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Examine" version="[0.1.81, 1.0.0)" />
+      <dependency id="Examine" version="[0.1.82, 1.0.0)" />
       <dependency id="ImageProcessor" version="[2.5.2, 3.0.0)" />
       <dependency id="ImageProcessor.Web" version="[4.8.2, 5.0.0)" />
       <dependency id="semver" version="[1.1.2, 2.0.0)" />

build/NuSpecs/UmbracoCms.nuspec

@@ -17,8 +17,8 @@
     <dependencies>
       <dependency id="UmbracoCms.Core" version="[$version$]" />
       <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Umbraco.ModelsBuilder" version="[3.0.5, 4.0.0)" />
-      <dependency id="ImageProcessor.Web.Config" version="[2.3.0, 3.0.0)" />	  
+      <dependency id="Umbraco.ModelsBuilder" version="[3.0.7, 4.0.0)" />
+      <dependency id="ImageProcessor.Web.Config" version="[2.3.0, 3.0.0)" />
     </dependencies>
   </metadata>
   <files>

build/NuSpecs/tools/Dashboard.config.install.xdt

@@ -41,16 +41,6 @@
         views/dashboard/developer/examinemanagement.html
       </control>
     </tab>
-    <tab caption="Health Check" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/healthcheck.html
-      </control>
-    </tab>
-    <tab caption="Redirect URL Management" xdt:Transform="InsertIfMissing" xdt:Locator="Match(caption)">
-      <control>
-        views/dashboard/developer/redirecturls.html
-      </control>
-    </tab>	
   </section>
   
   <section alias="StartupMediaDashboardSection" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
@@ -80,4 +70,26 @@
 	<tab caption="Change Password" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
 	<tab caption="Last Edits" xdt:Locator="Match(caption)" xdt:Transform="Remove" />
   </section>
+
+  <section alias="RedirectUrlManagement" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>content</area>
+    </areas>
+    <tab caption="Redirect URL Management">
+      <control>
+        views/dashboard/developer/redirecturls.html
+      </control>
+    </tab>
+  </section>
+
+  <section alias="UmbracoHealthCheck" xdt:Locator="Match(alias)" xdt:Transform="InsertIfMissing">
+    <areas>
+      <area>developer</area>
+    </areas>
+    <tab caption="Health Check">
+      <control>
+        views/dashboard/developer/healthcheck.html
+      </control>
+    </tab>
+  </section>
 </dashBoard>

build/UmbracoVersion.txt

@@ -1,2 +1,2 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
-7.5.9
+7.5.14

src/SolutionInfo.cs

@@ -11,5 +11,5 @@ using System.Resources;
 
 [assembly: AssemblyVersion("1.0.*")]
 
-[assembly: AssemblyFileVersion("7.5.9")]
-[assembly: AssemblyInformationalVersion("7.5.9")]
+[assembly: AssemblyFileVersion("7.5.14")]
+[assembly: AssemblyInformationalVersion("7.5.14")]

src/Umbraco.Core/Cache/DeepCloneRuntimeCacheProvider.cs

@@ -26,6 +26,9 @@ namespace Umbraco.Core.Cache
 
         public DeepCloneRuntimeCacheProvider(IRuntimeCacheProvider innerProvider)
         {
+            if (innerProvider.GetType() == typeof(DeepCloneRuntimeCacheProvider))
+                throw new InvalidOperationException("A " + typeof(DeepCloneRuntimeCacheProvider) + " cannot wrap another instance of " + typeof(DeepCloneRuntimeCacheProvider));
+
             InnerProvider = innerProvider;
         }
 
@@ -105,9 +108,11 @@ namespace Umbraco.Core.Cache
                 var value = result.Value; // force evaluation now - this may throw if cacheItem throws, and then nothing goes into cache
                 if (value == null) return null; // do not store null values (backward compat)
 
+                //Clone/reset to go into the cache
                 return CheckCloneableAndTracksChanges(value);
             }, timeout, isSliding, priority, removedCallback, dependentFiles);
 
+            //Clone/reset to go out of the cache
             return CheckCloneableAndTracksChanges(cached);
         }
 

src/Umbraco.Core/Configuration/ContentXmlStorage.cs

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Configuration
+{
+    internal enum ContentXmlStorage
+    {
+        Default,
+        AspNetTemp,
+        EnvironmentTemp
+    }
+}

src/Umbraco.Core/Configuration/CoreDebug.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace Umbraco.Core.Configuration
+{
+    internal static class CoreDebugExtensions
+    {
+        private static CoreDebug _coreDebug;
+
+        public static CoreDebug CoreDebug(this UmbracoConfig config)
+        {
+            return _coreDebug ?? (_coreDebug = new CoreDebug());
+        }
+    }
+
+    internal class CoreDebug
+    {
+        public CoreDebug()
+        {
+            var appSettings = System.Configuration.ConfigurationManager.AppSettings;
+            DumpOnTimeoutThreadAbort = string.Equals("true", appSettings["Umbraco.CoreDebug.DumpOnTimeoutThreadAbort"], StringComparison.OrdinalIgnoreCase);
+        }
+
+        // when true, the Logger creates a minidump of w3wp in ~/App_Data/MiniDump whenever it logs
+        // an error due to a ThreadAbortException that is due to a timeout.
+        public bool DumpOnTimeoutThreadAbort { get; private set; }
+    }
+}

src/Umbraco.Core/Configuration/GlobalSettings.cs

@@ -520,12 +520,25 @@ namespace Umbraco.Core.Configuration
         }
 
         internal static bool ContentCacheXmlStoredInCodeGen
+        {
+            get { return ContentCacheXmlStorageLocation == ContentXmlStorage.AspNetTemp; }
+        }
+
+        internal static ContentXmlStorage ContentCacheXmlStorageLocation
         {
             get
             {
-                //defaults to false
-                return ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp") 
-                    && bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]); //default to false
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLStorage"))
+                {
+                    return Enum<ContentXmlStorage>.Parse(ConfigurationManager.AppSettings["umbracoContentXMLStorage"]);
+                }
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp"))
+                {
+                    return bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]) 
+                        ? ContentXmlStorage.AspNetTemp 
+                        : ContentXmlStorage.Default;
+                }
+                return ContentXmlStorage.Default;
             }
         }
 

src/Umbraco.Core/Configuration/UmbracoSettings/IRepository.cs

@@ -8,6 +8,7 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
         Guid Id { get;  }
         string RepositoryUrl { get; }
         string WebServiceUrl { get; }
-        bool HasCustomWebServiceUrl { get; }
+        bool HasCustomWebServiceUrl { get; }
+        string RestApiUrl { get; }
     }
 }

src/Umbraco.Core/Configuration/UmbracoSettings/RepositoryConfigExtensions.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Linq;
+
+namespace Umbraco.Core.Configuration.UmbracoSettings
+{
+    public static class RepositoryConfigExtensions
+    {
+        //Our package repo
+        private static readonly Guid RepoGuid = new Guid("65194810-1f85-11dd-bd0b-0800200c9a66");
+
+        public static IRepository GetDefault(this IRepositoriesSection repos)
+        {
+            var found = repos.Repositories.FirstOrDefault(x => x.Id == RepoGuid);
+            if (found == null)
+                throw new InvalidOperationException("No default package repository found with id " + RepoGuid);
+            return found;
+        }
+    }
+}

src/Umbraco.Core/Configuration/UmbracoSettings/RepositoryElement.cs

@@ -38,9 +38,16 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
             get
             {
                 var prop = Properties["webserviceurl"];
-                var repoUrl = this[prop] as ConfigurationElement;
-                return (repoUrl != null && repoUrl.ElementInformation.IsPresent);
+                return (string) prop.DefaultValue != (string) this[prop];                
             }
         }
+
+        [ConfigurationProperty("restapiurl", DefaultValue = "https://our.umbraco.org/webapi/packages/v1")]
+        public string RestApiUrl
+        {
+            get { return (string)base["restapiurl"]; }
+            set { base["restapiurl"] = value; }
+        }
+        
     }
 }

src/Umbraco.Core/Configuration/UmbracoVersion.cs

@@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration
 {
     public class UmbracoVersion
     {
-        private static readonly Version Version = new Version("7.5.9");
+        private static readonly Version Version = new Version("7.5.14");
 
         /// <summary>
         /// Gets the current version of Umbraco.

src/Umbraco.Core/Diagnostics/MiniDump.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+using Umbraco.Core.IO;
+
+namespace Umbraco.Core.Diagnostics
+{
+    // taken from https://blogs.msdn.microsoft.com/dondu/2010/10/24/writing-minidumps-in-c/
+    // and https://blogs.msdn.microsoft.com/dondu/2010/10/31/writing-minidumps-from-exceptions-in-c/
+    // which itself got it from http://blog.kalmbach-software.de/2008/12/13/writing-minidumps-in-c/
+
+    internal static class MiniDump
+    {
+        private static readonly object LockO = new object();
+
+        [Flags]
+        public enum Option : uint
+        {
+            // From dbghelp.h:
+            Normal = 0x00000000,
+            WithDataSegs = 0x00000001,
+            WithFullMemory = 0x00000002,
+            WithHandleData = 0x00000004,
+            FilterMemory = 0x00000008,
+            ScanMemory = 0x00000010,
+            WithUnloadedModules = 0x00000020,
+            WithIndirectlyReferencedMemory = 0x00000040,
+            FilterModulePaths = 0x00000080,
+            WithProcessThreadData = 0x00000100,
+            WithPrivateReadWriteMemory = 0x00000200,
+            WithoutOptionalData = 0x00000400,
+            WithFullMemoryInfo = 0x00000800,
+            WithThreadInfo = 0x00001000,
+            WithCodeSegs = 0x00002000,
+            WithoutAuxiliaryState = 0x00004000,
+            WithFullAuxiliaryState = 0x00008000,
+            WithPrivateWriteCopyMemory = 0x00010000,
+            IgnoreInaccessibleMemory = 0x00020000,
+            ValidTypeFlags = 0x0003ffff,
+        }
+
+        //typedef struct _MINIDUMP_EXCEPTION_INFORMATION {
+        //    DWORD ThreadId;
+        //    PEXCEPTION_POINTERS ExceptionPointers;
+        //    BOOL ClientPointers;
+        //} MINIDUMP_EXCEPTION_INFORMATION, *PMINIDUMP_EXCEPTION_INFORMATION;
+        [StructLayout(LayoutKind.Sequential, Pack = 4)]  // Pack=4 is important! So it works also for x64!
+        public struct MiniDumpExceptionInformation
+        {
+            public uint ThreadId;
+            public IntPtr ExceptionPointers;
+            [MarshalAs(UnmanagedType.Bool)]
+            public bool ClientPointers;
+        }
+
+        //BOOL
+        //WINAPI
+        //MiniDumpWriteDump(
+        //    __in HANDLE hProcess,
+        //    __in DWORD ProcessId,
+        //    __in HANDLE hFile,
+        //    __in MINIDUMP_TYPE DumpType,
+        //    __in_opt PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam,
+        //    __in_opt PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam,
+        //    __in_opt PMINIDUMP_CALLBACK_INFORMATION CallbackParam
+        //    );
+
+        // Overload requiring MiniDumpExceptionInformation
+        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
+        private static extern bool MiniDumpWriteDump(IntPtr hProcess, uint processId, SafeHandle hFile, uint dumpType, ref MiniDumpExceptionInformation expParam, IntPtr userStreamParam, IntPtr callbackParam);
+
+        // Overload supporting MiniDumpExceptionInformation == NULL
+        [DllImport("dbghelp.dll", EntryPoint = "MiniDumpWriteDump", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
+        private static extern bool MiniDumpWriteDump(IntPtr hProcess, uint processId, SafeHandle hFile, uint dumpType, IntPtr expParam, IntPtr userStreamParam, IntPtr callbackParam);
+
+        [DllImport("kernel32.dll", EntryPoint = "GetCurrentThreadId", ExactSpelling = true)]
+        private static extern uint GetCurrentThreadId();
+
+        private static bool Write(SafeHandle fileHandle, Option options, bool withException = false)
+        {
+            var currentProcess = Process.GetCurrentProcess();
+            var currentProcessHandle = currentProcess.Handle;
+            var currentProcessId = (uint)currentProcess.Id;
+
+            MiniDumpExceptionInformation exp;
+
+            exp.ThreadId = GetCurrentThreadId();
+            exp.ClientPointers = false;
+            exp.ExceptionPointers = IntPtr.Zero;
+
+            if (withException)
+                exp.ExceptionPointers = Marshal.GetExceptionPointers();
+
+            var bRet = exp.ExceptionPointers == IntPtr.Zero
+                ? MiniDumpWriteDump(currentProcessHandle, currentProcessId, fileHandle, (uint) options, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero)
+                : MiniDumpWriteDump(currentProcessHandle, currentProcessId, fileHandle, (uint) options, ref exp, IntPtr.Zero, IntPtr.Zero);
+
+            return bRet;
+        }
+
+        public static bool Dump(Option options = Option.WithFullMemory, bool withException = false)
+        {
+            lock (LockO)
+            {
+                // work around "stack trace is not available while minidump debugging",
+                // by making sure a local var (that we can inspect) contains the stack trace.
+                // getting the call stack before it is unwound would require a special exception
+                // filter everywhere in our code = not!
+                var stacktrace = withException ? Environment.StackTrace : string.Empty;
+
+                var filepath = IOHelper.MapPath("~/App_Data/MiniDump");
+                if (Directory.Exists(filepath) == false)
+                    Directory.CreateDirectory(filepath);
+
+                var filename = Path.Combine(filepath, string.Format("{0:yyyyMMddTHHmmss}.{1}.dmp", DateTime.UtcNow, Guid.NewGuid().ToString("N").Substring(0, 4)));
+                using (var stream = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.Write))
+                {
+                    return Write(stream.SafeFileHandle, options, withException);
+                }
+            }
+        }
+
+        public static bool OkToDump()
+        {
+            lock (LockO)
+            {
+                var filepath = IOHelper.MapPath("~/App_Data/MiniDump");
+                if (Directory.Exists(filepath) == false) return true;
+                var count = Directory.GetFiles(filepath, "*.dmp").Length;
+                return count < 8;
+            }
+        }
+    }
+}

src/Umbraco.Core/Exceptions/ConnectionException.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace Umbraco.Core.Exceptions
+{
+    internal class ConnectionException : Exception
+    {
+        public ConnectionException(string message, Exception innerException) : base(message, innerException)
+        {
+
+        }
+    }
+}

src/Umbraco.Core/IO/IFileSystem.cs

@@ -7,39 +7,144 @@ namespace Umbraco.Core.IO
     //TODO: There is no way to create a directory here without creating a file in a directory and then deleting it
     //TODO: Should probably implement a rename?
 
-	public interface IFileSystem
+    /// <summary>
+    /// Provides methods allowing the manipulation of files within an Umbraco application.
+    /// </summary>
+    public interface IFileSystem
     {
+        /// <summary>
+        /// Gets all directories matching the given path.
+        /// </summary>
+        /// <param name="path">The path to the directories.</param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched directories.
+        /// </returns>
         IEnumerable<string> GetDirectories(string path);
 
+        /// <summary>
+        /// Deletes the specified directory.
+        /// </summary>
+        /// <param name="path">The name of the directory to remove.</param>
         void DeleteDirectory(string path);
 
+        /// <summary>
+        /// Deletes the specified directory and, if indicated, any subdirectories and files in the directory.
+        /// </summary>
+        /// <remarks>Azure blob storage has no real concept of directories so deletion is always recursive.</remarks>
+        /// <param name="path">The name of the directory to remove.</param>
+        /// <param name="recursive">Whether to remove directories, subdirectories, and files in path.</param>
         void DeleteDirectory(string path, bool recursive);
 
+        /// <summary>
+        /// Determines whether the specified directory exists.
+        /// </summary>
+        /// <param name="path">The directory to check.</param>
+        /// <returns>
+        /// <c>True</c> if the directory exists and the user has permission to view it; otherwise <c>false</c>.
+        /// </returns>
         bool DirectoryExists(string path);
-        
+
+        /// <summary>
+        /// Adds a file to the file system.
+        /// </summary>
+        /// <param name="path">The path to the given file.</param>
+        /// <param name="stream">The <see cref="Stream"/> containing the file contents.</param>
         void AddFile(string path, Stream stream);
 
+        /// <summary>
+        /// Adds a file to the file system.
+        /// </summary>
+        /// <param name="path">The path to the given file.</param>
+        /// <param name="stream">The <see cref="Stream"/> containing the file contents.</param>
+        /// <param name="overrideIfExists">Whether to override the file if it already exists.</param>
         void AddFile(string path, Stream stream, bool overrideIfExists);
 
+        /// <summary>
+        /// Gets all files matching the given path.
+        /// </summary>
+        /// <param name="path">The path to the files.</param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched files.
+        /// </returns>
         IEnumerable<string> GetFiles(string path);
 
+        /// <summary>
+        /// Gets all files matching the given path and filter.
+        /// </summary>
+        /// <param name="path">The path to the files.</param>
+        /// <param name="filter">A filter that allows the querying of file extension. <example>*.jpg</example></param>
+        /// <returns>
+        /// The <see cref="IEnumerable{String}"/> representing the matched files.
+        /// </returns>
         IEnumerable<string> GetFiles(string path, string filter);
 
+        /// <summary>
+        /// Gets a <see cref="Stream"/> representing the file at the gieven path.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="Stream"/>.
+        /// </returns>
         Stream OpenFile(string path);
 
+        /// <summary>
+        /// Deletes the specified file.
+        /// </summary>
+        /// <param name="path">The name of the file to remove.</param>
         void DeleteFile(string path);
 
+        /// <summary>
+        /// Determines whether the specified file exists.
+        /// </summary>
+        /// <param name="path">The file to check.</param>
+        /// <returns>
+        /// <c>True</c> if the file exists and the user has permission to view it; otherwise <c>false</c>.
+        /// </returns>
         bool FileExists(string path);
 
-
+        /// <summary>
+        /// Returns the application relative path to the file.
+        /// </summary>
+        /// <param name="fullPathOrUrl">The full path or url.</param>
+        /// <returns>
+        /// The <see cref="string"/> representing the relative path.
+        /// </returns>
         string GetRelativePath(string fullPathOrUrl);
 
+        /// <summary>
+        /// Gets the full qualified path to the file.
+        /// </summary>
+        /// <param name="path">The file to return the full path for.</param>
+        /// <returns>
+        /// The <see cref="string"/> representing the full path.
+        /// </returns>
         string GetFullPath(string path);
 
+        /// <summary>
+        /// Returns the application relative url to the file.
+        /// </summary>
+        /// <param name="path">The path to return the url for.</param>
+        /// <returns>
+        /// <see cref="string"/> representing the relative url.
+        /// </returns>
         string GetUrl(string path);
 
+        /// <summary>
+        /// Gets the last modified date/time of the file, expressed as a UTC value.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="DateTimeOffset"/>.
+        /// </returns>
         DateTimeOffset GetLastModified(string path);
 
+        /// <summary>
+        /// Gets the created date/time of the file, expressed as a UTC value.
+        /// </summary>
+        /// <param name="path">The path to the file.</param>
+        /// <returns>
+        /// <see cref="DateTimeOffset"/>.
+        /// </returns>
         DateTimeOffset GetCreated(string path);
     }
 }

src/Umbraco.Core/IO/PhysicalFileSystem.cs

@@ -122,7 +122,7 @@ namespace Umbraco.Core.IO
             Directory.CreateDirectory(Path.GetDirectoryName(fullPath)); // ensure it exists
 
             if (stream.CanSeek)
-                stream.Seek(0, 0);
+                stream.Seek(0, SeekOrigin.Begin);
 
             using (var destination = (Stream)File.Create(fullPath))
                 stream.CopyTo(destination);

src/Umbraco.Core/IO/SystemFiles.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -72,15 +73,28 @@ namespace Umbraco.Core.IO
         {
             get
             {
-                if (GlobalSettings.ContentCacheXmlStoredInCodeGen && SystemUtilities.GetCurrentTrustLevel() == AspNetHostingPermissionLevel.Unrestricted)
-                {
-                    return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                switch (GlobalSettings.ContentCacheXmlStorageLocation)
+                {                    
+                    case ContentXmlStorage.AspNetTemp:
+                        return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                    case ContentXmlStorage.EnvironmentTemp:
+                        var appDomainHash = HttpRuntime.AppDomainAppId.ToSHA1();
+                        var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "UmbracoXml",
+                            //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back
+                            // to worker A again, in theory the %temp%  folder should already be empty but we really want to make sure that its not
+                            // utilizing an old path
+                            appDomainHash);
+                        return Path.Combine(cachePath, "umbraco.config");
+                    case ContentXmlStorage.Default:
+                        return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
+                    default:
+                        throw new ArgumentOutOfRangeException();
                 }
-                return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
             }
         }
 
         [Obsolete("Use GlobalSettings.ContentCacheXmlStoredInCodeGen instead")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         internal static bool ContentCacheXmlStoredInCodeGen
         {
             get { return GlobalSettings.ContentCacheXmlStoredInCodeGen; }

src/Umbraco.Core/Logging/Logger.cs

@@ -2,10 +2,13 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Web;
 using log4net;
 using log4net.Config;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Diagnostics;
 
 namespace Umbraco.Core.Logging
 {
@@ -56,18 +59,72 @@ namespace Umbraco.Core.Logging
 		internal ILog LoggerFor(object getTypeFromInstance)
 		{
 			if (getTypeFromInstance == null) throw new ArgumentNullException("getTypeFromInstance");
-			
+
 			return LogManager.GetLogger(getTypeFromInstance.GetType());
 		}
-        
+
 		public void Error(Type callingType, string message, Exception exception)
 		{
-			var logger = LogManager.GetLogger(callingType);
-			if (logger != null)
-				logger.Error((message), exception);
+            var logger = LogManager.GetLogger(callingType);
+		    if (logger == null) return;
+
+		    var dump = false;
+
+		    if (IsTimeoutThreadAbortException(exception))
+		    {
+		        message += "\r\nThe thread has been aborted, because the request has timed out.";
+
+                // dump if configured, or if stacktrace contains Monitor.ReliableEnter
+		        dump = UmbracoConfig.For.CoreDebug().DumpOnTimeoutThreadAbort || IsMonitorEnterThreadAbortException(exception);
+
+                // dump if it is ok to dump (might have a cap on number of dump...)
+		        dump &= MiniDump.OkToDump();
+		    }
+
+            if (dump)
+		    {
+                try
+                {
+                    var dumped = MiniDump.Dump(withException: true);
+                    message += dumped
+                        ? "\r\nA minidump was created in App_Data/MiniDump"
+                        : "\r\nFailed to create a minidump";
+                }
+                catch (Exception e)
+                {
+                    message += string.Format("\r\nFailed to create a minidump ({0}: {1})", e.GetType().FullName, e.Message);
+                }
+            }
+
+            logger.Error(message, exception);
 		}
 
-		public void Warn(Type callingType, string message, params Func<object>[] formatItems)
+        private static bool IsMonitorEnterThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            var stacktrace = abort.StackTrace;
+            return stacktrace.Contains("System.Threading.Monitor.ReliableEnter");
+        }
+
+        private static bool IsTimeoutThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            if (abort.ExceptionState == null) return false;
+
+            var stateType = abort.ExceptionState.GetType();
+            if (stateType.FullName != "System.Web.HttpApplication+CancelModuleException") return false;
+
+            var timeoutField = stateType.GetField("_timeout", BindingFlags.Instance | BindingFlags.NonPublic);
+            if (timeoutField == null) return false;
+
+            return (bool) timeoutField.GetValue(abort.ExceptionState);
+        }
+
+        public void Warn(Type callingType, string message, params Func<object>[] formatItems)
 		{
 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;
@@ -82,7 +139,7 @@ namespace Umbraco.Core.Logging
 			if (showHttpTrace && HttpContext.Current != null)
 			{
 				HttpContext.Current.Trace.Warn(callingType.Name, string.Format(message, formatItems.Select(x => x.Invoke()).ToArray()));
-			}	
+			}
 
 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;
@@ -99,7 +156,7 @@ namespace Umbraco.Core.Logging
             var logger = LogManager.GetLogger(callingType);
             if (logger == null || logger.IsWarnEnabled == false) return;
             var executedParams = formatItems.Select(x => x.Invoke()).ToArray();
-            logger.WarnFormat((message) + ". Exception: " + e, executedParams);		
+            logger.WarnFormat((message) + ". Exception: " + e, executedParams);
 		}
 
 		/// <summary>

src/Umbraco.Core/Media/ImageHelper.cs

@@ -227,35 +227,37 @@ namespace Umbraco.Core.Media
                     }
 
                     // Set compresion ratio to 90%
-                    var ep = new EncoderParameters();
-                    ep.Param[0] = new EncoderParameter(Encoder.Quality, 90L);
-
-                    // Save the new image using the dimensions of the image
-                    var predictableThumbnailName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", maxWidthHeight.ToString(CultureInfo.InvariantCulture));
-                    var predictableThumbnailNameJpg = predictableThumbnailName.Substring(0, predictableThumbnailName.LastIndexOf(".", StringComparison.Ordinal)) + ".jpg";
-                    using (var ms = new MemoryStream())
+                    string newFileName;
+                    using (var ep = new EncoderParameters())
                     {
-                        bp.Save(ms, codec, ep);
-                        ms.Seek(0, 0);
+                        ep.Param[0] = new EncoderParameter(Encoder.Quality, 90L);
 
-                        fs.AddFile(predictableThumbnailName, ms);
-                        fs.AddFile(predictableThumbnailNameJpg, ms);
-                    }
+                        // Save the new image using the dimensions of the image
+                        var predictableThumbnailName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", maxWidthHeight.ToString(CultureInfo.InvariantCulture));
+                        var predictableThumbnailNameJpg = predictableThumbnailName.Substring(0, predictableThumbnailName.LastIndexOf(".", StringComparison.Ordinal)) + ".jpg";
+                        using (var ms = new MemoryStream())
+                        {
+                            bp.Save(ms, codec, ep);
+                        ms.Seek(0, SeekOrigin.Begin);
 
-                    // TODO: Remove this, this is ONLY here for backwards compatibility but it is essentially completely unusable see U4-5385
-                    var newFileName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", string.Format("{0}x{1}", widthTh, heightTh));
-                    using (var ms = new MemoryStream())
-                    {
-                        bp.Save(ms, codec, ep);
-                        ms.Seek(0, 0);
+                            fs.AddFile(predictableThumbnailName, ms);
+                            fs.AddFile(predictableThumbnailNameJpg, ms);
+                        }
 
-                        fs.AddFile(newFileName, ms);
+                        // TODO: Remove this, this is ONLY here for backwards compatibility but it is essentially completely unusable see U4-5385
+                        newFileName = thumbnailFileName.Replace("UMBRACOSYSTHUMBNAIL", string.Format("{0}x{1}", widthTh, heightTh));
+                        using (var ms = new MemoryStream())
+                        {
+                            bp.Save(ms, codec, ep);
+                        ms.Seek(0, SeekOrigin.Begin);
+
+                            fs.AddFile(newFileName, ms);
+                        }
                     }
 
                     return new ResizedImage(widthTh, heightTh, newFileName);
                 }
             }
         }
-
     }
 }

src/Umbraco.Core/Models/Content.cs

@@ -274,6 +274,9 @@ namespace Umbraco.Core.Models
         /// </summary>
         public bool HasPublishedVersion { get { return PublishedVersionGuid != default(Guid); } }
 
+        [IgnoreDataMember]
+        internal DateTime PublishedDate { get; set; }
+
         /// <summary>
         /// Changes the Trashed state of the content object
         /// </summary>

src/Umbraco.Core/Models/ContentExtensions.cs

@@ -545,7 +545,7 @@ namespace Umbraco.Core.Models
             if (supportsResizing)
             {
                 //get the original image from the original stream
-                if (fileStream.CanSeek) fileStream.Seek(0, 0);
+                if (fileStream.CanSeek) fileStream.Seek(0, SeekOrigin.Begin);
                 using (var originalImage = Image.FromStream(fileStream))
                 {
                     var additionalSizes = new List<int>();

src/Umbraco.Core/Models/PropertyTypeCollection.cs

@@ -69,6 +69,7 @@ namespace Umbraco.Core.Models
             OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
         }
 
+        //TODO: Instead of 'new' this should explicitly implement one of the collection interfaces members
         internal new void Add(PropertyType item)
         {
             using (new WriteLock(_addLocker))

src/Umbraco.Core/Models/PublicAccessEntry.cs

@@ -110,6 +110,12 @@ namespace Umbraco.Core.Models
             _ruleCollection.Clear();
         }
 
+
+        internal void ClearRemovedRules()
+        {
+            _removedRules.Clear();
+        }
+
         [DataMember]
         public int LoginNodeId
         {

src/Umbraco.Core/Models/PublishedContent/PublishedContentType.cs

@@ -179,7 +179,7 @@ namespace Umbraco.Core.Models.PublishedContent
             }
 
             if (contentType == null)
-                throw new Exception(string.Format("ContentTypeService failed to find a {0} type with alias \"{1}\".",
+                throw new InvalidOperationException(string.Format("ContentTypeService failed to find a {0} type with alias \"{1}\".",
                     itemType.ToString().ToLower(), alias));
 
             return new PublishedContentType(contentType);

src/Umbraco.Core/Models/Rdbms/DocumentPublishedReadOnlyDto.cs

@@ -19,5 +19,8 @@ namespace Umbraco.Core.Models.Rdbms
 
         [Column("newest")]
         public bool Newest { get; set; }
+
+        [Column("updateDate")]
+        public DateTime VersionDate { get; set; }
     }
 }

src/Umbraco.Core/ObjectResolution/ManyObjectsResolverBase.cs

@@ -22,6 +22,7 @@ namespace Umbraco.Core.ObjectResolution
         private readonly string _httpContextKey;
         private readonly List<Type> _instanceTypes = new List<Type>();
         private IEnumerable<TResolved> _sortedValues;
+        private readonly Func<HttpContextBase> _httpContextGetter;
 
         private int _defaultPluginWeight = 100;
 
@@ -42,12 +43,7 @@ namespace Umbraco.Core.ObjectResolution
             if (logger == null) throw new ArgumentNullException("logger");
             CanResolveBeforeFrozen = false;
             if (scope == ObjectLifetimeScope.HttpRequest)
-            {
-                if (HttpContext.Current == null)
-                    throw new InvalidOperationException("Use alternative constructor accepting a HttpContextBase object in order to set the lifetime scope to HttpRequest when HttpContext.Current is null");
-
-                CurrentHttpContext = new HttpContextWrapper(HttpContext.Current);
-            }
+                _httpContextGetter = () => new HttpContextWrapper(HttpContext.Current);
 
             ServiceProvider = serviceProvider;
             Logger = logger;
@@ -84,7 +80,7 @@ namespace Umbraco.Core.ObjectResolution
             LifetimeScope = ObjectLifetimeScope.HttpRequest;
             _httpContextKey = GetType().FullName;
             ServiceProvider = serviceProvider;
-            CurrentHttpContext = httpContext;
+            _httpContextGetter = () => httpContext;
             _instanceTypes = new List<Type>();
 
             InitializeAppInstances();
@@ -160,7 +156,16 @@ namespace Umbraco.Core.ObjectResolution
         /// Gets or sets the <see cref="HttpContextBase"/> used to initialize this object, if any.
         /// </summary>
         /// <remarks>If not null, then <c>LifetimeScope</c> will be <c>ObjectLifetimeScope.HttpRequest</c>.</remarks>
-        protected HttpContextBase CurrentHttpContext { get; private set; }
+        protected HttpContextBase CurrentHttpContext
+        {
+            get
+            {
+                var context = _httpContextGetter == null ? null : _httpContextGetter();
+                if (context == null)
+                    throw new InvalidOperationException("Cannot use this resolver with lifetime 'HttpRequest' when there is no current HttpContext. Either use the ctor accepting an HttpContextBase, or use the resolver from within a request exclusively.");
+                return context;
+            }
+        }
 
         /// <summary>
         /// Returns the service provider used to instantiate objects
@@ -196,7 +201,7 @@ namespace Umbraco.Core.ObjectResolution
         /// <summary>
         /// Gets or sets the default type weight.
         /// </summary>
-        /// <remarks>Determines the weight of types that do not have a <c>WeightAttribute</c> set on 
+        /// <remarks>Determines the weight of types that do not have a <c>WeightAttribute</c> set on
         /// them, when calling <c>GetSortedValues</c>.</remarks>
         protected virtual int DefaultPluginWeight
         {
@@ -276,7 +281,7 @@ namespace Umbraco.Core.ObjectResolution
         /// Removes a type.
         /// </summary>
         /// <param name="value">The type to remove.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or
         /// the type is not a valid type for the resolver.</exception>
         public virtual void RemoveType(Type value)
         {
@@ -296,7 +301,7 @@ namespace Umbraco.Core.ObjectResolution
         /// Removes a type.
         /// </summary>
         /// <typeparam name="T">The type to remove.</typeparam>
-        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support removing types, or
         /// the type is not a valid type for the resolver.</exception>
         public void RemoveType<T>()
             where T : TResolved
@@ -309,7 +314,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <param name="types">The types to add.</param>
         /// <remarks>The types are appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
         /// a type is not a valid type for the resolver, or a type is already in the collection of types.</exception>
         protected void AddTypes(IEnumerable<Type> types)
         {
@@ -336,7 +341,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <param name="value">The type to add.</param>
         /// <remarks>The type is appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
         /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
         public virtual void AddType(Type value)
         {
@@ -362,7 +367,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <typeparam name="T">The type to add.</typeparam>
         /// <remarks>The type is appended at the end of the list.</remarks>
-        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support adding types, or
         /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
         public void AddType<T>()
             where T : TResolved
@@ -404,7 +409,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <param name="index">The zero-based index at which the type should be inserted.</param>
         /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
         /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
         /// <exception cref="ArgumentOutOfRangeException"><paramref name="index"/> is out of range.</exception>
         public virtual void InsertType(int index, Type value)
@@ -430,7 +435,7 @@ namespace Umbraco.Core.ObjectResolution
         /// Inserts a type at the beginning of the list.
         /// </summary>
         /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
         /// the type is not a valid type for the resolver, or the type is already in the collection of types.</exception>
         public virtual void InsertType(Type value)
         {
@@ -464,7 +469,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <param name="existingType">The existing type before which to insert.</param>
         /// <param name="value">The type to insert.</param>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
         /// one of the types is not a valid type for the resolver, or the existing type is not in the collection,
         /// or the new type is already in the collection of types.</exception>
         public virtual void InsertTypeBefore(Type existingType, Type value)
@@ -498,7 +503,7 @@ namespace Umbraco.Core.ObjectResolution
         /// </summary>
         /// <typeparam name="TExisting">The existing type before which to insert.</typeparam>
         /// <typeparam name="T">The type to insert.</typeparam>
-        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or 
+        /// <exception cref="InvalidOperationException">the resolver does not support inserting types, or
         /// one of the types is not a valid type for the resolver, or the existing type is not in the collection,
         /// or the new type is already in the collection of types.</exception>
         public void InsertTypeBefore<TExisting, T>()

src/Umbraco.Core/Persistence/Factories/ContentFactory.cs

@@ -72,6 +72,9 @@ namespace Umbraco.Core.Persistence.Factories
                 content.PublishedVersionGuid = publishedDto == null
                     ? (dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId)
                     : publishedDto.VersionId;
+                content.PublishedDate = publishedDto == null
+                    ? (dto.DocumentPublishedReadOnlyDto == null ? default(DateTime) : dto.DocumentPublishedReadOnlyDto.VersionDate)
+                    : publishedDto.VersionDate;
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Factories/MemberFactory.cs

@@ -28,17 +28,17 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IMedia,ContentVersionDto>
 
-        public IMember BuildEntity(MemberDto dto)
+        public static IMember BuildEntity(MemberDto dto, IMemberType contentType)
         {
             var member = new Member(
                 dto.ContentVersionDto.ContentDto.NodeDto.Text,
-                dto.Email, dto.LoginName, dto.Password, _contentType);
+                dto.Email, dto.LoginName, dto.Password, contentType);
 
             try
             {
                 member.DisableChangeTracking();
 
-                member.Id = _id;
+                member.Id = dto.NodeId;
                 member.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                 member.Path = dto.ContentVersionDto.ContentDto.NodeDto.Path;
                 member.CreatorId = dto.ContentVersionDto.ContentDto.NodeDto.UserId.Value;
@@ -62,6 +62,12 @@ namespace Umbraco.Core.Persistence.Factories
             }
         }
 
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMember BuildEntity(MemberDto dto)
+        {
+            return BuildEntity(dto, _contentType);
+        }
+
         public MemberDto BuildDto(IMember entity)
         {
             var dto = new MemberDto

src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Factories
 {
@@ -17,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories
             
             //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data
             foreach (var k in originalEntityProperties.Keys
-                .Select(x => new { orig = x, title = x.ConvertCase(StringAliasCaseType.PascalCase) })
+                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase | CleanStringType.Ascii | CleanStringType.ConvertCase) })
                 .Where(x => entityProps.InvariantContains(x.title) == false))
             {
                 entity.AdditionalData[k.title] = originalEntityProperties[k.orig];
@@ -75,65 +76,6 @@ namespace Umbraco.Core.Persistence.Factories
                 entity.EnableChangeTracking();
             }
         }
-
-        public UmbracoEntity BuildEntity(EntityRepository.UmbracoEntityDto dto)
-        {
-            var entity = new UmbracoEntity(dto.Trashed)
-                             {
-                                 CreateDate = dto.CreateDate,
-                                 CreatorId = dto.UserId.Value,
-                                 Id = dto.NodeId,
-                                 Key = dto.UniqueId,
-                                 Level = dto.Level,
-                                 Name = dto.Text,
-                                 NodeObjectTypeId = dto.NodeObjectType.Value,
-                                 ParentId = dto.ParentId,
-                                 Path = dto.Path,
-                                 SortOrder = dto.SortOrder,
-                                 HasChildren = dto.Children > 0,
-                                 ContentTypeAlias = dto.Alias ?? string.Empty,
-                                 ContentTypeIcon = dto.Icon ?? string.Empty,
-                                 ContentTypeThumbnail = dto.Thumbnail ?? string.Empty,
-                             };
-
-            entity.IsPublished = dto.PublishedVersion != default(Guid) || (dto.NewestVersion != default(Guid) && dto.PublishedVersion == dto.NewestVersion);
-            entity.IsDraft = dto.NewestVersion != default(Guid) && (dto.PublishedVersion == default(Guid) || dto.PublishedVersion != dto.NewestVersion);
-            entity.HasPendingChanges = (dto.PublishedVersion != default(Guid) && dto.NewestVersion != default(Guid)) && dto.PublishedVersion != dto.NewestVersion;
-
-            if (dto.UmbracoPropertyDtos != null)
-            {
-                foreach (var propertyDto in dto.UmbracoPropertyDtos)
-                {
-                    entity.AdditionalData[propertyDto.PropertyAlias] = new UmbracoEntity.EntityProperty
-                    {
-                        PropertyEditorAlias = propertyDto.PropertyEditorAlias,
-                        Value = propertyDto.NTextValue.IsNullOrWhiteSpace()
-                            ? propertyDto.NVarcharValue
-                            : propertyDto.NTextValue.ConvertToJsonIfPossible()
-                    };
-                }
-            }
-
-            return entity;
-        }
-
-        public EntityRepository.UmbracoEntityDto BuildDto(UmbracoEntity entity)
-        {
-            var node = new EntityRepository.UmbracoEntityDto
-                           {
-                               CreateDate = entity.CreateDate,
-                               Level = short.Parse(entity.Level.ToString(CultureInfo.InvariantCulture)),
-                               NodeId = entity.Id,
-                               NodeObjectType = entity.NodeObjectTypeId,
-                               ParentId = entity.ParentId,
-                               Path = entity.Path,
-                               SortOrder = entity.SortOrder,
-                               Text = entity.Name,
-                               Trashed = entity.Trashed,
-                               UniqueId = entity.Key,
-                               UserId = entity.CreatorId
-                           };
-            return node;
-        }
+        
     }
 }

src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs

@@ -31,7 +31,7 @@ namespace Umbraco.Core.Persistence.Mappers
         /// </summary>
         /// <param name="type"></param>
         /// <returns></returns>
-        internal BaseMapper ResolveMapperByType(Type type)
+        public virtual BaseMapper ResolveMapperByType(Type type)
         {
             return _mapperCache.GetOrAdd(type, type1 =>
                 {
@@ -67,7 +67,7 @@ namespace Umbraco.Core.Persistence.Mappers
             return Attempt<BaseMapper>.Succeed(mapper);
         }  
 
-        internal string GetMapping(Type type, string propertyName)
+        public virtual string GetMapping(Type type, string propertyName)
         {
             var mapper = ResolveMapperByType(type);
             var result = mapper.Map(propertyName);

src/Umbraco.Core/Persistence/Migrations/MigrationBase.cs

@@ -23,7 +23,7 @@ namespace Umbraco.Core.Persistence.Migrations
             Logger = logger;
         }
 
-        internal IMigrationContext Context;
+        public IMigrationContext Context;
 
         public abstract void Up();
         public abstract void Down();

src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs

@@ -581,6 +581,18 @@ namespace Umbraco.Core.Persistence.Querying
                 case "InvariantContains":
                 case "InvariantEquals":
 
+                    //special case, if it is 'Contains' and the argument that Contains is being called on is 
+                    //Enumerable and the methodArgs is the actual member access, then it's an SQL IN clause
+                    if (m.Object == null 
+                        && m.Arguments[0].Type != typeof(string)
+                        && m.Arguments.Count == 2
+                        && methodArgs.Length == 1 
+                        && methodArgs[0].NodeType == ExpressionType.MemberAccess
+                        && TypeHelper.IsTypeAssignableFrom<IEnumerable>(m.Arguments[0].Type))
+                    {
+                        goto case "SqlIn";
+                    }
+
                     string compareValue;
 
                     if (methodArgs[0].NodeType != ExpressionType.Constant)
@@ -597,13 +609,6 @@ namespace Umbraco.Core.Persistence.Querying
                         compareValue = methodArgs[0].ToString();
                     }
 
-                    //special case, if it is 'Contains' and the member that Contains is being called on is not a string, then
-                    // we should be doing an 'In' clause - but we currently do not support this
-                    if (methodArgs[0].Type != typeof(string) && TypeHelper.IsTypeAssignableFrom<IEnumerable>(methodArgs[0].Type))
-                    {
-                        throw new NotSupportedException("An array Contains method is not supported");
-                    }
-
                     //default column type
                     var colType = TextColumnType.NVarchar;
 
@@ -705,29 +710,33 @@ namespace Umbraco.Core.Persistence.Querying
                 //    }
                 //    return string.Format("{0}{1}", r, s);
 
-                //case "In":
+                case "SqlIn":
 
-                //    var member = Expression.Convert(m.Arguments[0], typeof(object));
-                //    var lambda = Expression.Lambda<Func<object>>(member);
-                //    var getter = lambda.Compile();
+                    if (m.Object == null && methodArgs.Length == 1 && methodArgs[0].NodeType == ExpressionType.MemberAccess)
+                    {
+                        var memberAccess = VisitMemberAccess((MemberExpression) methodArgs[0]);
+                        
+                        var member = Expression.Convert(m.Arguments[0], typeof(object));
+                        var lambda = Expression.Lambda<Func<object>>(member);
+                        var getter = lambda.Compile();
 
-                //    var inArgs = (object[])getter();
+                        var inArgs = (IEnumerable)getter();
 
-                //    var sIn = new StringBuilder();
-                //    foreach (var e in inArgs)
-                //    {
-                //        SqlParameters.Add(e);
+                        var sIn = new StringBuilder();
+                        foreach (var e in inArgs)
+                        {
+                            SqlParameters.Add(e);
 
-                //        sIn.AppendFormat("{0}{1}",
-                //                     sIn.Length > 0 ? "," : "",
-                //                                    string.Format("@{0}", SqlParameters.Count - 1));
+                            sIn.AppendFormat("{0}{1}",
+                                sIn.Length > 0 ? "," : "",
+                                string.Format("@{0}", SqlParameters.Count - 1));
+                        }
 
-                //        //sIn.AppendFormat("{0}{1}",
-                //        //             sIn.Length > 0 ? "," : "",
-                //        //                            GetQuotedValue(e, e.GetType()));
-                //    }
+                        return string.Format("{0} IN ({1})", memberAccess, sIn);
+                    }
+
+                    throw new NotSupportedException("SqlIn must contain the member being accessed");
 
-                //    return string.Format("{0} {1} ({2})", r, m.Method.Name, sIn.ToString());
                 //case "Desc":
                 //    return string.Format("{0} DESC", r);
                 //case "Alias":

src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq.Expressions;
+using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Mappers;
 using Umbraco.Core.Persistence.SqlSyntax;
 
@@ -12,17 +13,19 @@ namespace Umbraco.Core.Persistence.Querying
     /// <remarks>This object is stateful and cannot be re-used to parse an expression.</remarks>
     internal class ModelToSqlExpressionVisitor<T> : ExpressionVisitorBase
     {
+        private readonly MappingResolver _mappingResolver;
         private readonly BaseMapper _mapper;
 
-        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, BaseMapper mapper)
+        public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, MappingResolver mappingResolver)
             : base(sqlSyntax)
         {
-            _mapper = mapper;
+            _mapper = mappingResolver.ResolveMapperByType(typeof(T));
+            _mappingResolver = mappingResolver;
         }
 
         [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public ModelToSqlExpressionVisitor()
-            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T)))
+            : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current)
         { }
 
         protected override string VisitMemberAccess(MemberExpression m)
@@ -36,7 +39,7 @@ namespace Umbraco.Core.Persistence.Querying
                 {
                     var field = _mapper.Map(m.Member.Name, true);
                     if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                     return field;
                 }
                 //already compiled, return
@@ -50,13 +53,42 @@ namespace Umbraco.Core.Persistence.Querying
                 {
                     var field = _mapper.Map(m.Member.Name, true);
                     if (field.IsNullOrWhiteSpace())
-                        throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name);
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
                     return field;
                 }
                 //already compiled, return
                 return string.Empty;
             }
 
+            if (m.Expression != null 
+                && m.Expression.Type != typeof(T) 
+                && TypeHelper.IsTypeAssignableFrom<IUmbracoEntity>(m.Expression.Type)
+                && EndsWithConstant(m) == false)
+            {
+                //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test";
+                //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column
+
+                //don't execute if compiled
+                if (Visited == false)
+                {
+                    var subMapper = _mappingResolver.ResolveMapperByType(m.Expression.Type);
+                    if (subMapper == null)
+                        throw new NullReferenceException("No mapper found for type " + m.Expression.Type);
+                    var field = subMapper.Map(m.Member.Name, true);
+                    if (field.IsNullOrWhiteSpace())
+                        throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type));
+                    return field;
+                }
+                //already compiled, return
+                return string.Empty;
+            }
+
+            //TODO: When m.Expression.NodeType == ExpressionType.Constant and it's an expression like: content => aliases.Contains(content.ContentType.Alias);
+            // then an SQL parameter will be added for aliases as an array, however in SqlIn on the subclass it will manually add these SqlParameters anyways,
+            // however the query will still execute because the SQL that is written will only contain the correct indexes of SQL parameters, this would be ignored,
+            // I'm just unsure right now due to time constraints how to make it correct. It won't matter right now and has been working already with this bug but I've
+            // only just discovered what it is actually doing.
+
             var member = Expression.Convert(m, typeof(object));
             var lambda = Expression.Lambda<Func<object>>(member);
             var getter = lambda.Compile();
@@ -71,5 +103,24 @@ namespace Umbraco.Core.Persistence.Querying
             return string.Empty;
 
         }
+
+        /// <summary>
+        /// Determines if the MemberExpression ends in a Constant value
+        /// </summary>
+        /// <param name="m"></param>
+        /// <returns></returns>
+        private bool EndsWithConstant(MemberExpression m)
+        {
+            Expression expr = m;
+            
+            while (expr is MemberExpression)
+            {
+                var memberExpr = expr as MemberExpression;
+                expr = memberExpr.Expression;
+            }
+            
+            var constExpr = expr as ConstantExpression;
+            return constExpr != null;
+        }
     }
 }

src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs

@@ -1,4 +1,5 @@
-using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace Umbraco.Core.Persistence.Querying
@@ -6,8 +7,13 @@ namespace Umbraco.Core.Persistence.Querying
     /// <summary>
     /// String extension methods used specifically to translate into SQL
     /// </summary>
-    internal static class SqlStringExtensions
+    internal static class SqlExpressionExtensions
     {
+        public static bool SqlIn<T>(this IEnumerable<T> collection, T item)
+        {
+            return collection.Contains(item);
+        }
+
         public static bool SqlWildcard(this string str, string txt, TextColumnType columnType)
         {
             var wildcardmatch = new Regex("^" + Regex.Escape(txt).

src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs

@@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateContentFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateContentFromDto(dto, sql);
 
             return content;
         }
@@ -134,6 +134,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 .InnerJoin<NodeDto>(SqlSyntax)
                 .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+            //TODO: IF we want to enable querying on content type information this will need to be joined
+            //.InnerJoin<ContentTypeDto>(SqlSyntax)
+            //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
 
             if (queryType == BaseQueryType.FullSingle)
             {
@@ -260,6 +263,39 @@ namespace Umbraco.Core.Persistence.Repositories
                 }
                 baseId = xmlItems[xmlItems.Count - 1].NodeId;
             }
+
+            //now delete the items that shouldn't be there
+            var sqlAllIds = translate(0, GetBaseQuery(BaseQueryType.Ids));
+            var allContentIds = Database.Fetch<int>(sqlAllIds);
+            var docObjectType = Guid.Parse(Constants.ObjectTypes.Document);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allContentIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }                
+            }
+                
         }
 
         public override IEnumerable<IContent> GetAllVersions(int id)
@@ -273,12 +309,13 @@ namespace Umbraco.Core.Persistence.Repositories
             var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple));
             var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
             
-            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true);
+            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true,  includeAllVersions:true);
         }
 
         public override IContent GetByVersion(Guid versionId)
         {
             var sql = GetBaseQuery(BaseQueryType.FullSingle);
+            //TODO: cmsContentVersion.VersionId has a Unique Index constraint applied, seems silly then to also add OrderByDescending since it would be impossible to return more than one.
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
             sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
@@ -287,7 +324,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateContentFromDto(dto, versionId, sql);
+            var content = CreateContentFromDto(dto, sql);
 
             return content;
         }
@@ -297,7 +334,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From<DocumentDto>(SqlSyntax)
-                .InnerJoin<ContentVersionDto>(SqlSyntax).On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
                 .Where<ContentVersionDto>(x => x.VersionId == versionId, SqlSyntax)
                 .Where<DocumentDto>(x => x.Newest != true, SqlSyntax);
             var dto = Database.Fetch<DocumentDto, ContentVersionDto>(sql).FirstOrDefault();
@@ -317,7 +355,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentVersionDto>()
+                .On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
                 .Where<ContentVersionDto>(x => x.NodeId == id)
                 .Where<ContentVersionDto>(x => x.VersionDate < versionDate)
                 .Where<DocumentDto>(x => x.Newest != true);
@@ -474,11 +513,13 @@ namespace Umbraco.Core.Persistence.Repositories
                 dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                 {
                     VersionId = dto.VersionId,
+                    VersionDate = dto.UpdateDate,
                     Newest = true,
                     NodeId = dto.NodeId,
-                    Published = true
+                    Published = true               
                 };
-                ((Content)entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedDate = dto.UpdateDate;
             }
 
             entity.ResetDirtyProperties();
@@ -557,6 +598,7 @@ namespace Umbraco.Core.Persistence.Repositories
             //if (((ICanBeDirty)entity).IsPropertyDirty("Published") && (entity.Published || publishedState == PublishedState.Unpublished))
             if (entity.ShouldClearPublishedFlagForPreviousVersions(publishedState, shouldCreateNewVersion))
             {
+                //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
                 var publishedDocs = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND published = @IsPublished", new { Id = entity.Id, IsPublished = true });
                 foreach (var doc in publishedDocs)
                 {
@@ -570,6 +612,7 @@ namespace Umbraco.Core.Persistence.Repositories
             }
 
             //Look up (newest) entries by id in cmsDocument table to set newest = false
+            //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
             var documentDtos = Database.Fetch<DocumentDto>("WHERE nodeId = @Id AND newest = @IsNewest", new { Id = entity.Id, IsNewest = true });
             foreach (var documentDto in documentDtos)
             {
@@ -646,22 +689,26 @@ namespace Umbraco.Core.Persistence.Repositories
                 dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                 {
                     VersionId = dto.VersionId,
+                    VersionDate = dto.UpdateDate,
                     Newest = true,
                     NodeId = dto.NodeId,
                     Published = true
                 };
-                ((Content)entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedVersionGuid = dto.VersionId;
+                ((Content) entity).PublishedDate = dto.UpdateDate;
             }
             else if (publishedStateChanged)
             {
                 dto.DocumentPublishedReadOnlyDto = new DocumentPublishedReadOnlyDto
                 {
-                    VersionId = default(Guid),
+                    VersionId = default (Guid),
+                    VersionDate = default (DateTime),
                     Newest = false,
                     NodeId = dto.NodeId,
                     Published = false
                 };
-                ((Content)entity).PublishedVersionGuid = default(Guid);
+                ((Content) entity).PublishedVersionGuid = default(Guid);
+                ((Content) entity).PublishedDate = default (DateTime);
             }
 
             entity.ResetDirtyProperties();
@@ -784,13 +831,8 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
         public void ClearPublished(IContent content)
         {
-            // race cond!
-            var documentDtos = Database.Fetch<DocumentDto>("WHERE nodeId=@id AND published=@published", new { id = content.Id, published = true });
-            foreach (var documentDto in documentDtos)
-            {
-                documentDto.Published = false;
-                Database.Update(documentDto);
-            }
+            var sql = "UPDATE cmsDocument SET published=0 WHERE nodeId=@id AND published=1";
+            Database.Execute(sql, new {id = content.Id});
         }
 
         /// <summary>
@@ -902,7 +944,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
             return base.GetDatabaseFieldNameForOrderBy(orderBy);
         }
-        
+
         /// <summary>
         /// This is the underlying method that processes most queries for this repository
         /// </summary>
@@ -913,8 +955,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
         /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
         /// </param>
         /// <param name="withCache"></param>
+        /// <param name="includeAllVersions">
+        /// Generally when querying for content we only want to return the most recent version of the content item, however in some cases like when 
+        /// we want to return all versions of a content item, we can't simply return the latest
+        /// </param>
         /// <returns></returns>
-        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
+        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false, bool includeAllVersions = false)
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
@@ -933,13 +979,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
                 parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
             }            
 
-            var publishedSql = new Sql(@"SELECT *
-FROM cmsDocument AS doc2
-INNER JOIN 
-	(" + parsedOriginalSql + @") as docData
-ON doc2.nodeId = docData.nodeId
-WHERE doc2.published = 1
-ORDER BY doc2.nodeId
+            //order by update date DESC, if there is corrupted published flags we only want the latest!
+            var publishedSql = new Sql(@"SELECT cmsDocument.nodeId, cmsDocument.published, cmsDocument.versionId, cmsDocument.updateDate, cmsDocument.newest
+FROM cmsDocument INNER JOIN cmsContentVersion ON cmsContentVersion.VersionId = cmsDocument.versionId
+WHERE cmsDocument.published = 1 AND cmsDocument.nodeId IN 
+(" + parsedOriginalSql + @")
+ORDER BY cmsContentVersion.id DESC
 ", sqlFull.Arguments);
 
             //go and get the published version data, we do a Query here and not a Fetch so we are
@@ -954,9 +999,9 @@ ORDER BY doc2.nodeId
                     publishedDataCollection.Add(publishedDto);
             }
 
-
-            var content = new IContent[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IContent, bool>>();
+            var defs = new DocumentDefinitionCollection(includeAllVersions);
             var templateIds = new List<int>();
             
             //track the looked up content types, even though the content types are cached
@@ -964,9 +1009,8 @@ ORDER BY doc2.nodeId
             // the overhead of deep cloning them on every item in this loop
             var contentTypes = new Dictionary<int, IContentType>();
             
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
                 DocumentPublishedReadOnlyDto publishedDto;
                 publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto);
 
@@ -974,10 +1018,10 @@ ORDER BY doc2.nodeId
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem<IContent>(GetCacheIdKey<IContent>(dto.NodeId));
-                    //only use this cached version if the dto returned is also the publish version, they must match
-                    if (cached != null && cached.Published && dto.Published)
+                    //only use this cached version if the dto returned is also the publish version, they must match and be teh same version
+                    if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IContent, bool>(cached, true));
                         continue;
                     }
                 }
@@ -995,21 +1039,16 @@ ORDER BY doc2.nodeId
                     contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
                     contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
                 }
-                
-                content[i] = ContentFactory.BuildEntity(dto, contentType, publishedDto);
 
-                // need template
-                if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
-                    templateIds.Add(dto.TemplateId.Value);
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    // assign template
+                    if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
+                        templateIds.Add(dto.TemplateId.Value);
 
-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                    content.Add(new Tuple<IContent, bool>(ContentFactory.BuildEntity(dto, contentType, publishedDto), false));
+                }
             }
 
             // load all required templates in 1 query
@@ -1019,38 +1058,38 @@ ORDER BY doc2.nodeId
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign template and property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id];
 
-                // complete the item
-                var cc = content[dtoIndex];
-                var dto = dtos[dtoIndex];
                 ITemplate template = null;
-                if (dto.TemplateId.HasValue)
-                    templates.TryGetValue(dto.TemplateId.Value, out template); // else null
+                if (def.DocumentDto.TemplateId.HasValue)
+                    templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null
                 cc.Template = template;
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// <summary>
         /// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion.
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql)
+        private IContent CreateContentFromDto(DocumentDto dto, Sql docSql)
         {
             var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
@@ -1062,11 +1101,11 @@ ORDER BY doc2.nodeId
                 content.Template = _templateRepository.Get(dto.TemplateId.Value);
             }
 
-            var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType);
+            var docDef = new DocumentDefinition(dto, contentType);
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            content.Properties = properties[dto.NodeId];
+            content.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/ContentTypeBaseRepository.cs

@@ -1250,5 +1250,19 @@ WHERE cmsContentType." + aliasColumn + @" LIKE @pattern",
             while (aliases.Contains(test = alias + i)) i++;
             return test;
         }
+
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        public bool HasContainerInPath(string contentPath)
+        {
+            var ids = contentPath.Split(',').Select(int.Parse);
+            var sql = new Sql(@"SELECT COUNT(*) FROM cmsContentType
+INNER JOIN cmsContent ON cmsContentType.nodeId=cmsContent.contentType 
+WHERE cmsContent.nodeId IN (@ids) AND cmsContentType.isContainer=@isContainer", new { ids, isContainer = true });
+            return Database.ExecuteScalar<int>(sql) > 0;
+        }
     }
 }

src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs

@@ -2,19 +2,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using Umbraco.Core.Cache;
-using Umbraco.Core.Events;
-using Umbraco.Core.Exceptions;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
-
-using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Persistence.Relators;
 using Umbraco.Core.Persistence.SqlSyntax;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Services;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
@@ -118,7 +112,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             if (objectTypes.Any())
             {
-                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", objectTypes);
+                sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", new {objectTypes = objectTypes});
             }
 
             return Database.Fetch<string>(sql);

src/Umbraco.Core/Persistence/Repositories/DataTypeDefinitionRepository.cs

@@ -34,7 +34,7 @@ namespace Umbraco.Core.Persistence.Repositories
             : base(work, cache, logger, sqlSyntax)
         {
             _contentTypeRepository = contentTypeRepository;
-            _preValRepository = new DataTypePreValueRepository(work, CacheHelper.CreateDisabledCacheHelper(), logger, sqlSyntax);            
+            _preValRepository = new DataTypePreValueRepository(work, CacheHelper.CreateDisabledCacheHelper(), logger, sqlSyntax);
         }
 
         #region Overrides of RepositoryBase<int,DataTypeDefinition>
@@ -276,47 +276,33 @@ AND umbracoNode.id <> @id",
 
         public PreValueCollection GetPreValuesCollectionByDataTypeId(int dataTypeId)
         {
-            var cached = RuntimeCache.GetCacheItemsByKeySearch<PreValueCollection>(GetPrefixedCacheKey(dataTypeId));
-            if (cached != null && cached.Any())
-            {
-                //return from the cache, ensure it's a cloned result
-                return (PreValueCollection)cached.First().DeepClone();
-            }
-
-            return GetAndCachePreValueCollection(dataTypeId);
-        }
-
-        internal static string GetCacheKeyRegex(int preValueId)
-        {
-            return CacheKeys.DataTypePreValuesCacheKey + @"[-\d]+-([\d]*,)*" + preValueId + @"(?!\d)[,\d$]*";
+            var collection = GetCachedPreValueCollection(dataTypeId);
+            return collection;
         }
 
+        /// <summary>
+        /// Gets a specific PreValue by its Id
+        /// </summary>
+        /// <param name="preValueId">Id of the PreValue to retrieve the value from</param>
+        /// <returns>PreValue as a string</returns>
         public string GetPreValueAsString(int preValueId)
         {
-            //We need to see if we can find the cached PreValueCollection based on the cache key above
+            var collections = RuntimeCache.GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + "_");
 
-            var cached = RuntimeCache.GetCacheItemsByKeyExpression<PreValueCollection>(GetCacheKeyRegex(preValueId));
-            if (cached != null && cached.Any())
-            {
-                //return from the cache
-                var collection = cached.First();
-				var preVal = collection.FormatAsDictionary().Single(x => x.Value.Id == preValueId);
-				return preVal.Value.Value;
-            }
+            var preValue = collections.SelectMany(x => x.FormatAsDictionary().Values).FirstOrDefault(x => x.Id == preValueId);
+            if (preValue != null)
+                return preValue.Value;
 
-            //go and find the data type id for the pre val id passed in
-
-            var dto = Database.FirstOrDefault<DataTypePreValueDto>("WHERE id = @preValueId", new { preValueId = preValueId });
+            var dto = Database.FirstOrDefault<DataTypePreValueDto>("WHERE id = @preValueId", new { preValueId });
             if (dto == null)
-            {
                 return string.Empty;
-            }
-            // go cache the collection
-            var preVals = GetAndCachePreValueCollection(dto.DataTypeNodeId);
 
-            //return the single value for this id
-            var pv = preVals.FormatAsDictionary().Single(x => x.Value.Id == preValueId);
-            return pv.Value.Value;
+            var collection = GetCachedPreValueCollection(dto.DataTypeNodeId);
+            if (collection == null)
+                return string.Empty;
+
+            preValue = collection.FormatAsDictionary().Values.FirstOrDefault(x => x.Id == preValueId);
+            return preValue == null ? string.Empty : preValue.Value;
         }
 
         public void AddOrUpdatePreValues(int dataTypeId, IDictionary<string, PreValue> values)
@@ -441,40 +427,28 @@ AND umbracoNode.id <> @id",
 
                 sortOrder++;
             }
-
         }
 
-        private string GetPrefixedCacheKey(int dataTypeId)
+        private static string GetPrefixedCacheKey(int dataTypeId)
         {
-            return CacheKeys.DataTypePreValuesCacheKey + dataTypeId + "-";
+            return CacheKeys.DataTypePreValuesCacheKey + "_" + dataTypeId;
         }
 
-        private PreValueCollection GetAndCachePreValueCollection(int dataTypeId)
+        private PreValueCollection GetCachedPreValueCollection(int datetypeId)
         {
-            //go get the data
-            var dtos = Database.Fetch<DataTypePreValueDto>("WHERE datatypeNodeId = @Id", new { Id = dataTypeId });
-            var list = dtos.Select(x => new Tuple<PreValue, string, int>(new PreValue(x.Id, x.Value, x.SortOrder), x.Alias, x.SortOrder)).ToList();
-            var collection = PreValueConverter.ConvertToPreValuesCollection(list);
-
-            //now create the cache key, this needs to include all pre-value ids so that we can use this cached item in the GetPreValuesAsString method
-            //the key will be: "UmbracoPreValDATATYPEID-CSVOFPREVALIDS
-
-            var key = GetPrefixedCacheKey(dataTypeId)
-                      + string.Join(",", collection.FormatAsDictionary().Select(x => x.Value.Id).ToArray());
-
-            //store into cache
-            RuntimeCache.InsertCacheItem(key, () => collection,
-                //30 mins
-                new TimeSpan(0, 0, 30),
-                //sliding is true
-                true);
-
-            return collection;
+            var key = GetPrefixedCacheKey(datetypeId);
+            return RuntimeCache.GetCacheItem<PreValueCollection>(key, () =>
+            {
+                var dtos = Database.Fetch<DataTypePreValueDto>("WHERE datatypeNodeId = @Id", new { Id = datetypeId });
+                var list = dtos.Select(x => new Tuple<PreValue, string, int>(new PreValue(x.Id, x.Value, x.SortOrder), x.Alias, x.SortOrder)).ToList();
+                var collection = PreValueConverter.ConvertToPreValuesCollection(list);
+                return collection;
+            }, TimeSpan.FromMinutes(20), isSliding: true);
         }
 
         private string EnsureUniqueNodeName(string nodeName, int id = 0)
         {
-         
+
 
             var sql = new Sql();
             sql.Select("*")
@@ -576,7 +550,7 @@ AND umbracoNode.id <> @id",
                 }
 
                 //NOTE: We used to check that the Alias was unique for the given DataTypeNodeId prevalues list, BUT
-                // in reality there is no need to check the uniqueness of this alias because the only way that this code executes is 
+                // in reality there is no need to check the uniqueness of this alias because the only way that this code executes is
                 // based on an IDictionary<string, PreValue> dictionary being passed to this repository and a dictionary
                 // must have unique aliases by definition, so there is no need for this additional check
 
@@ -596,10 +570,10 @@ AND umbracoNode.id <> @id",
                 {
                     throw new InvalidOperationException("Cannot update a pre value for a data type that has no identity");
                 }
-                
+
                 //NOTE: We used to check that the Alias was unique for the given DataTypeNodeId prevalues list, BUT
                 // this causes issues when sorting the pre-values (http://issues.umbraco.org/issue/U4-5670) but in reality
-                // there is no need to check the uniqueness of this alias because the only way that this code executes is 
+                // there is no need to check the uniqueness of this alias because the only way that this code executes is
                 // based on an IDictionary<string, PreValue> dictionary being passed to this repository and a dictionary
                 // must have unique aliases by definition, so there is no need for this additional check
 
@@ -614,7 +588,7 @@ AND umbracoNode.id <> @id",
                 Database.Update(dto);
             }
 
-            
+
         }
 
         internal static class PreValueConverter

src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs

@@ -1,18 +1,13 @@
 using System;
 using System.Collections.Generic;
-using System.Dynamic;
-using System.Globalization;
+using System.Collections.ObjectModel;
 using System.Linq;
-using System.Reflection;
-using System.Text;
 using Umbraco.Core.Models;
-using Umbraco.Core;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
@@ -68,11 +63,12 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(key, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
 
@@ -80,14 +76,16 @@ namespace Umbraco.Core.Persistence.Repositories
             }
             else
             {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;
 
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;                
             }
             
             
@@ -112,29 +110,29 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(id, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
 
                 return entities.FirstOrDefault();
             }
             else
-            {
-                var nodeDto = _work.Database.FirstOrDefault<dynamic>(sql);
-                if (nodeDto == null)
-                    return null;
-
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+            {                
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;
             }
-
-            
         }
 
         public virtual IEnumerable<IUmbracoEntity> GetAll(Guid objectTypeId, params int[] ids)
@@ -171,22 +169,21 @@ namespace Umbraco.Core.Persistence.Repositories
 
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too                
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, sql);
-                foreach (var entity in entities)
-                {
-                    yield return entity;
-                }
+                return entities;
             }
             else
             {
-                var dtos = _work.Database.Fetch<dynamic>(sql);
-                foreach (var entity in dtos.Select(dto => factory.BuildEntityFromDynamic(dto)))
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
                 {
-                    yield return entity;
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
                 }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();                
             }
         }
 
@@ -231,8 +228,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     }
                 });
 
-                //treat media differently for now 
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too    
                 var entities = _work.Database.Fetch<dynamic, UmbracoPropertyDto, UmbracoEntity>(
                     new UmbracoEntityRelator().Map, mediaSql);
                 return entities;
@@ -241,8 +237,15 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData
                 var finalSql = entitySql.Append(GetGroupBy(isContent, false));
-                var dtos = _work.Database.Fetch<dynamic>(finalSql);
-                return dtos.Select(factory.BuildEntityFromDynamic).Cast<IUmbracoEntity>().ToList();
+
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query<dynamic>(finalSql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();
             }
         }
 
@@ -315,25 +318,31 @@ namespace Umbraco.Core.Persistence.Repositories
         protected virtual Sql GetBase(bool isContent, bool isMedia, Action<Sql> customFilter)
         {
             var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate",
-                                  "COUNT(parent.parentID) as children"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate",
+                "COUNT(parent.parentID) as children"
+            };
 
             if (isContent || isMedia)
             {
-                columns.Add("published.versionId as publishedVersion");
-                columns.Add("latest.versionId as newestVersion");
+                if (isContent)
+                {
+                    //only content has/needs this info
+                    columns.Add("published.versionId as publishedVersion");
+                    columns.Add("document.versionId as newestVersion");
+                    columns.Add("contentversion.id as versionId");
+                }
+                
                 columns.Add("contenttype.alias");
                 columns.Add("contenttype.icon");
                 columns.Add("contenttype.thumbnail");
@@ -345,17 +354,22 @@ namespace Umbraco.Core.Persistence.Repositories
             var entitySql = new Sql()
                 .Select(columns.ToArray())
                 .From("umbracoNode umbracoNode");
-                
+
             if (isContent || isMedia)
             {
-                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id")
-                   .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published")
-                   .On("umbracoNode.id = published.nodeId")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest")
-                   .On("umbracoNode.id = latest.nodeId");
+                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id");
+
+                if (isContent)
+                {
+                    //only content has/needs this info
+                    entitySql                        
+                        .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id")
+                        .InnerJoin("cmsContentVersion contentversion").On("contentversion.VersionId = document.versionId")
+                        .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published")
+                        .On("umbracoNode.id = published.nodeId");
+                }
+
+                entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType");
             }
 
             entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id");
@@ -372,22 +386,42 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = baseQuery(isContent, isMedia, filter)
                 .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, int id)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.id = @Id", new { Id = id })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.id = @Id", new { Id = id });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func<bool, bool, Action<Sql>, Sql> baseQuery, bool isContent, bool isMedia, Guid key)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
@@ -396,6 +430,12 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new {Id = id, NodeObjectType = nodeObjectType});
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
@@ -404,36 +444,46 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new { UniqueID = key, NodeObjectType = nodeObjectType });
+
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
+
             return sql;
         }
 
         protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
         {
             var columns = new List<object>
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate"
+            };
 
             if (isContent || isMedia)
             {
-                columns.Add("published.versionId");
-                columns.Add("latest.versionId");
+                if (isContent)
+                {
+                    columns.Add("published.versionId");
+                    columns.Add("document.versionId");
+                    columns.Add("contentversion.id");
+                }
                 columns.Add("contenttype.alias");
                 columns.Add("contenttype.icon");
                 columns.Add("contenttype.thumbnail");
-                columns.Add("contenttype.isContainer");
+                columns.Add("contenttype.isContainer");                
             }
-            
+
             var sql = new Sql()
                 .GroupBy(columns.ToArray());
 
@@ -444,7 +494,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             return sql;
         }
-        
+
         #endregion
 
         /// <summary>
@@ -458,33 +508,7 @@ namespace Umbraco.Core.Persistence.Repositories
             UnitOfWork.DisposeIfDisposable();
         }
 
-        #region umbracoNode POCO - Extends NodeDto
-        [TableName("umbracoNode")]
-        [PrimaryKey("id")]
-        [ExplicitColumns]
-        internal class UmbracoEntityDto : NodeDto
-        {
-            [Column("children")]
-            public int Children { get; set; }
-
-            [Column("publishedVersion")]
-            public Guid PublishedVersion { get; set; }
-
-            [Column("newestVersion")]
-            public Guid NewestVersion { get; set; }
-
-            [Column("alias")]
-            public string Alias { get; set; }
-
-            [Column("icon")]
-            public string Icon { get; set; }
-
-            [Column("thumbnail")]
-            public string Thumbnail { get; set; }
-
-            [ResultColumn]
-            public List<UmbracoPropertyDto> UmbracoPropertyDtos { get; set; }
-        }
+        #region private classes
         
         [ExplicitColumns]
         internal class UmbracoPropertyDto
@@ -568,6 +592,99 @@ namespace Umbraco.Core.Persistence.Repositories
                 return prev;
             }
         }
+
+        private class EntityDefinitionCollection : KeyedCollection<int, EntityDefinition>
+        {
+            protected override int GetKeyForItem(EntityDefinition item)
+            {
+                return item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(EntityDefinition item)
+            {
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                EntityDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Id);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+
+                base.Add(item);
+                return true;
+            }
+
+            private bool TryGetValue(int key, out EntityDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        private class EntityDefinition
+        {
+            private readonly UmbracoEntityFactory _factory;
+            private readonly dynamic _entity;
+            private readonly bool _isContent;
+            private readonly bool _isMedia;
+
+            public EntityDefinition(UmbracoEntityFactory factory, dynamic entity, bool isContent, bool isMedia)
+            {
+                _factory = factory;
+                _entity = entity;
+                _isContent = isContent;
+                _isMedia = isMedia;
+            }
+
+            public IUmbracoEntity BuildFromDynamic()
+            {
+                return _factory.BuildEntityFromDynamic(_entity);
+            }
+
+            public int Id
+            {
+                get { return _entity.id; }
+            }
+
+            public int VersionId
+            {
+                get
+                {
+                    if (_isContent || _isMedia)
+                    {
+                        return _entity.versionId;
+                    }
+                    return _entity.id;
+                }
+            }
+        }
         #endregion
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentTypeRepository.cs

@@ -8,6 +8,13 @@ namespace Umbraco.Core.Persistence.Repositories
 {
     public interface IContentTypeRepository : IContentTypeCompositionRepository<IContentType>
     {
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        bool HasContainerInPath(string contentPath);
+
         /// <summary>
         /// Gets all entities of the specified <see cref="PropertyType"/> query
         /// </summary>

src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs

@@ -20,7 +20,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="serializer">The serializer to convert TEntity to Xml</param>
         /// <param name="groupSize">Structures will be rebuilt in chunks of this size</param>
         /// <param name="contentTypeIds"></param>
-        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null);
+        void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null);
 
         /// <summary>
         /// Get the total count of entities

src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs

@@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMediaFromDto(dto, dto.VersionId, sql);
+            var content = CreateMediaFromDto(dto, sql);
 
             return content;
         }
@@ -94,6 +94,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 .InnerJoin<NodeDto>(SqlSyntax)
                 .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax)
+                //TODO: IF we want to enable querying on content type information this will need to be joined
+                //.InnerJoin<ContentTypeDto>(SqlSyntax)
+                //.On<ContentDto, ContentTypeDto>(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax);
                 .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
             return sql;
         }
@@ -161,25 +164,27 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sqlFull);
-            var content = new IMedia[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMedia, bool>>();
+            var defs = new DocumentDefinitionCollection();
 
             //track the looked up content types, even though the content types are cached
             // they still need to be deep cloned out of the cache and we don't want to add
             // the overhead of deep cloning them on every item in this loop
             var contentTypes = new Dictionary<int, IMediaType>();
 
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
-
                 // if the cache contains the item, use it
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem<IMedia>(GetCacheIdKey<IMedia>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, media doesn't 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.VersionId)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMedia, bool>(cached, true));
                         continue;
                     }
                 }
@@ -197,39 +202,34 @@ namespace Umbraco.Core.Persistence.Repositories
                     contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
                     contentTypes[dto.ContentDto.ContentTypeId] = contentType;
                 }
-                
-                content[i] = MediaFactory.BuildEntity(dto, contentType);
 
-                // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.VersionId,
-                    dto.VersionDate,
-                    dto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                {
+                    content.Add(new Tuple<IMedia, bool>(MediaFactory.BuildEntity(dto, contentType), false));
+                }
             }
 
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
 
-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         public override IMedia GetByVersion(Guid versionId)
@@ -243,7 +243,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMediaFromDto(dto, versionId, sql);
+            var content = CreateMediaFromDto(dto, sql);
 
             return content;
         }
@@ -263,10 +263,12 @@ namespace Umbraco.Core.Persistence.Repositories
                 // get the next group of nodes
                 var query = GetBaseQuery(false);
                 if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                {
+                    query = query.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
                 query = query
                     .Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
+                    .Where<NodeDto>(x => x.Trashed == false, SqlSyntax)
                     .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
                 var sql = SqlSyntax.SelectTop(query, groupSize);
                 var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
@@ -290,7 +292,38 @@ namespace Umbraco.Core.Persistence.Repositories
                         Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
                     }
                 }
-                baseId = xmlItems.Last().NodeId;
+                baseId = xmlItems[xmlItems.Count - 1].NodeId;
+            }
+
+            //now delete the items that shouldn't be there
+            var allMediaIds = Database.Fetch<int>(GetBaseQuery(BaseQueryType.Ids).Where<NodeDto>(x => x.Trashed == false, SqlSyntax));
+            var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From<ContentXmlDto>(SqlSyntax)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentXmlDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+                
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.InnerJoin<ContentDto>(SqlSyntax)
+                    .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin<ContentTypeDto>(SqlSyntax)
+                    .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
+            xmlIdsQuery.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+            
+            var allXmlIds = Database.Fetch<int>(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allMediaIds).ToArray();
+            if (toRemove.Length > 0)
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }
             }
         }
 
@@ -528,20 +561,19 @@ namespace Umbraco.Core.Persistence.Repositories
         /// Private method to create a media object from a ContentDto
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
+        private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql)
         {
             var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
             
             var media = MediaFactory.BuildEntity(dto, contentType);
 
-            var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
+            var docDef = new DocumentDefinition(dto, contentType);
 
             var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });
 
-            media.Properties = properties[dto.NodeId];
+            media.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs

@@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories
             if (dto == null)
                 return null;
 
-            var content = CreateMemberFromDto(dto, dto.ContentVersionDto.VersionId, sql);
+            var content = CreateMemberFromDto(dto, sql);
 
             return content;
 
@@ -448,16 +448,16 @@ namespace Umbraco.Core.Persistence.Repositories
             var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
             var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            var member = factory.BuildEntity(dto);
 
-            var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
+            var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) });
 
-            media.Properties = properties[dto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            ((Entity)member).ResetDirtyProperties(false);
+            return member;
 
         }
 
@@ -681,20 +681,21 @@ namespace Umbraco.Core.Persistence.Repositories
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
 
-            var content = new IMember[dtos.Count];
-            var defs = new List<DocumentDefinition>();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List<Tuple<IMember, bool>>();
+            var defs = new DocumentDefinitionCollection();
 
-            for (var i = 0; i < dtos.Count; i++)
+            foreach (var dto in dtos)
             {
-                var dto = dtos[i];
-
                 // if the cache contains the item, use it
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem<IMember>(GetCacheIdKey<IMember>(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, members dont 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.ContentVersionDto.VersionId)
                     {
-                        content[i] = cached;
+                        content.Add(new Tuple<IMember, bool>(cached, true));
                         continue;
                     }
                 }
@@ -702,60 +703,54 @@ namespace Umbraco.Core.Persistence.Repositories
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
                 var contentType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new MemberFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
 
                 // need properties
-                defs.Add(new DocumentDefinition(
-                    dto.NodeId,
-                    dto.ContentVersionDto.VersionId,
-                    dto.ContentVersionDto.VersionDate,
-                    dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
-                    contentType
-                ));
+                if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType)))
+                {
+                    content.Add(new Tuple<IMember, bool>(MemberFactory.BuildEntity(dto, contentType), false));
+                }
             }
 
             // load all properties for all documents from database in 1 query
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
-            // assign
-            var dtoIndex = 0;
-            foreach (var def in defs)
+            // assign property data
+            foreach (var contentItem in content)
             {
-                // move to corresponding item (which has to exist)
-                while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++;
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
 
-                // complete the item
-                var cc = content[dtoIndex];
-                cc.Properties = propertyData[cc.Id];
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity)cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// <summary>
         /// Private method to create a member object from a MemberDto
         /// </summary>
         /// <param name="dto"></param>
-        /// <param name="versionId"></param>
         /// <param name="docSql"></param>
         /// <returns></returns>
-        private IMember CreateMemberFromDto(MemberDto dto, Guid versionId, Sql docSql)
+        private IMember CreateMemberFromDto(MemberDto dto, Sql docSql)
         {
             var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
             var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.ContentVersionDto.NodeId);
             var member = factory.BuildEntity(dto);
 
-            var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType);
+            var docDef = new DocumentDefinition(dto.ContentVersionDto, memberType);
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            member.Properties = properties[dto.ContentVersionDto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946

src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs

@@ -44,10 +44,9 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 sql.Where("umbracoAccess.id IN (@ids)", new { ids = ids });
             }
-
-            sql.OrderBy<AccessDto>(x => x.NodeId, SqlSyntax);
-
+            
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -59,6 +58,7 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = translator.Translate();
 
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch<AccessDto, AccessRuleDto, AccessDto>(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -69,7 +69,9 @@ namespace Umbraco.Core.Persistence.Repositories
             sql.Select("*")
                 .From<AccessDto>(SqlSyntax)
                 .LeftJoin<AccessRuleDto>(SqlSyntax)
-                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId);
+                .On<AccessDto, AccessRuleDto>(SqlSyntax, left => left.Id, right => right.AccessId)
+                //MUST be ordered by this GUID ID for the AccessRulesRelator to work
+                .OrderBy<AccessDto>(dto => dto.Id, SqlSyntax);
 
             return sql;
         }
@@ -110,6 +112,11 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 rule.AccessId = entity.Key;
                 Database.Insert(rule);
+                //update the id so HasEntity is correct
+                var entityRule = entity.Rules.First(x => x.Key == rule.Id);
+                entityRule.Id = entityRule.Key.GetHashCode();
+                //double make sure that this is set since it is possible to add rules via ctor without AddRule
+                entityRule.AccessEntryId = entity.Key;
             }
 
             entity.ResetDirtyProperties();
@@ -130,7 +137,7 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 if (rule.HasIdentity)
                 {
-                    var count = Database.Update(dto.Rules.Single(x => x.Id == rule.Key));
+                    var count = Database.Update(dto.Rules.First(x => x.Id == rule.Key));
                     if (count == 0)
                     {
                         throw new InvalidOperationException("No rows were updated for the access rule");
@@ -156,6 +163,8 @@ namespace Umbraco.Core.Persistence.Repositories
                 Database.Delete<AccessRuleDto>("WHERE id=@Id", new {Id = removedRule});
             }
 
+            entity.ClearRemovedRules();
+
             entity.ResetDirtyProperties();
         }
 
@@ -164,4 +173,4 @@ namespace Umbraco.Core.Persistence.Repositories
             return entity.Key;
         }
     }
-}
+}

src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Data.SqlTypes;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -32,6 +33,11 @@ namespace Umbraco.Core.Persistence.Repositories
     {
         private readonly IContentSection _contentSection;
 
+        /// <summary>
+        /// This is used for unit tests ONLY
+        /// </summary>
+        internal static bool ThrowOnWarning = false;
+
         protected VersionableRepositoryBase(IDatabaseUnitOfWork work, CacheHelper cache, ILogger logger, ISqlSyntaxProvider sqlSyntax, IContentSection contentSection)
             : base(work, cache, logger, sqlSyntax)
         {
@@ -502,7 +508,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="sql"></param>
         /// <param name="documentDefs"></param>
         /// <returns></returns>
-        protected IDictionary<int, PropertyCollection> GetPropertyCollection(
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
             Sql sql,
             IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
@@ -515,11 +521,11 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="pagingSqlQuery"></param>
         /// <param name="documentDefs"></param>
         /// <returns></returns>
-        protected IDictionary<int, PropertyCollection> GetPropertyCollection(
+        protected IDictionary<Guid, PropertyCollection> GetPropertyCollection(
             PagingSqlQuery pagingSqlQuery,
             IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
-            if (documentDefs.Count == 0) return new Dictionary<int, PropertyCollection>();
+            if (documentDefs.Count == 0) return new Dictionary<Guid, PropertyCollection>();
 
             //initialize to the query passed in
             var docSql = pagingSqlQuery.PrePagedSql;
@@ -575,16 +581,16 @@ ON cmsPropertyData.propertytypeid = cmsPropertyType.id
 INNER JOIN 
 	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
 ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-ORDER BY contentNodeId, propertytypeid
+ORDER BY contentNodeId, versionId, propertytypeid
 ", docSql.Arguments);
-
+            
             //This does NOT fetch all data into memory in a list, this will read
             // over the records as a data reader, this is much better for performance and memory,
             // but it means that during the reading of this data set, nothing else can be read
             // from SQL server otherwise we'll get an exception.
             var allPropertyData = Database.Query<PropertyDataDto>(propSql);
 
-            var result = new Dictionary<int, PropertyCollection>();
+            var result = new Dictionary<Guid, PropertyCollection>();
             var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
             //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
             var resolvedCompositionProperties = new Dictionary<int, PropertyType[]>();
@@ -593,11 +599,13 @@ ORDER BY contentNodeId, propertytypeid
             var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
             var hasCurrent = false; // initially there is no enumerator.Current
 
+            var comparer = new DocumentDefinitionComparer(SqlSyntax);
+
             try
             {
                 //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
                 // which allows us to more efficiently iterate over the large data set of property values
-                foreach (var def in documentDefs.OrderBy(x => x.Id))
+                foreach (var def in documentDefs.OrderBy(x => x.Id).ThenBy(x => x.Version, comparer))
                 {
                     // get the resolved properties from our local cache, or resolve them and put them in cache
                     PropertyType[] compositionProperties;
@@ -616,15 +624,17 @@ ORDER BY contentNodeId, propertytypeid
                     var propertyDataDtos = new List<PropertyDataDto>();
                     while (hasCurrent || propertyDataSetEnumerator.MoveNext())
                     {
-                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        //Not checking null on VersionId because it can never be null - no idea why it's set to nullable
+                        // ReSharper disable once PossibleInvalidOperationException
+                        if (propertyDataSetEnumerator.Current.VersionId.Value == def.Version)
                         {
                             hasCurrent = false; // enumerator.Current is not available
                             propertyDataDtos.Add(propertyDataSetEnumerator.Current);
                         }
                         else
                         {
-                            hasCurrent = true; // enumerator.Current is available for another def
-                            break; // no more propertyDataDto for this def
+                            hasCurrent = true;  // enumerator.Current is available for another def
+                            break;              // no more propertyDataDto for this def
                         }
                     }
 
@@ -661,11 +671,19 @@ ORDER BY contentNodeId, propertytypeid
                         }
                     }
 
-                    if (result.ContainsKey(def.Id))
+                    if (result.ContainsKey(def.Version))
                     {
-                        Logger.Warn<VersionableRepositoryBase<TId, TEntity>>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name);
+                        var msg = string.Format("The query returned multiple property sets for document definition {0}, {1}, {2}", def.Id, def.Version, def.Composition.Name);
+                        if (ThrowOnWarning)
+                        {
+                            throw new InvalidOperationException(msg);
+                        }
+                        else
+                        {
+                            Logger.Warn<VersionableRepositoryBase<TId, TEntity>>(msg);
+                        }
                     }
-                    result[def.Id] = new PropertyCollection(properties);
+                    result[def.Version] = new PropertyCollection(properties);
                 }
             }
             finally
@@ -688,8 +706,8 @@ ORDER BY contentNodeId, propertytypeid
                 case "NAME":
                     return "umbracoNode.text";
                 case "PUBLISHED":
-                    return "cmsDocument.published";
                 case "OWNER":
+                    return "cmsDocument.published";
                     //TODO: This isn't going to work very nicely because it's going to order by ID, not by letter
                     return "umbracoNode.nodeUser";
                 // Members only
@@ -773,25 +791,159 @@ ORDER BY contentNodeId, propertytypeid
         /// <returns></returns>
         protected abstract Sql GetBaseQuery(BaseQueryType queryType);
 
+        internal class DocumentDefinitionCollection : KeyedCollection<ValueType, DocumentDefinition>
+        {
+            private readonly bool _includeAllVersions;
+
+            /// <summary>
+            /// Constructor specifying if all versions should be allowed, in that case the key for the collection becomes the versionId (GUID)
+            /// </summary>
+            /// <param name="includeAllVersions"></param>
+            public DocumentDefinitionCollection(bool includeAllVersions = false)
+            {
+                _includeAllVersions = includeAllVersions;
+            }
+
+            protected override ValueType GetKeyForItem(DocumentDefinition item)
+            {
+                return _includeAllVersions ? (ValueType)item.Version : item.Id;
+            }
+
+            /// <summary>
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// </summary>
+            /// <param name="item"></param>
+            /// <returns></returns>
+            public bool AddOrUpdate(DocumentDefinition item)
+            {
+                //if we are including all versions then just add, we aren't checking for latest
+                if (_includeAllVersions)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                DocumentDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    if (item.VersionId > found.VersionId)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Version);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+                
+                base.Add(item);
+                return true;
+            }
+          
+            public bool TryGetValue(ValueType key, out DocumentDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        /// <summary>
+        /// A custom comparer required for sorting entities by GUIDs to match how the sorting of GUIDs works on SQL server
+        /// </summary>
+        /// <remarks>
+        /// MySql sorts GUIDs as a string, MSSQL sorts based on byte sections, this comparer will allow sorting GUIDs to be the same as how SQL server does
+        /// </remarks>
+        private class DocumentDefinitionComparer : IComparer<Guid>
+        {
+            private readonly ISqlSyntaxProvider _sqlSyntax;
+
+            public DocumentDefinitionComparer(ISqlSyntaxProvider sqlSyntax)
+            {
+                _sqlSyntax = sqlSyntax;
+            }
+
+            public int Compare(Guid x, Guid y)
+            {
+                //MySql sorts on GUIDs as strings (i.e. normal)
+                if (_sqlSyntax is MySqlSyntaxProvider)
+                {
+                    return x.CompareTo(y);
+                }
+
+                //MSSQL doesn't it sorts them on byte sections!
+                return new SqlGuid(x).CompareTo(new SqlGuid(y));
+            }
+        }
+
         internal class DocumentDefinition
         {
             /// <summary>
             /// Initializes a new instance of the <see cref="T:System.Object"/> class.
             /// </summary>
-            public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition)
+            public DocumentDefinition(DocumentDto dto, IContentTypeComposition composition)
             {
-                Id = id;
-                Version = version;
-                VersionDate = versionDate;
-                CreateDate = createDate;
+                DocumentDto = dto;
+                ContentVersionDto = dto.ContentVersionDto;
                 Composition = composition;
             }
 
-            public int Id { get; set; }
-            public Guid Version { get; set; }
-            public DateTime VersionDate { get; set; }
-            public DateTime CreateDate { get; set; }
-            public IContentTypeComposition Composition { get; set; }
+            public DocumentDefinition(ContentVersionDto dto, IContentTypeComposition composition)
+            {
+                ContentVersionDto = dto;
+                Composition = composition;
+            }
+
+            public DocumentDto DocumentDto { get; private set; }
+            public ContentVersionDto ContentVersionDto { get; private set; }
+
+            public int Id
+            {
+                get { return ContentVersionDto.NodeId; }
+            }
+            
+            public Guid Version
+            {
+                get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; }
+            }
+
+            /// <summary>
+            /// This is used to determien which version is the most recent
+            /// </summary>
+            public int VersionId
+            {
+                get { return ContentVersionDto.Id; }
+            }
+
+            public DateTime VersionDate
+            {
+                get { return ContentVersionDto.VersionDate; }
+            }
+
+            public DateTime CreateDate
+            {
+                get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; }
+            }
+
+            public IContentTypeComposition Composition { get; set; }            
+
+            
         }
 
         /// <summary>

src/Umbraco.Core/Persistence/RepositoryFactory.cs

@@ -48,7 +48,12 @@ namespace Umbraco.Core.Persistence
                 _cacheHelper.IsolatedRuntimeCache.CacheFactory = type =>
                 {
                     var cache = origFactory(type);
-                    return new DeepCloneRuntimeCacheProvider(cache);
+
+                    //if the result is already a DeepCloneRuntimeCacheProvider then return it, otherwise
+                    //wrap the result with a DeepCloneRuntimeCacheProvider
+                    return cache is DeepCloneRuntimeCacheProvider 
+                        ? cache 
+                        : new DeepCloneRuntimeCacheProvider(cache);
                 };
             }
 

src/Umbraco.Core/Publishing/PublishStatus.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using Umbraco.Core.Events;
 using Umbraco.Core.Models;
 using Umbraco.Core.Services;
@@ -12,7 +13,8 @@ namespace Umbraco.Core.Publishing
     {
         public PublishStatus(IContent content, PublishStatusType statusType, EventMessages eventMessages)
             : base(content, statusType, eventMessages)
-        {            
+        {
+            InvalidProperties = Enumerable.Empty<Property>();
         }
 
         /// <summary>
@@ -20,8 +22,7 @@ namespace Umbraco.Core.Publishing
         /// </summary>
         public PublishStatus(IContent content, EventMessages eventMessages)
             : this(content, PublishStatusType.Success, eventMessages)
-        {
-        }
+        { }
 
         public IContent ContentItem
         {

src/Umbraco.Core/Publishing/PublishingStrategy.cs

@@ -30,7 +30,7 @@ namespace Umbraco.Core.Publishing
         /// Publishes a single piece of Content
         /// </summary>
         /// <param name="content"><see cref="IContent"/> to publish</param>
-        /// <param name="userId">Id of the User issueing the publish operation</param>        
+        /// <param name="userId">Id of the User issueing the publish operation</param>
         internal Attempt<PublishStatus> PublishInternal(IContent content, int userId)
         {
             var evtMsgs = _eventMessagesFactory.Get();
@@ -99,26 +99,26 @@ namespace Umbraco.Core.Publishing
         /// By default this is set to true which means that it will publish any content item in the list that is completely unpublished and
         /// not visible on the front-end. If set to false, this will only publish content that is live on the front-end but has new versions
         /// that have yet to be published.
-        /// </param>        
+        /// </param>
         /// <returns></returns>
         /// <remarks>
-        /// 
+        ///
         /// This method becomes complex once we start to be able to cancel events or stop publishing a content item in any way because if a
         /// content item is not published then it's children shouldn't be published either. This rule will apply for the following conditions:
         /// * If a document fails to be published, do not proceed to publish it's children if:
         /// ** The document does not have a publish version
         /// ** The document does have a published version but the includeUnpublishedDocuments = false
-        /// 
+        ///
         /// In order to do this, we will order the content by level and begin by publishing each item at that level, then proceed to the next
-        /// level and so on. If we detect that the above rule applies when the document publishing is cancelled we'll add it to the list of 
+        /// level and so on. If we detect that the above rule applies when the document publishing is cancelled we'll add it to the list of
         /// parentsIdsCancelled so that it's children don't get published.
-        /// 
+        ///
         /// Its important to note that all 'root' documents included in the list *will* be published regardless of the rules mentioned
-        /// above (unless it is invalid)!! By 'root' documents we are referring to documents in the list with the minimum value for their 'level'. 
-        /// In most cases the 'root' documents will only be one document since under normal circumstance we only publish one document and 
+        /// above (unless it is invalid)!! By 'root' documents we are referring to documents in the list with the minimum value for their 'level'.
+        /// In most cases the 'root' documents will only be one document since under normal circumstance we only publish one document and
         /// its children. The reason we have to do this is because if a user is publishing a document and it's children, it is implied that
         /// the user definitely wants to publish it even if it has never been published before.
-        /// 
+        ///
         /// </remarks>
         internal IEnumerable<Attempt<PublishStatus>> PublishWithChildrenInternal(
             IEnumerable<IContent> content, int userId, bool includeUnpublishedDocuments = true)
@@ -132,7 +132,7 @@ namespace Umbraco.Core.Publishing
 
             //group by levels and iterate over the sorted ascending level.
             //TODO: This will cause all queries to execute, they will not be lazy but I'm not really sure being lazy actually made
-            // much difference because we iterate over them all anyways?? Morten?       
+            // much difference because we iterate over them all anyways?? Morten?
             // Because we're grouping I think this will execute all the queries anyways so need to fetch it all first.
             var fetchedContent = content.ToArray();
 
@@ -149,7 +149,7 @@ namespace Umbraco.Core.Publishing
             var levelGroups = fetchedContent.GroupBy(x => x.Level);
             foreach (var level in levelGroups.OrderBy(x => x.Key))
             {
-                //set the first level flag, used to ensure that all documents at the first level will 
+                //set the first level flag, used to ensure that all documents at the first level will
                 //be published regardless of the rules mentioned in the remarks.
                 if (!firstLevel.HasValue)
                 {
@@ -200,7 +200,10 @@ namespace Umbraco.Core.Publishing
                         _logger.Info<PublishingStrategy>(
                             string.Format("Content '{0}' with Id '{1}' will not be published because some of it's content is not passing validation rules.",
                                           item.Name, item.Id));
-                        statuses.Add(Attempt.Fail(new PublishStatus(item, PublishStatusType.FailedContentInvalid, evtMsgs)));
+                        statuses.Add(Attempt.Fail(new PublishStatus(item, PublishStatusType.FailedContentInvalid, evtMsgs)
+                        {
+                            InvalidProperties = ((ContentBase)item).LastInvalidProperties
+                        }));
 
                         //Does this document apply to our rule to cancel it's children being published?
                         CheckCancellingOfChildPublishing(item, parentsIdsCancelled, includeUnpublishedDocuments);
@@ -272,11 +275,11 @@ namespace Umbraco.Core.Publishing
         /// <param name="includeUnpublishedDocuments"></param>
         /// <remarks>
         /// See remarks on method: PublishWithChildrenInternal
-        /// </remarks> 
+        /// </remarks>
         private void CheckCancellingOfChildPublishing(IContent content, List<int> parentsIdsCancelled, bool includeUnpublishedDocuments)
         {
             //Does this document apply to our rule to cancel it's children being published?
-            //TODO: We're going back to the service layer here... not sure how to avoid this? And this will add extra overhead to 
+            //TODO: We're going back to the service layer here... not sure how to avoid this? And this will add extra overhead to
             // any document that fails to publish...
             var hasPublishedVersion = ApplicationContext.Current.Services.ContentService.HasPublishedVersion(content.Id);
 
@@ -306,8 +309,8 @@ namespace Umbraco.Core.Publishing
             //NOTE: This previously always returned true so I've left it that way. It returned true because (from Morten)...
             // ... if one item couldn't be published it wouldn't be correct to return false.
             // in retrospect it should have returned a list of with Ids and Publish Status
-            // come to think of it ... the cache would still be updated for a failed item or at least tried updated. 
-            // It would call the Published event for the entire list, but if the Published property isn't set to True it 
+            // come to think of it ... the cache would still be updated for a failed item or at least tried updated.
+            // It would call the Published event for the entire list, but if the Published property isn't set to True it
             // wouldn't actually update the cache for that item. But not really ideal nevertheless...
             return true;
         }
@@ -345,7 +348,7 @@ namespace Umbraco.Core.Publishing
             var evtMsgs = _eventMessagesFactory.Get();
 
             //Fire UnPublishing event
-            if (UnPublishing.IsRaisedEventCancelled(                
+            if (UnPublishing.IsRaisedEventCancelled(
                 new PublishEventArgs<IContent>(content, evtMsgs), this))
             {
                 _logger.Info<PublishingStrategy>(
@@ -388,8 +391,8 @@ namespace Umbraco.Core.Publishing
             //NOTE: This previously always returned true so I've left it that way. It returned true because (from Morten)...
             // ... if one item couldn't be published it wouldn't be correct to return false.
             // in retrospect it should have returned a list of with Ids and Publish Status
-            // come to think of it ... the cache would still be updated for a failed item or at least tried updated. 
-            // It would call the Published event for the entire list, but if the Published property isn't set to True it 
+            // come to think of it ... the cache would still be updated for a failed item or at least tried updated.
+            // It would call the Published event for the entire list, but if the Published property isn't set to True it
             // wouldn't actually update the cache for that item. But not really ideal nevertheless...
             return true;
         }
@@ -405,7 +408,7 @@ namespace Umbraco.Core.Publishing
         public override void PublishingFinalized(IContent content)
         {
             var evtMsgs = _eventMessagesFactory.Get();
-            Published.RaiseEvent(                
+            Published.RaiseEvent(
                 new PublishEventArgs<IContent>(content, false, false, evtMsgs), this);
         }
 

src/Umbraco.Core/Security/BackOfficeSignInManager.cs

@@ -129,6 +129,7 @@ namespace Umbraco.Core.Security
 
             //track the last login date
             user.LastLoginDateUtc = DateTime.UtcNow;
+            user.AccessFailedCount = 0;
             await UserManager.UpdateAsync(user);
 
             _logger.WriteCore(TraceEventType.Information, 0,
@@ -138,4 +139,4 @@ namespace Umbraco.Core.Security
                     _request.RemoteIpAddress), null, null);
         }
     }
-}
+}

src/Umbraco.Core/Services/ContentService.cs

@@ -2185,6 +2185,7 @@ namespace Umbraco.Core.Services
                 //We need to check if children and their publish state to ensure that we 'republish' content that was previously published
                 if (published && previouslyPublished == false && HasChildren(content.Id))
                 {
+                    //TODO: Horrible for performance if there are lots of descendents! We should page if anything but this is crazy
                     var descendants = GetPublishedDescendants(content);
 
                     _publishingStrategy.PublishingFinalized(descendants, false);

src/Umbraco.Core/Services/ContentTypeService.cs

@@ -1,22 +1,15 @@
 using System;
 using System.Collections.Generic;
-using System.Data;
-using System.Diagnostics;
 using System.Linq;
 using System.Text;
-using System.Xml.Linq;
 using System.Threading;
-using AutoMapper;
-using Umbraco.Core.Auditing;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.Exceptions;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
-using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Persistence.UnitOfWork;
 
 namespace Umbraco.Core.Services
@@ -26,14 +19,15 @@ namespace Umbraco.Core.Services
     /// </summary>
     public class ContentTypeService : ContentTypeServiceBase, IContentTypeService
     {
-	    private readonly IContentService _contentService;
+        private readonly IContentService _contentService;
         private readonly IMediaService _mediaService;
 
         //Support recursive locks because some of the methods that require locking call other methods that require locking.
         //for example, the Move method needs to be locked but this calls the Save method which also needs to be locked.
         private static readonly ReaderWriterLockSlim Locker = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
 
-        public ContentTypeService(IDatabaseUnitOfWorkProvider provider, RepositoryFactory repositoryFactory, ILogger logger, IEventMessagesFactory eventMessagesFactory, IContentService contentService, IMediaService mediaService)
+        public ContentTypeService(IDatabaseUnitOfWorkProvider provider, RepositoryFactory repositoryFactory, ILogger logger, IEventMessagesFactory eventMessagesFactory, IContentService contentService,
+            IMediaService mediaService)
             : base(provider, repositoryFactory, logger, eventMessagesFactory)
         {
             if (contentService == null) throw new ArgumentNullException("contentService");
@@ -154,8 +148,8 @@ namespace Umbraco.Core.Services
             }
 
             if (savingEvent.IsRaisedEventCancelled(
-                        new SaveEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                new SaveEventArgs<EntityContainer>(container, evtMsgs),
+                this))
             {
                 return OperationStatus.Cancelled(evtMsgs);
             }
@@ -214,7 +208,7 @@ namespace Umbraco.Core.Services
 
         public IEnumerable<EntityContainer> GetMediaTypeContainers(IMediaType mediaType)
         {
-            var ancestorIds = mediaType.Path.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
+            var ancestorIds = mediaType.Path.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries)
                 .Select(x =>
                 {
                     var asInt = x.TryConvertTo<int>();
@@ -290,8 +284,8 @@ namespace Umbraco.Core.Services
                 if (container == null) return OperationStatus.NoOperation(evtMsgs);
 
                 if (DeletingContentTypeContainer.IsRaisedEventCancelled(
-                        new DeleteEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                    new DeleteEventArgs<EntityContainer>(container, evtMsgs),
+                    this))
                 {
                     return Attempt.Fail(new OperationStatus(OperationStatusType.FailedCancelledByEvent, evtMsgs));
                 }
@@ -316,8 +310,8 @@ namespace Umbraco.Core.Services
                 if (container == null) return OperationStatus.NoOperation(evtMsgs);
 
                 if (DeletingMediaTypeContainer.IsRaisedEventCancelled(
-                        new DeleteEventArgs<EntityContainer>(container, evtMsgs),
-                        this))
+                    new DeleteEventArgs<EntityContainer>(container, evtMsgs),
+                    this))
                 {
                     return Attempt.Fail(new OperationStatus(OperationStatusType.FailedCancelledByEvent, evtMsgs));
                 }
@@ -421,7 +415,7 @@ namespace Umbraco.Core.Services
 
             clone.Name = name;
 
-            var compositionAliases = clone.CompositionAliases().Except(new[] { alias }).ToList();
+            var compositionAliases = clone.CompositionAliases().Except(new[] {alias}).ToList();
             //remove all composition that is not it's current alias
             foreach (var a in compositionAliases)
             {
@@ -680,7 +674,7 @@ namespace Umbraco.Core.Services
             var comparer = new DelegateEqualityComparer<IContentTypeComposition>((x, y) => x.Id == y.Id, x => x.Id);
             var dependencies = new HashSet<IContentTypeComposition>(compositions, comparer);
             var stack = new Stack<IContentTypeComposition>();
-            indirectReferences.ForEach(stack.Push);//Push indirect references to a stack, so we can add recursively
+            indirectReferences.ForEach(stack.Push); //Push indirect references to a stack, so we can add recursively
             while (stack.Count > 0)
             {
                 var indirectReference = stack.Pop();
@@ -734,6 +728,8 @@ namespace Umbraco.Core.Services
                 {
                     ValidateLocked(contentType); // throws if invalid
                     contentType.CreatorId = userId;
+                    if (contentType.Description == string.Empty)
+                        contentType.Description = null;
                     repository.AddOrUpdate(contentType);
 
                     uow.Commit();
@@ -742,7 +738,7 @@ namespace Umbraco.Core.Services
                 UpdateContentXmlStructure(contentType);
             }
             SavedContentType.RaiseEvent(new SaveEventArgs<IContentType>(contentType, false), this);
-	        Audit(AuditType.Save, string.Format("Save ContentType performed by user"), userId, contentType.Id);
+            Audit(AuditType.Save, string.Format("Save ContentType performed by user"), userId, contentType.Id);
         }
 
         /// <summary>
@@ -755,7 +751,7 @@ namespace Umbraco.Core.Services
             var asArray = contentTypes.ToArray();
 
             if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(asArray), this))
-				return;
+                return;
 
             using (new WriteLock(Locker))
             {
@@ -770,6 +766,8 @@ namespace Umbraco.Core.Services
                     foreach (var contentType in asArray)
                     {
                         contentType.CreatorId = userId;
+                        if (contentType.Description == string.Empty)
+                            contentType.Description = null;
                         repository.AddOrUpdate(contentType);
                     }
 
@@ -780,7 +778,7 @@ namespace Umbraco.Core.Services
                 UpdateContentXmlStructure(asArray.Cast<IContentTypeBase>().ToArray());
             }
             SavedContentType.RaiseEvent(new SaveEventArgs<IContentType>(asArray, false), this);
-	        Audit(AuditType.Save, string.Format("Save ContentTypes performed by user"), userId, -1);
+            Audit(AuditType.Save, string.Format("Save ContentTypes performed by user"), userId, -1);
         }
 
         /// <summary>
@@ -791,8 +789,8 @@ namespace Umbraco.Core.Services
         /// <remarks>Deleting a <see cref="IContentType"/> will delete all the <see cref="IContent"/> objects based on this <see cref="IContentType"/></remarks>
         public void Delete(IContentType contentType, int userId = 0)
         {
-	        if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(contentType), this))
-				return;
+            if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(contentType), this))
+                return;
 
             using (new WriteLock(Locker))
             {
@@ -836,7 +834,7 @@ namespace Umbraco.Core.Services
             var asArray = contentTypes.ToArray();
 
             if (DeletingContentType.IsRaisedEventCancelled(new DeleteEventArgs<IContentType>(asArray), this))
-				return;
+                return;
 
             using (new WriteLock(Locker))
             {
@@ -1004,8 +1002,8 @@ namespace Umbraco.Core.Services
             var evtMsgs = EventMessagesFactory.Get();
 
             if (MovingMediaType.IsRaisedEventCancelled(
-                  new MoveEventArgs<IMediaType>(evtMsgs, new MoveEventInfo<IMediaType>(toMove, toMove.Path, containerId)),
-                  this))
+                new MoveEventArgs<IMediaType>(evtMsgs, new MoveEventInfo<IMediaType>(toMove, toMove.Path, containerId)),
+                this))
             {
                 return Attempt.Fail(
                     new OperationStatus<MoveOperationStatusType>(
@@ -1047,8 +1045,8 @@ namespace Umbraco.Core.Services
             var evtMsgs = EventMessagesFactory.Get();
 
             if (MovingContentType.IsRaisedEventCancelled(
-                  new MoveEventArgs<IContentType>(evtMsgs, new MoveEventInfo<IContentType>(toMove, toMove.Path, containerId)),
-                  this))
+                new MoveEventArgs<IContentType>(evtMsgs, new MoveEventInfo<IContentType>(toMove, toMove.Path, containerId)),
+                this))
             {
                 return Attempt.Fail(
                     new OperationStatus<MoveOperationStatusType>(
@@ -1178,8 +1176,8 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Optional Id of the user saving the MediaType</param>
         public void Save(IMediaType mediaType, int userId = 0)
         {
-	        if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(mediaType), this))
-				return;
+            if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(mediaType), this))
+                return;
 
             using (new WriteLock(Locker))
             {
@@ -1188,6 +1186,8 @@ namespace Umbraco.Core.Services
                 {
                     ValidateLocked(mediaType); // throws if invalid
                     mediaType.CreatorId = userId;
+                    if (mediaType.Description == string.Empty)
+                        mediaType.Description = null;
                     repository.AddOrUpdate(mediaType);
                     uow.Commit();
 
@@ -1197,7 +1197,7 @@ namespace Umbraco.Core.Services
             }
 
             SavedMediaType.RaiseEvent(new SaveEventArgs<IMediaType>(mediaType, false), this);
-	        Audit(AuditType.Save, string.Format("Save MediaType performed by user"), userId, mediaType.Id);
+            Audit(AuditType.Save, string.Format("Save MediaType performed by user"), userId, mediaType.Id);
         }
 
         /// <summary>
@@ -1210,7 +1210,7 @@ namespace Umbraco.Core.Services
             var asArray = mediaTypes.ToArray();
 
             if (SavingMediaType.IsRaisedEventCancelled(new SaveEventArgs<IMediaType>(asArray), this))
-				return;
+                return;
 
             using (new WriteLock(Locker))
             {
@@ -1225,6 +1225,8 @@ namespace Umbraco.Core.Services
                     foreach (var mediaType in asArray)
                     {
                         mediaType.CreatorId = userId;
+                        if (mediaType.Description == string.Empty)
+                            mediaType.Description = null;
                         repository.AddOrUpdate(mediaType);
                     }
 
@@ -1236,7 +1238,7 @@ namespace Umbraco.Core.Services
             }
 
             SavedMediaType.RaiseEvent(new SaveEventArgs<IMediaType>(asArray, false), this);
-			Audit(AuditType.Save, string.Format("Save MediaTypes performed by user"), userId, -1);
+            Audit(AuditType.Save, string.Format("Save MediaTypes performed by user"), userId, -1);
         }
 
         /// <summary>
@@ -1247,8 +1249,8 @@ namespace Umbraco.Core.Services
         /// <remarks>Deleting a <see cref="IMediaType"/> will delete all the <see cref="IMedia"/> objects based on this <see cref="IMediaType"/></remarks>
         public void Delete(IMediaType mediaType, int userId = 0)
         {
-	        if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(mediaType), this))
-				return;
+            if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(mediaType), this))
+                return;
             using (new WriteLock(Locker))
             {
                 _mediaService.DeleteMediaOfType(mediaType.Id, userId);
@@ -1280,7 +1282,7 @@ namespace Umbraco.Core.Services
             var asArray = mediaTypes.ToArray();
 
             if (DeletingMediaType.IsRaisedEventCancelled(new DeleteEventArgs<IMediaType>(asArray), this))
-				return;
+                return;
             using (new WriteLock(Locker))
             {
                 foreach (var mediaType in asArray)
@@ -1390,40 +1392,40 @@ namespace Umbraco.Core.Services
         /// </summary>
         public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletingContentType;
 
-		/// <summary>
-		/// Occurs after Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletedContentType;
+        /// <summary>
+        /// Occurs after Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IContentType>> DeletedContentType;
 
-		/// <summary>
-		/// Occurs before Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletingMediaType;
+        /// <summary>
+        /// Occurs before Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletingMediaType;
 
-		/// <summary>
-		/// Occurs after Delete
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletedMediaType;
+        /// <summary>
+        /// Occurs after Delete
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, DeleteEventArgs<IMediaType>> DeletedMediaType;
 
         /// <summary>
         /// Occurs before Save
         /// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavingContentType;
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavingContentType;
 
         /// <summary>
         /// Occurs after Save
         /// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavedContentType;
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IContentType>> SavedContentType;
 
-		/// <summary>
-		/// Occurs before Save
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavingMediaType;
+        /// <summary>
+        /// Occurs before Save
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavingMediaType;
 
-		/// <summary>
-		/// Occurs after Save
-		/// </summary>
-		public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavedMediaType;
+        /// <summary>
+        /// Occurs after Save
+        /// </summary>
+        public static event TypedEventHandler<IContentTypeService, SaveEventArgs<IMediaType>> SavedMediaType;
 
         /// <summary>
         /// Occurs before Move

src/Umbraco.Core/Services/ContentTypeServiceBase.cs

@@ -5,6 +5,7 @@ using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Persistence.UnitOfWork;
 
 namespace Umbraco.Core.Services
@@ -70,5 +71,20 @@ namespace Umbraco.Core.Services
             return toUpdate;
 
         }
+
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        public bool HasContainerInPath(string contentPath)
+        {
+            using (var uow = UowProvider.GetUnitOfWork())
+            {
+                // can use same repo for both content and media
+                var repository = RepositoryFactory.CreateContentTypeRepository(uow);
+                return repository.HasContainerInPath(contentPath);
+            }
+        }
     }
 }

src/Umbraco.Core/Services/IContentTypeService.cs

@@ -11,6 +11,13 @@ namespace Umbraco.Core.Services
     /// </summary>
     public interface IContentTypeService : IService
     {
+        /// <summary>
+        /// Given the path of a content item, this will return true if the content item exists underneath a list view content item
+        /// </summary>
+        /// <param name="contentPath"></param>
+        /// <returns></returns>
+        bool HasContainerInPath(string contentPath);
+
         int CountContentTypes();
         int CountMediaTypes();
 

src/Umbraco.Core/Services/IMediaService.cs

@@ -80,6 +80,7 @@ namespace Umbraco.Core.Services
         /// </param>
         void RebuildXmlStructures(params int[] contentTypeIds);
 
+        int CountNotTrashed(string contentTypeAlias = null);
         int Count(string contentTypeAlias = null);
         int CountChildren(int parentId, string contentTypeAlias = null);
         int CountDescendants(int parentId, string contentTypeAlias = null);
@@ -166,6 +167,22 @@ namespace Umbraco.Core.Services
         IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter);
 
+        /// <summary>
+        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
+        /// </summary>
+        /// <param name="id">Id of the Parent to retrieve Children from</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
+        /// <param name="filter">Search text filter</param>
+        /// <param name="contentTypeFilter">A list of content type Ids to filter the list by</param>
+        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
+        IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords,
+            string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter);
+
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,

src/Umbraco.Core/Services/IPackagingService.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Xml.Linq;
 using Umbraco.Core.Models;
 
@@ -6,6 +7,15 @@ namespace Umbraco.Core.Services
 {
     public interface IPackagingService : IService
     {
+        /// <summary>
+        /// This will fetch an Umbraco package file from the package repository and return the relative file path to the downloaded package file
+        /// </summary>
+        /// <param name="packageId"></param>
+        /// <param name="umbracoVersion"></param>
+        /// <param name="userId">The current user id performing the operation</param>
+        /// <returns></returns>
+        string FetchPackageFile(Guid packageId, Version umbracoVersion, int userId);
+
         /// <summary>
         /// Imports and saves package xml as <see cref="IContent"/>
         /// </summary>

src/Umbraco.Core/Services/MediaService.cs

@@ -7,7 +7,6 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Xml.Linq;
-using Umbraco.Core.Auditing;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.Logging;
@@ -17,9 +16,7 @@ using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.Repositories;
-using Umbraco.Core.Persistence.SqlSyntax;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Publishing;
 
 namespace Umbraco.Core.Services
 {
@@ -248,6 +245,29 @@ namespace Umbraco.Core.Services
             }
         }
 
+        public int CountNotTrashed(string contentTypeAlias = null)
+        {
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMediaRepository(uow))
+            using (var mediaTypeRepository = RepositoryFactory.CreateMediaTypeRepository(uow))
+            {
+                var mediaTypeId = 0;
+                if (contentTypeAlias.IsNullOrWhiteSpace() == false)
+                {
+                    var mediaType = mediaTypeRepository.Get(contentTypeAlias);
+                    if (mediaType == null) return 0;
+                    mediaTypeId = mediaType.Id;
+                }
+
+                var query = Query<IMedia>.Builder.Where(media => media.Trashed == false);
+                if (mediaTypeId > 0)
+                {
+                    query.Where(media => media.ContentTypeId == mediaTypeId);
+                }
+                return repository.Count(query);
+            }
+        }
+
         public int Count(string contentTypeAlias = null)
         {
             var uow = UowProvider.GetUnitOfWork();
@@ -435,6 +455,25 @@ namespace Umbraco.Core.Services
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         public IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter)
+        {
+            return GetPagedChildren(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter, null);
+        }
+
+        /// <summary>
+        /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
+        /// </summary>
+        /// <param name="id">Id of the Parent to retrieve Children from</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalChildren">Total records query would return without paging</param>
+        /// <param name="orderBy">Field to order by</param>
+        /// <param name="orderDirection">Direction to order by</param>
+        /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
+        /// <param name="filter">Search text filter</param>
+        /// <param name="contentTypeFilter">A list of content type Ids to filter the list by</param>
+        /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
+        public IEnumerable<IMedia> GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren,
+            string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter)
         {
             Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
             Mandate.ParameterCondition(pageSize > 0, "pageSize");
@@ -443,6 +482,11 @@ namespace Umbraco.Core.Services
                 var query = Query<IMedia>.Builder;
                 query.Where(x => x.ParentId == id);
 
+                if (contentTypeFilter != null && contentTypeFilter.Length > 0)
+                {
+                    query.Where(x => contentTypeFilter.Contains(x.ContentTypeId));
+                }
+
                 var medias = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filter);
 
                 return medias;
@@ -847,7 +891,12 @@ namespace Umbraco.Core.Services
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateMediaRepository(uow))
             {
-                media.CreatorId = userId;
+                //set the creator id if it's new
+                if (media.HasIdentity == false)
+                {
+                    media.CreatorId = userId;
+                }
+
                 repository.AddOrUpdate(media);
                 repository.AddOrUpdateContentXml(media, m => _entitySerializer.Serialize(this, _dataTypeService, _userService, m));
                 // generate preview for blame history?

src/Umbraco.Core/Services/MemberTypeService.cs

@@ -2,12 +2,10 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
-using Umbraco.Core.Auditing;
 using Umbraco.Core.Events;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Persistence;
-using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
 
 namespace Umbraco.Core.Services
@@ -89,6 +87,8 @@ namespace Umbraco.Core.Services
                 using (var repository = RepositoryFactory.CreateMemberTypeRepository(uow))
                 {
                     memberType.CreatorId = userId;
+                    if (memberType.Description == string.Empty)
+                        memberType.Description = null;
                     repository.AddOrUpdate(memberType);
 
                     uow.Commit();
@@ -114,6 +114,8 @@ namespace Umbraco.Core.Services
                     foreach (var memberType in asArray)
                     {
                         memberType.CreatorId = userId;
+                        if (memberType.Description == string.Empty)
+                            memberType.Description = null;
                         repository.AddOrUpdate(memberType);
                     }
 

src/Umbraco.Core/Services/PackagingService.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Web;
 using System.Web.UI.WebControls;
@@ -9,7 +11,9 @@ using System.Xml.Linq;
 using System.Xml.XPath;
 using Newtonsoft.Json;
 using Umbraco.Core.Configuration;
+using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.Events;
+using Umbraco.Core.Exceptions;
 using Umbraco.Core.IO;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
@@ -58,7 +62,7 @@ namespace Umbraco.Core.Services
             IEntityService entityService,
             IUserService userService,
             RepositoryFactory repositoryFactory,
-            IDatabaseUnitOfWorkProvider uowProvider)
+            IDatabaseUnitOfWorkProvider uowProvider) 
         {
             _logger = logger;
             _contentService = contentService;
@@ -75,6 +79,61 @@ namespace Umbraco.Core.Services
             _importedContentTypes = new Dictionary<string, IContentType>();
         }
 
+        /// <summary>
+        /// This will fetch an Umbraco package file from the package repository and return the relative file path to the downloaded package file
+        /// </summary>
+        /// <param name="packageId"></param>
+        /// <param name="umbracoVersion"></param>
+        /// /// <param name="userId">The current user id performing the operation</param>
+        /// <returns></returns>
+        public string FetchPackageFile(Guid packageId, Version umbracoVersion, int userId)
+        {
+            var packageRepo = UmbracoConfig.For.UmbracoSettings().PackageRepositories.GetDefault();
+
+            using (var httpClient = new HttpClient())
+            using (var uow = _uowProvider.GetUnitOfWork())
+            {
+                //includeHidden = true because we don't care if it's hidden we want to get the file regardless
+                var url = string.Format("{0}/{1}?version={2}&includeHidden=true&asFile=true", packageRepo.RestApiUrl, packageId, umbracoVersion.ToString(3));
+                byte[] bytes;
+                try
+                {
+                    bytes = httpClient.GetByteArrayAsync(url).GetAwaiter().GetResult();
+                }
+                catch (HttpRequestException ex)
+                {
+                    throw new ConnectionException("An error occuring downloading the package from " + url, ex);
+                }
+
+                //successfull 
+                if (bytes.Length > 0)
+                {
+                    var packagePath = IOHelper.MapPath(SystemDirectories.Packages);
+
+                    // Check for package directory
+                    if (Directory.Exists(packagePath) == false)
+                        Directory.CreateDirectory(packagePath);
+
+                    var packageFilePath = Path.Combine(packagePath, packageId + ".umb");
+
+                    using (var fs1 = new FileStream(packageFilePath, FileMode.Create))
+                    {
+                        fs1.Write(bytes, 0, bytes.Length);
+                        return "packages\\" + packageId + ".umb";
+                    }
+                }
+
+                Audit(uow, AuditType.PackagerInstall, string.Format("Package {0} fetched from {1}", packageId, packageRepo.Id), userId, -1);
+                return null;
+            }
+        }
+
+        private void Audit(IDatabaseUnitOfWork uow, AuditType type, string message, int userId, int objectId)
+        {
+            var auditRepo = _repositoryFactory.CreateAuditRepository(uow);
+            auditRepo.AddOrUpdate(new AuditItem(objectId, message, type, userId));
+        }
+
         #region Content
 
         /// <summary>

src/Umbraco.Core/Sync/ConfigServerRegistrar.cs

@@ -4,6 +4,7 @@ using System.Web;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.IO;
+using Umbraco.Core.Logging;
 
 namespace Umbraco.Core.Sync
 {
@@ -40,8 +41,9 @@ namespace Umbraco.Core.Sync
                 .ToList();
 
             if (serversA.Length == 0)
-            {
+            {               
                 _serverRole = ServerRole.Unknown; // config error, actually
+                LogHelper.Debug<ConfigServerRegistrar>(string.Format("Server Role Unknown: DistributedCalls are enabled but no servers are listed"));
             }
             else
             {
@@ -50,7 +52,10 @@ namespace Umbraco.Core.Sync
                 var serverName = master.ServerName;
 
                 if (appId.IsNullOrWhiteSpace() && serverName.IsNullOrWhiteSpace())
+                {
                     _serverRole = ServerRole.Unknown; // config error, actually
+                    LogHelper.Debug<ConfigServerRegistrar>(string.Format("Server Role Unknown: Server Name or AppId missing from Server configuration in DistributedCalls settings"));
+                }
                 else
                     _serverRole = IsCurrentServer(appId, serverName)
                         ? ServerRole.Master

src/Umbraco.Core/Umbraco.Core.csproj

@@ -179,6 +179,8 @@
     <Compile Include="Configuration\BaseRest\ExtensionElement.cs" />
     <Compile Include="Configuration\BaseRest\ExtensionElementCollection.cs" />
     <Compile Include="Configuration\BaseRest\MethodElement.cs" />
+    <Compile Include="Configuration\ContentXmlStorage.cs" />
+    <Compile Include="Configuration\CoreDebug.cs" />
     <Compile Include="Configuration\Dashboard\AccessElement.cs" />
     <Compile Include="Configuration\Dashboard\AccessItem.cs" />
     <Compile Include="Configuration\Dashboard\AccessType.cs" />
@@ -272,6 +274,7 @@
     <Compile Include="Configuration\UmbracoSettings\RazorStaticMappingElement.cs" />
     <Compile Include="Configuration\UmbracoSettings\RepositoriesCollection.cs" />
     <Compile Include="Configuration\UmbracoSettings\RepositoriesElement.cs" />
+    <Compile Include="Configuration\UmbracoSettings\RepositoryConfigExtensions.cs" />
     <Compile Include="Configuration\UmbracoSettings\RepositoryElement.cs" />
     <Compile Include="Configuration\UmbracoSettings\RequestHandlerElement.cs" />
     <Compile Include="Configuration\UmbracoSettings\ScheduledTaskElement.cs" />
@@ -299,6 +302,7 @@
     <Compile Include="DateTimeExtensions.cs" />
     <Compile Include="DecimalExtensions.cs" />
     <Compile Include="DelegateExtensions.cs" />
+    <Compile Include="Diagnostics\MiniDump.cs" />
     <Compile Include="DictionaryExtensions.cs" />
     <Compile Include="Dictionary\CultureDictionaryFactoryResolver.cs" />
     <Compile Include="Dictionary\ICultureDictionaryFactory.cs" />
@@ -332,6 +336,7 @@
     <Compile Include="Events\RollbackEventArgs.cs" />
     <Compile Include="Events\SaveEventArgs.cs" />
     <Compile Include="Events\SendToPublishEventArgs.cs" />
+    <Compile Include="Exceptions\ConnectionException.cs" />
     <Compile Include="Exceptions\DataOperationException.cs" />
     <Compile Include="Exceptions\InvalidCompositionException.cs" />
     <Compile Include="Exceptions\UmbracoStartupFailedException.cs" />
@@ -464,6 +469,7 @@
     <Compile Include="Persistence\Migrations\Upgrades\TargetVersionSevenThreeOne\UpdateUserLanguagesToIsoCode.cs" />
     <Compile Include="Persistence\PocoDataDataReader.cs" />
     <Compile Include="Persistence\Querying\QueryExtensions.cs" />
+    <Compile Include="Persistence\Querying\SqlExpressionExtensions.cs" />
     <Compile Include="Persistence\RecordPersistenceType.cs" />
     <Compile Include="Persistence\Relators\AccessRulesRelator.cs" />
     <Compile Include="Persistence\Repositories\AuditRepository.cs" />
@@ -649,7 +655,6 @@
     <Compile Include="PropertyEditors\DefaultPropertyValueConverterAttribute.cs" />
     <Compile Include="Persistence\Migrations\Upgrades\TargetVersionSeven\UpdateRelatedLinksData.cs" />
     <Compile Include="PropertyEditors\IValueEditor.cs" />
-    <Compile Include="Persistence\Querying\SqlStringExtensions.cs" />
     <Compile Include="Persistence\Querying\StringPropertyMatchType.cs" />
     <Compile Include="Persistence\Querying\TextColumnType.cs" />
     <Compile Include="Persistence\Querying\ValuePropertyMatchType.cs" />

src/Umbraco.Core/XmlExtensions.cs

@@ -16,32 +16,6 @@ namespace Umbraco.Core
 	/// </summary>
 	internal static class XmlExtensions
 	{
-        /// <summary>
-        /// Saves the xml document async
-        /// </summary>
-        /// <param name="xdoc"></param>
-        /// <param name="filename"></param>
-        /// <returns></returns>
-	    public static async Task SaveAsync(this XmlDocument xdoc, string filename)
-	    {
-            if (xdoc.DocumentElement == null)
-                throw new XmlException("Cannot save xml document, there is no root element");
-
-            using (var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true))
-            using (var xmlWriter = XmlWriter.Create(fs, new XmlWriterSettings
-            {
-                Async = true,
-                Encoding = Encoding.UTF8,
-                Indent = true
-            }))
-            {
-                //NOTE: There are no nice methods to write it async, only flushing it async. We
-                // could implement this ourselves but it'd be a very manual process.
-                xdoc.WriteTo(xmlWriter);
-                await xmlWriter.FlushAsync().ConfigureAwait(false);
-            }
-	    }
-
         public static bool HasAttribute(this XmlAttributeCollection attributes, string attributeName)
         {
             return attributes.Cast<XmlAttribute>().Any(x => x.Name == attributeName);

src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs

@@ -16,6 +16,7 @@ using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
+using Moq;
 using Umbraco.Core;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
@@ -43,15 +44,18 @@ namespace Umbraco.Tests.Benchmarks
 
         public ModelToSqlExpressionHelperBenchmarks()
         {
-            _contentMapper = new ContentMapper(_syntaxProvider);
-            _contentMapper.BuildMap();
+            var contentMapper = new ContentMapper(_syntaxProvider);
+            contentMapper.BuildMap();
             _cachedExpression = new CachedExpression();
+            var mappingResolver = new Mock<MappingResolver>();
+            mappingResolver.Setup(resolver => resolver.ResolveMapperByType(It.IsAny<Type>())).Returns(contentMapper);
+            _mappingResolver = mappingResolver.Object;
         }
         
         private readonly ISqlSyntaxProvider _syntaxProvider = new SqlCeSyntaxProvider();
-        private readonly BaseMapper _contentMapper;
         private readonly CachedExpression _cachedExpression;
-        
+        private readonly MappingResolver _mappingResolver;
+
         [Benchmark(Baseline = true)]
         public void WithNonCached()
         {
@@ -62,7 +66,7 @@ namespace Umbraco.Tests.Benchmarks
                 Expression<Func<IContent, bool>> predicate = content =>
                 content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b);
 
-                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _contentMapper);
+                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _mappingResolver);
                 var result = modelToSqlExpressionHelper.Visit(predicate);
             }
             
@@ -80,7 +84,7 @@ namespace Umbraco.Tests.Benchmarks
                 Expression<Func<IContent, bool>> predicate = content =>
                 content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b);
                 
-                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _contentMapper);
+                var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>(_syntaxProvider, _mappingResolver);
 
                 //wrap it!
                 _cachedExpression.Wrap(predicate);

src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj

@@ -49,6 +49,9 @@
     <Reference Include="BenchmarkDotNet.Toolchains.Roslyn, Version=0.9.9.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
       <HintPath>..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll</HintPath>
     </Reference>
+    <Reference Include="Castle.Core, Version=4.0.0.0, Culture=neutral, PublicKeyToken=407dd0808d44fbdc, processorArchitecture=MSIL">
+      <HintPath>..\packages\Castle.Core.4.0.0\lib\net45\Castle.Core.dll</HintPath>
+    </Reference>
     <Reference Include="Microsoft.CodeAnalysis, Version=1.3.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.CodeAnalysis.Common.1.3.2\lib\net45\Microsoft.CodeAnalysis.dll</HintPath>
     </Reference>
@@ -58,6 +61,9 @@
     <Reference Include="Microsoft.Diagnostics.Tracing.TraceEvent, Version=1.0.41.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.41\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll</HintPath>
     </Reference>
+    <Reference Include="Moq, Version=4.7.0.0, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
+      <HintPath>..\packages\Moq.4.7.0\lib\net45\Moq.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Collections.Immutable, Version=1.1.37.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Collections.Immutable.1.1.37\lib\dotnet\System.Collections.Immutable.dll</HintPath>

src/Umbraco.Tests.Benchmarks/packages.config

@@ -4,11 +4,13 @@
   <package id="BenchmarkDotNet.Core" version="0.9.9" targetFramework="net45" />
   <package id="BenchmarkDotNet.Diagnostics.Windows" version="0.9.9" targetFramework="net45" />
   <package id="BenchmarkDotNet.Toolchains.Roslyn" version="0.9.9" targetFramework="net45" />
+  <package id="Castle.Core" version="4.0.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Analyzers" version="1.1.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Common" version="1.3.2" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.CSharp" version="1.3.2" targetFramework="net45" />
   <package id="Microsoft.Diagnostics.Tracing.TraceEvent" version="1.0.41" targetFramework="net45" />
   <package id="Microsoft.SqlServer.Compact" version="4.0.8854.1" targetFramework="net45" />
+  <package id="Moq" version="4.7.0" targetFramework="net45" />
   <package id="SqlServerCE" version="4.0.0.1" targetFramework="net45" />
   <package id="System.Collections" version="4.0.0" targetFramework="net45" />
   <package id="System.Collections.Immutable" version="1.1.37" targetFramework="net45" />

src/Umbraco.Tests/Issues/U9560.cs

@@ -0,0 +1,92 @@
+using System;
+using NUnit.Framework;
+using Umbraco.Core.Models;
+using Umbraco.Tests.TestHelpers;
+
+namespace Umbraco.Tests.Issues
+{
+    [TestFixture]
+    [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
+    public class U9560 : BaseDatabaseFactoryTest
+    {
+        [Test]
+        public void Test()
+        {
+            // create a content type and some properties
+            var contentType = new ContentType(-1);
+            contentType.Alias = "test";
+            contentType.Name = "test";
+            var propertyType = new PropertyType("test", DataTypeDatabaseType.Ntext, "prop") { Name = "Prop", Description = "", Mandatory = false, SortOrder = 1, DataTypeDefinitionId = -88 };
+            contentType.PropertyTypeCollection.Add(propertyType);
+            ServiceContext.ContentTypeService.Save(contentType);
+
+            var aliasName = string.Empty;
+
+            // read fields, same as what we do with PetaPoco Fetch<dynamic>
+            var db = DatabaseContext.Database;
+            db.OpenSharedConnection();
+            try
+            {
+                var conn = db.Connection;
+                var cmd = conn.CreateCommand();
+                cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id;
+                using (var reader = cmd.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        for (var i = 0; i < reader.FieldCount; i++)
+                            Console.WriteLine(reader.GetName(i));
+                        aliasName = reader.GetName(5);
+                    }
+                }
+            }
+            finally
+            {
+                db.CloseSharedConnection();
+            }
+
+            // note that although the query is for 'alias' the field is named 'Alias'
+            Assert.AreEqual("Alias", aliasName);
+
+            // try differently
+            db.OpenSharedConnection();
+            try
+            {
+                var conn = db.Connection;
+                var cmd = conn.CreateCommand();
+                cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias as alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id;
+                using (var reader = cmd.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        for (var i = 0; i < reader.FieldCount; i++)
+                            Console.WriteLine(reader.GetName(i));
+                        aliasName = reader.GetName(5);
+                    }
+                }
+            }
+            finally
+            {
+                db.CloseSharedConnection();
+            }
+
+            // and now it is OK
+            Assert.AreEqual("alias", aliasName);
+
+            // get the legacy content type
+            var legacyContentType = new umbraco.cms.businesslogic.ContentType(contentType.Id);
+            Assert.AreEqual("test", legacyContentType.Alias);
+
+            // get the legacy properties
+            var legacyProperties = legacyContentType.PropertyTypes;
+
+            // without the fix, due to some (swallowed) inner exception, we have no properties
+            //Assert.IsNull(legacyProperties);
+
+            // thanks to the fix, it works
+            Assert.IsNotNull(legacyProperties);
+            Assert.AreEqual(1, legacyProperties.Count);
+            Assert.AreEqual("prop", legacyProperties[0].Alias);
+        }
+    }
+}

src/Umbraco.Tests/Persistence/DatabaseContextTests.cs

@@ -34,7 +34,7 @@ namespace Umbraco.Tests.Persistence
 				{
 					DatabaseContext = _dbContext,
 					IsReady = true
-				};			
+				};
 		}
 
 		[TearDown]
@@ -44,6 +44,13 @@ namespace Umbraco.Tests.Persistence
 			ApplicationContext.Current = null;
 		}
 
+        [Test]
+        public void Database_Connection()
+        {
+            var db = _dbContext.Database;
+            Assert.IsNull(db.Connection);
+        }
+
         [Test]
         public void Can_Verify_Single_Database_Instance()
         {
@@ -99,7 +106,7 @@ namespace Umbraco.Tests.Persistence
 
             var appCtx = new ApplicationContext(
                 new DatabaseContext(Mock.Of<IDatabaseFactory>(), Mock.Of<ILogger>(), Mock.Of<ISqlSyntaxProvider>(), "test"),
-                new ServiceContext(migrationEntryService: Mock.Of<IMigrationEntryService>()), 
+                new ServiceContext(migrationEntryService: Mock.Of<IMigrationEntryService>()),
                 CacheHelper.CreateDisabledCacheHelper(),
                 new ProfilingLogger(Mock.Of<ILogger>(), Mock.Of<IProfiler>()));
 

src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Diagnostics;
+using System.Linq;
 using System.Linq.Expressions;
 using Moq;
 using NUnit.Framework;
@@ -16,19 +17,52 @@ namespace Umbraco.Tests.Persistence.Querying
     [TestFixture]
     public class ExpressionTests : BaseUsingSqlCeSyntax
     {
-        //    [Test]
-        //    public void Can_Query_With_Content_Type_Alias()
-        //    {
-        //        //Arrange
-        //        Expression<Func<IMedia, bool>> predicate = content => content.ContentType.Alias == "Test";
-        //        var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>();
-        //        var result = modelToSqlExpressionHelper.Visit(predicate);
+        [Test]
+        public void Equals_Claus_With_Two_Entity_Values()
+        {
+            var dataType = new DataTypeDefinition(-1, "Test")
+            {
+                Id = 12345
+            };
+            Expression<Func<PropertyType, bool>> predicate = p => p.DataTypeDefinitionId == dataType.Id;
+            var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<PropertyType>();
+            var result = modelToSqlExpressionHelper.Visit(predicate);
 
-        //        Debug.Print("Model to Sql ExpressionHelper: \n" + result);
+            Debug.Print("Model to Sql ExpressionHelper: \n" + result);
 
-        //        Assert.AreEqual("[cmsContentType].[alias] = @0", result);
-        //        Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]);
-        //    }
+            Assert.AreEqual("([cmsPropertyType].[dataTypeId] = @0)", result);
+            Assert.AreEqual(12345, modelToSqlExpressionHelper.GetSqlParameters()[0]);
+        }
+
+        [Test]
+        public void Can_Query_With_Content_Type_Alias()
+        {
+            //Arrange
+            Expression<Func<IMedia, bool>> predicate = content => content.ContentType.Alias == "Test";
+            var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>();
+            var result = modelToSqlExpressionHelper.Visit(predicate);
+
+            Debug.Print("Model to Sql ExpressionHelper: \n" + result);
+
+            Assert.AreEqual("([cmsContentType].[alias] = @0)", result);
+            Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]);
+        }
+
+        [Test]
+        public void Can_Query_With_Content_Type_Aliases()
+        {
+            //Arrange
+            var aliases = new[] {"Test1", "Test2"};
+            Expression<Func<IMedia, bool>> predicate = content => aliases.Contains(content.ContentType.Alias);
+            var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor<IContent>();
+            var result = modelToSqlExpressionHelper.Visit(predicate);
+
+            Debug.Print("Model to Sql ExpressionHelper: \n" + result);
+
+            Assert.AreEqual("[cmsContentType].[alias] IN (@1,@2)", result);
+            Assert.AreEqual("Test1", modelToSqlExpressionHelper.GetSqlParameters()[1]);
+            Assert.AreEqual("Test2", modelToSqlExpressionHelper.GetSqlParameters()[2]);
+        }
 
         [Test]
         public void CachedExpression_Can_Verify_Path_StartsWith_Predicate_In_Same_Result()

src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Xml.Linq;
@@ -21,7 +22,6 @@ using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.TestHelpers.Entities;
 using umbraco.editorControls.tinyMCE3;
 using umbraco.interfaces;
-using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 
 namespace Umbraco.Tests.Persistence.Repositories
@@ -36,11 +36,15 @@ namespace Umbraco.Tests.Persistence.Repositories
             base.Initialize();
 
             CreateTestData();
+
+            VersionableRepositoryBase<int, IContent>.ThrowOnWarning = true;
         }
 
         [TearDown]
         public override void TearDown()
         {
+            VersionableRepositoryBase<int, IContent>.ThrowOnWarning = false;
+
             base.TearDown();
         }
 
@@ -67,6 +71,131 @@ namespace Umbraco.Tests.Persistence.Repositories
             return repository;
         }
 
+        [Test]
+        public void Get_Always_Returns_Latest_Version()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;
+
+            var versions = new List<Guid>();
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                //save version
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+
+                //publish version
+                content1.ChangePublishedState(PublishedState.Published);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+
+                //change something and make a pending version
+                content1.Name = "new name";
+                content1.ChangePublishedState(PublishedState.Saved);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+            }
+
+            // Assert
+            Assert.AreEqual(3, versions.Distinct().Count());
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var content = repository.GetByQuery(new Query<IContent>().Where(c => c.Id == content1.Id)).ToArray()[0];
+                Assert.AreEqual(versions[2], content.Version);
+
+                content = repository.Get(content1.Id);
+                Assert.AreEqual(versions[2], content.Version);
+
+                foreach (var version in versions)
+                {
+                    content = repository.GetByVersion(version);
+                    Assert.IsNotNull(content);
+                    Assert.AreEqual(version, content.Version);
+                }
+            }
+        }
+
+        [Test]
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;
+
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+                
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);                
+                unitOfWork.Commit();                
+            }
+
+            var versionDtos = new List<ContentVersionDto>();
+
+            //Now manually corrupt the data
+            var versions = new[] { Guid.NewGuid(), Guid.NewGuid() };
+            for (var index = 0; index < versions.Length; index++)
+            {
+                var version = versions[index];
+                var versionDate = DateTime.Now.AddMinutes(index);
+                var versionDto = new ContentVersionDto
+                {
+                    NodeId = content1.Id,
+                    VersionDate = versionDate,
+                    VersionId = version
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                versionDtos.Add(versionDto);
+                this.DatabaseContext.Database.Insert(new DocumentDto
+                {
+                    Newest = true,
+                    NodeId = content1.Id,
+                    Published = true,
+                    Text = content1.Name,
+                    VersionId = version,
+                    WriterUserId = 0,
+                    UpdateDate = versionDate,
+                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?) content1.Template.Id
+                });
+            }
+
+            // Assert
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var content = repository.GetByQuery(new Query<IContent>().Where(c => c.Id == content1.Id)).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(content[0].Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId);
+                Assert.AreEqual(content[0].UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
+
+                var contentItem = repository.GetByVersion(content1.Version);
+                Assert.IsNotNull(contentItem);
+
+                contentItem = repository.Get(content1.Id);
+                Assert.IsNotNull(contentItem);
+                Assert.AreEqual(contentItem.UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
+                Assert.AreEqual(contentItem.Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId);
+
+                var allVersions = repository.GetAllVersions(content[0].Id);
+                var allKnownVersions = versionDtos.Select(x => x.VersionId).Union(new[]{ content1.Version }).ToArray();
+                Assert.IsTrue(allKnownVersions.ContainsAll(allVersions.Select(x => x.Version)));
+                Assert.IsTrue(allVersions.Select(x => x.Version).ContainsAll(allKnownVersions));
+            }
+        }
+
         /// <summary>
         /// This tests the regression issue of U4-9438
         /// </summary>
@@ -226,6 +355,62 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                //delete all xml
+                unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
+
+                var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1");
+                contentTypeRepository.AddOrUpdate(contentType1);
+                var allCreated = new List<IContent>();
+
+                for (var i = 0; i < 100; i++)
+                {
+                    //These will be non-published so shouldn't show up
+                    var c1 = MockedContent.CreateSimpleContent(contentType1);
+                    repository.AddOrUpdate(c1);
+                    allCreated.Add(c1);
+                }
+                for (var i = 0; i < 100; i++)
+                {
+                    var c1 = MockedContent.CreateSimpleContent(contentType1);
+                    c1.ChangePublishedState(PublishedState.Published);
+                    repository.AddOrUpdate(c1);
+                    allCreated.Add(c1);
+                }
+                unitOfWork.Commit();
+
+                //now create some versions of this content - this shouldn't affect the xml structures saved
+                for (int i = 0; i < allCreated.Count; i++)
+                {
+                    allCreated[i].Name = "blah" + i;
+                    repository.AddOrUpdate(allCreated[i]);
+                }
+                unitOfWork.Commit();
+
+                //Add some extra orphaned rows that shouldn't be there
+                var notPublished = MockedContent.CreateSimpleContent(contentType1);
+                repository.AddOrUpdate(notPublished);
+                unitOfWork.Commit();
+                //Force add it
+                unitOfWork.Database.Insert(new ContentXmlDto
+                {
+                    NodeId = notPublished.Id,
+                    Xml = "<test></test>"
+                });
+
+                repository.RebuildXmlStructures(media => new XElement("test"), 10);
+
+                Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
+            }
+        }
+
         [Test]
         public void Rebuild_All_Xml_Structures_For_Content_Type()
         {

src/Umbraco.Tests/Persistence/Repositories/DataTypeDefinitionRepositoryTest.cs

@@ -46,23 +46,6 @@ namespace Umbraco.Tests.Persistence.Repositories
             return new EntityContainerRepository(unitOfWork, CacheHelper.CreateDisabledCacheHelper(), Mock.Of<ILogger>(), SqlSyntax, Constants.ObjectTypes.DataTypeContainerGuid);
         }
 
-        [TestCase("UmbracoPreVal87-21,3,48", 3, true)]
-        [TestCase("UmbracoPreVal87-21,33,48", 3, false)]
-        [TestCase("UmbracoPreVal87-21,33,48", 33, true)]
-        [TestCase("UmbracoPreVal87-21,3,48", 33, false)]
-        [TestCase("UmbracoPreVal87-21,3,48", 21, true)]
-        [TestCase("UmbracoPreVal87-21,3,48", 48, true)]
-        [TestCase("UmbracoPreVal87-22,33,48", 2, false)]
-        [TestCase("UmbracoPreVal87-22,33,48", 22, true)]
-        [TestCase("UmbracoPreVal87-22,33,44", 4, false)]
-        [TestCase("UmbracoPreVal87-22,33,44", 44, true)]
-        [TestCase("UmbracoPreVal87-22,333,44", 33, false)]
-        [TestCase("UmbracoPreVal87-22,333,44", 333, true)]
-        public void Pre_Value_Cache_Key_Tests(string cacheKey, int preValueId, bool outcome)
-        {
-            Assert.AreEqual(outcome, Regex.IsMatch(cacheKey, DataTypeDefinitionRepository.GetCacheKeyRegex(preValueId)));
-        }
-
         [Test]
         public void Can_Move()
         {
@@ -537,7 +520,7 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
 
             var cached = cache.IsolatedRuntimeCache.GetCache<IDataTypeDefinition>().Result
-                .GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + dtd.Id + "-");
+                .GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + "_" + dtd.Id);
 
             Assert.IsNotNull(cached);
             Assert.AreEqual(1, cached.Count());
@@ -581,7 +564,7 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
 
             var cached = cache.IsolatedRuntimeCache.GetCache<IDataTypeDefinition>().Result
-                .GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + dtd.Id + "-");
+                .GetCacheItemsByKeySearch<PreValueCollection>(CacheKeys.DataTypePreValuesCacheKey + "_" + dtd.Id);
 
             Assert.IsNotNull(cached);
             Assert.AreEqual(1, cached.Count());

src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs

@@ -0,0 +1,183 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Core;
+using Umbraco.Core.Configuration.UmbracoSettings;
+using Umbraco.Core.IO;
+using Umbraco.Core.Models;
+using Umbraco.Core.Models.EntityBase;
+using Umbraco.Core.Models.Rdbms;
+using Umbraco.Core.Persistence.Querying;
+using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Persistence.UnitOfWork;
+using Umbraco.Tests.TestHelpers;
+using Umbraco.Tests.TestHelpers.Entities;
+
+namespace Umbraco.Tests.Persistence.Repositories
+{
+    [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
+    [TestFixture]
+    public class EntityRepositoryTest : BaseDatabaseFactoryTest
+    {
+        [SetUp]
+        public override void Initialize()
+        {
+            base.Initialize();            
+        }
+
+        [TearDown]
+        public override void TearDown()
+        {
+            base.TearDown();
+        }
+        
+        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository)
+        {
+            TemplateRepository tr;
+            return CreateContentRepository(unitOfWork, out contentTypeRepository, out tr);
+        }
+
+        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository, out TemplateRepository templateRepository)
+        {
+            templateRepository = new TemplateRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, Mock.Of<IFileSystem>(), Mock.Of<IFileSystem>(), Mock.Of<ITemplatesSection>());
+            var tagRepository = new TagRepository(unitOfWork, CacheHelper, Logger, SqlSyntax);
+            contentTypeRepository = new ContentTypeRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, templateRepository);
+            var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of<IContentSection>());
+            return repository;
+        }
+
+        [Test]
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Full_Model()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;            
+
+            using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+            }
+
+            var versionDtos = new List<ContentVersionDto>();
+
+            //Now manually corrupt the data
+            var versions = new[] {Guid.NewGuid(), Guid.NewGuid()};
+            for (var index = 0; index < versions.Length; index++)
+            {
+                var version = versions[index];
+                var versionDate = DateTime.Now.AddMinutes(index);
+                var versionDto = new ContentVersionDto
+                {
+                    NodeId = content1.Id,
+                    VersionDate = versionDate,
+                    VersionId = version
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                versionDtos.Add(versionDto);
+                this.DatabaseContext.Database.Insert(new DocumentDto
+                {
+                    Newest = true,
+                    NodeId = content1.Id,
+                    Published = true,
+                    Text = content1.Name,
+                    VersionId = version,
+                    WriterUserId = 0,
+                    UpdateDate = versionDate,
+                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id
+                });
+            }
+
+            // Assert
+            using (var repository = new EntityRepository(unitOfWork))
+            {
+                var content = repository.GetByQuery(new Query<IUmbracoEntity>().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Key).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Id).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                var contentItem = repository.Get(content[0].Id, Constants.ObjectTypes.DocumentGuid);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]);
+
+                contentItem = repository.GetByKey(content[0].Key, Constants.ObjectTypes.DocumentGuid);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]);
+            }
+        }
+
+        /// <summary>
+        /// The Slim model will test the EntityRepository when it doesn't know the object type so it will 
+        /// make the simplest (slim) query
+        /// </summary>
+        [Test]
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Slim_Model()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;
+
+            using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+            }
+
+            //Now manually corrupt the data
+            var versions = new[] { Guid.NewGuid(), Guid.NewGuid() };
+            for (var index = 0; index < versions.Length; index++)
+            {
+                var version = versions[index];
+                var versionDate = DateTime.Now.AddMinutes(index);
+                var versionDto = new ContentVersionDto
+                {
+                    NodeId = content1.Id,
+                    VersionDate = versionDate,
+                    VersionId = version
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                this.DatabaseContext.Database.Insert(new DocumentDto
+                {
+                    Newest = true,
+                    NodeId = content1.Id,
+                    Published = true,
+                    Text = content1.Name,
+                    VersionId = version,
+                    WriterUserId = 0,
+                    UpdateDate = versionDate,
+                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id
+                });
+            }
+
+            // Assert
+            using (var repository = new EntityRepository(unitOfWork))
+            {
+                var content = repository.GetByQuery(new Query<IUmbracoEntity>().Where(c => c.Id == content1.Id)).ToArray();
+                Assert.AreEqual(1, content.Length);
+                var contentItem = repository.Get(content[0].Id);
+                Assert.IsNotNull(contentItem);
+                contentItem = repository.GetByKey(content[0].Key);
+                Assert.IsNotNull(contentItem);
+            }
+        }
+    }
+}

src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Xml.Linq;
 using Moq;
 using NUnit.Framework;
+using Umbraco.Core;
 using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
@@ -70,6 +71,44 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            MediaTypeRepository mediaTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
+            {
+                //delete all xml
+                unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
+
+                var mediaType = mediaTypeRepository.Get(1032);
+
+                for (var i = 0; i < 100; i++)
+                {
+                    var image = MockedMedia.CreateMediaImage(mediaType, -1);
+                    repository.AddOrUpdate(image);
+                }
+                unitOfWork.Commit();
+
+                //Add some extra orphaned rows that shouldn't be there
+                var trashed = MockedMedia.CreateMediaImage(mediaType, -1);
+                trashed.ChangeTrashedState(true, Constants.System.RecycleBinMedia);
+                repository.AddOrUpdate(trashed);
+                unitOfWork.Commit();
+                //Force add it
+                unitOfWork.Database.Insert(new ContentXmlDto
+                {
+                    NodeId = trashed.Id,
+                    Xml = "<test></test>"
+                });
+
+                repository.RebuildXmlStructures(media => new XElement("test"), 10);
+
+                Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
+            }
+        }
+
         [Test]
         public void Rebuild_Some_Xml_Structures()
         {
@@ -345,6 +384,61 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Id_Filter()
+        {
+            // Arrange            
+            var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031);
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            MediaTypeRepository mediaTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
+            {
+                // Act
+                for (int i = 0; i < 10; i++)
+                {
+                    var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1);
+                    repository.AddOrUpdate(folder);
+                }
+                unitOfWork.Commit();
+
+                var types = new[] { 1031 };
+                var query = Query<IMedia>.Builder.Where(x => types.Contains(x.ContentTypeId));
+                var result = repository.GetByQuery(query);
+
+                // Assert
+                Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11)); 
+            }
+        }
+
+        [Ignore("We could allow this to work but it requires an extra join on the query used which currently we don't absolutely need so leaving this out for now")]
+        [Test]
+        public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Alias_Filter()
+        {
+            // Arrange            
+            var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031);
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            MediaTypeRepository mediaTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
+            {
+                // Act
+                for (int i = 0; i < 10; i++)
+                {
+                    var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1);
+                    repository.AddOrUpdate(folder);
+                }
+                unitOfWork.Commit();
+
+                var types = new[] { "Folder" };
+                var query = Query<IMedia>.Builder.Where(x => types.Contains(x.ContentType.Alias));
+                var result = repository.GetByQuery(query);
+
+                // Assert
+                Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11));
+            }
+        }
+
         [Test]
         public void Can_Perform_GetPagedResultsByQuery_ForFirstPage_On_MediaRepository()
         {
@@ -470,7 +564,7 @@ namespace Umbraco.Tests.Persistence.Repositories
                 Assert.That(result.First().Name, Is.EqualTo("Test File"));
             }
         }
-
+        
         [Test]
         public void Can_Perform_GetPagedResultsByQuery_WithFilterMatchingAll_On_MediaRepository()
         {

src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs

@@ -154,36 +154,61 @@ namespace Umbraco.Tests.Persistence.Repositories
         [Test]
         public void Get_All()
         {
-            var content = CreateTestData(3).ToArray();
+            var content = CreateTestData(30).ToArray();
 
             var provider = new PetaPocoUnitOfWorkProvider(Logger);
             var unitOfWork = provider.GetUnitOfWork();
             using (var repo = new PublicAccessRepository(unitOfWork, CacheHelper, Logger, SqlSyntax))
-            {                
-                var entry1 = new PublicAccessEntry(content[0], content[1], content[2], new[]
+            {
+                var allEntries = new List<PublicAccessEntry>();
+                for (int i = 0; i < 10; i++)
                 {
-                    new PublicAccessRule
+                    var rules = new List<PublicAccessRule>();
+                    for (int j = 0; j < 50; j++)
                     {
-                        RuleValue = "test",
-                        RuleType = "RoleName"
-                    },
-                });
-                repo.AddOrUpdate(entry1);
+                        rules.Add(new PublicAccessRule
+                        {
+                            RuleValue = "test" + j,
+                            RuleType = "RoleName" + j
+                        });
+                    }                    
+                    var entry1 = new PublicAccessEntry(content[i], content[i+1], content[i+2], rules);
+                    repo.AddOrUpdate(entry1);                    
+                    unitOfWork.Commit();
+                    allEntries.Add(entry1);
+                }
 
-                var entry2 = new PublicAccessEntry(content[1], content[0], content[2], new[]
+                //now remove a few rules from a few of the items and then add some more, this will put things 'out of order' which 
+                //we need to verify our sort order is working for the relator
+                for (int i = 0; i < allEntries.Count; i++)
                 {
-                    new PublicAccessRule
+                    //all the even ones
+                    if (i % 2 == 0)
                     {
-                        RuleValue = "test",
-                        RuleType = "RoleName"
-                    },
-                });
-                repo.AddOrUpdate(entry2);
-                
-                unitOfWork.Commit();
+                        var rules = allEntries[i].Rules.ToArray();
+                        for (int j = 0; j < rules.Length; j++)
+                        {
+                            //all the even ones
+                            if (j % 2 == 0)
+                            {
+                                allEntries[i].RemoveRule(rules[j]);
+                            }
+                        }
+                        allEntries[i].AddRule("newrule" + i, "newrule" + i);
+                        repo.AddOrUpdate(allEntries[i]);
+                        unitOfWork.Commit();
+                    }
+                }
 
                 var found = repo.GetAll().ToArray();
-                Assert.AreEqual(2, found.Count());
+                Assert.AreEqual(10, found.Length);
+
+                foreach (var publicAccessEntry in found)
+                {
+                    var matched = allEntries.First(x => x.Key == publicAccessEntry.Key);
+
+                    Assert.AreEqual(matched.Rules.Count(), publicAccessEntry.Rules.Count());
+                }
             }
         }
 

src/Umbraco.Tests/Routing/SiteDomainHelperTests.cs

@@ -25,14 +25,14 @@ namespace Umbraco.Tests.Routing
         {
             SiteDomainHelper.AddSite("site1", "domain1.com", "domain1.net", "domain1.org");
             SiteDomainHelper.AddSite("site2", "domain2.com", "domain2.net", "domain2.org");
-            
+
             var sites = SiteDomainHelper.Sites;
 
             Assert.AreEqual(2, sites.Count);
 
             Assert.Contains("site1", sites.Keys);
             Assert.Contains("site2", sites.Keys);
-            
+
             var domains = sites["site1"];
             Assert.AreEqual(3, domains.Count());
             Assert.Contains("domain1.com", domains);
@@ -84,7 +84,7 @@ namespace Umbraco.Tests.Routing
 
             Assert.Contains("site2", sites.Keys);
         }
-    
+
         [Test]
         public void AddSiteAgain()
         {
@@ -175,15 +175,15 @@ namespace Umbraco.Tests.Routing
 
             // map methods are not static because we can override them
             var helper = new SiteDomainHelper();
-            
+
             // current is a site1 uri, domains contain current
             // so we'll get current
             //
             var current = new Uri("http://domain1.com/foo/bar");
             var output = helper.MapDomain(current, new[]
                 {
-                    new DomainAndUri(new UmbracoDomain("domain1.com"), Uri.UriSchemeHttp), 
-                    new DomainAndUri(new UmbracoDomain("domain2.com"), Uri.UriSchemeHttp), 
+                    new DomainAndUri(new UmbracoDomain("domain1.com"), Uri.UriSchemeHttp),
+                    new DomainAndUri(new UmbracoDomain("domain2.com"), Uri.UriSchemeHttp),
                 }).Uri.ToString();
             Assert.AreEqual("http://domain1.com/", output);
 
@@ -193,7 +193,7 @@ namespace Umbraco.Tests.Routing
             current = new Uri("http://domain1.com/foo/bar");
             output = helper.MapDomain(current, new[]
                 {
-                    new DomainAndUri(new UmbracoDomain("domain1.net"), Uri.UriSchemeHttp), 
+                    new DomainAndUri(new UmbracoDomain("domain1.net"), Uri.UriSchemeHttp),
                     new DomainAndUri(new UmbracoDomain("domain2.net"), Uri.UriSchemeHttp)
                 }).Uri.ToString();
             Assert.AreEqual("http://domain1.net/", output);
@@ -205,12 +205,73 @@ namespace Umbraco.Tests.Routing
             current = new Uri("http://domain1.com/foo/bar");
             output = helper.MapDomain(current, new[]
                 {
-                    new DomainAndUri(new UmbracoDomain("domain2.net"), Uri.UriSchemeHttp), 
+                    new DomainAndUri(new UmbracoDomain("domain2.net"), Uri.UriSchemeHttp),
                     new DomainAndUri(new UmbracoDomain("domain1.net"), Uri.UriSchemeHttp)
                 }).Uri.ToString();
             Assert.AreEqual("http://domain1.net/", output);
         }
 
+        private DomainAndUri[] DomainAndUris(Uri current, IDomain[] domains)
+        {
+            var scheme = current == null ? Uri.UriSchemeHttp : current.Scheme;
+            return domains
+                .Where(d => d.IsWildcard == false)
+                .Select(d => new DomainAndUri(d, scheme))
+                .OrderByDescending(d => d.Uri.ToString())
+                .ToArray();
+        }
+
+        [Test]
+        public void MapDomainWithScheme()
+        {
+            SiteDomainHelper.AddSite("site1", "domain1.com", "domain1.net", "domain1.org");
+            SiteDomainHelper.AddSite("site2", "domain2.com", "domain2.net", "domain2.org");
+            SiteDomainHelper.AddSite("site3", "domain3.com", "domain3.net", "domain3.org");
+            SiteDomainHelper.AddSite("site4", "https://domain4.com", "https://domain4.net", "https://domain4.org");
+
+            // map methods are not static because we can override them
+            var helper = new SiteDomainHelper();
+
+            // this works, but it's purely by chance / arbitrary
+            // don't use the www in tests here!
+            var current = new Uri("https://www.domain1.com/foo/bar");
+            var domainAndUris = DomainAndUris(current, new []
+            {
+                new UmbracoDomain("domain2.com"),
+                new UmbracoDomain("domain1.com"),
+            });
+            var output = helper.MapDomain(current, domainAndUris).Uri.ToString();
+            Assert.AreEqual("https://domain1.com/", output);
+
+            // will pick it all right
+            current = new Uri("https://domain1.com/foo/bar");
+            domainAndUris = DomainAndUris(current, new[]
+            {
+                new UmbracoDomain("https://domain1.com"),
+                new UmbracoDomain("https://domain2.com")
+            });
+            output = helper.MapDomain(current, domainAndUris).Uri.ToString();
+            Assert.AreEqual("https://domain1.com/", output);
+
+            current = new Uri("https://domain1.com/foo/bar");
+            domainAndUris = DomainAndUris(current, new[]
+            {
+                new UmbracoDomain("https://domain1.com"),
+                new UmbracoDomain("https://domain4.com")
+            });
+            output = helper.MapDomain(current, domainAndUris).Uri.ToString();
+            Assert.AreEqual("https://domain1.com/", output);
+
+            current = new Uri("https://domain4.com/foo/bar");
+            domainAndUris = DomainAndUris(current, new[]
+            {
+                new UmbracoDomain("https://domain1.com"),
+                new UmbracoDomain("https://domain4.com")
+            });
+            output = helper.MapDomain(current, domainAndUris).Uri.ToString();
+            Assert.AreEqual("https://domain4.com/", output);
+        }
+
         [Test]
         public void MapDomains()
         {

src/Umbraco.Tests/Services/ContentServiceTests.cs

@@ -36,11 +36,15 @@ namespace Umbraco.Tests.Services
         public override void Initialize()
         {
             base.Initialize();
+
+            VersionableRepositoryBase<int, IContent>.ThrowOnWarning = true;
         }
 
         [TearDown]
         public override void TearDown()
         {
+            VersionableRepositoryBase<int, IContent>.ThrowOnWarning = false;
+
             base.TearDown();
         }
 
@@ -762,16 +766,26 @@ namespace Umbraco.Tests.Services
             var parent = ServiceContext.ContentService.GetById(NodeDto.NodeIdSeed + 1);
             ServiceContext.ContentService.Publish(parent);//Publishing root, so Text Page 2 can be updated.
             var subpage2 = contentService.GetById(NodeDto.NodeIdSeed + 3);
+
             subpage2.Name = "Text Page 2 Updated";
             subpage2.SetValue("author", "Jane Doe");
             contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version.
 
+            subpage2.Name = "Text Page 2 Updated again";
+            subpage2.SetValue("author", "Bob Hope");
+            contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version.
+
             // Act
             var versions = contentService.GetVersions(NodeDto.NodeIdSeed + 3).ToList();
 
             // Assert
             Assert.That(versions.Any(), Is.True);
             Assert.That(versions.Count(), Is.GreaterThanOrEqualTo(2));
+
+            //ensure each version contains the correct property values
+            Assert.AreEqual("John Doe", versions[2].GetValue("author"));
+            Assert.AreEqual("Jane Doe", versions[1].GetValue("author"));
+            Assert.AreEqual("Bob Hope", versions[0].GetValue("author"));
         }
 
         [Test]

src/Umbraco.Tests/Services/ContentTypeServiceTests.cs

@@ -1,4 +1,3 @@
-using System.Runtime.Remoting;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
@@ -8,7 +7,6 @@ using Umbraco.Core.Exceptions;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Services;
-using Umbraco.Tests.CodeFirst.TestModels.Composition;
 using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.TestHelpers.Entities;
 
@@ -1420,6 +1418,38 @@ namespace Umbraco.Tests.Services
             Assert.That(descriptionPropertyTypeReloaded.PropertyGroupId.IsValueCreated, Is.False);
         }
 
+        [Test]
+        public void Empty_Description_Is_Always_Null_After_Saving_Content_Type()
+        {
+            var service = ServiceContext.ContentTypeService;
+            var contentType = MockedContentTypes.CreateBasicContentType();
+            contentType.Description = null;
+            service.Save(contentType);
+
+            var contentType2 = MockedContentTypes.CreateBasicContentType("basePage2", "Base Page 2");
+            contentType2.Description = string.Empty;
+            service.Save(contentType2);
+
+            Assert.IsNull(contentType.Description);
+            Assert.IsNull(contentType2.Description);
+        }
+
+        [Test]
+        public void Empty_Description_Is_Always_Null_After_Saving_Media_Type()
+        {
+            var service = ServiceContext.ContentTypeService;
+            var mediaType = MockedContentTypes.CreateSimpleMediaType("mediaType", "Media Type");
+            mediaType.Description = null;
+            service.Save(mediaType);
+
+            var mediaType2 = MockedContentTypes.CreateSimpleMediaType("mediaType2", "Media Type 2");
+            mediaType2.Description = string.Empty;
+            service.Save(mediaType2);
+
+            Assert.IsNull(mediaType.Description);
+            Assert.IsNull(mediaType2.Description);
+        }
+
         private ContentType CreateComponent()
         {
             var component = new ContentType(-1)

src/Umbraco.Tests/Services/MediaServiceTests.cs

@@ -7,6 +7,7 @@ using Umbraco.Core;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Repositories;
 using Umbraco.Core.Persistence.UnitOfWork;
 using Umbraco.Tests.TestHelpers;
@@ -30,6 +31,33 @@ namespace Umbraco.Tests.Services
             base.TearDown();
         }
 
+        [Test]
+        public void Get_Paged_Children_With_Media_Type_Filter()
+        {
+            var mediaService = ServiceContext.MediaService;
+            var mediaType1 = MockedContentTypes.CreateImageMediaType("Image2");
+            ServiceContext.ContentTypeService.Save(mediaType1);
+            var mediaType2 = MockedContentTypes.CreateImageMediaType("Image3");
+            ServiceContext.ContentTypeService.Save(mediaType2);
+
+            for (int i = 0; i < 10; i++)
+            {
+                var m1 = MockedMedia.CreateMediaImage(mediaType1, -1);
+                mediaService.Save(m1);
+                var m2 = MockedMedia.CreateMediaImage(mediaType2, -1);
+                mediaService.Save(m2);
+            }
+
+            long total;
+            var result = ServiceContext.MediaService.GetPagedChildren(-1, 0, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] {mediaType1.Id, mediaType2.Id});
+            Assert.AreEqual(11, result.Count());
+            Assert.AreEqual(20, total);
+
+            result = ServiceContext.MediaService.GetPagedChildren(-1, 1, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] { mediaType1.Id, mediaType2.Id });
+            Assert.AreEqual(9, result.Count());
+            Assert.AreEqual(20, total);
+        }
+
         [Test]
         public void Can_Move_Media()
         {

src/Umbraco.Tests/Services/MemberTypeServiceTests.cs

@@ -1,8 +1,6 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using NUnit.Framework;
-using umbraco.cms.presentation.create.controls;
 using Umbraco.Core;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Rdbms;
@@ -183,6 +181,22 @@ namespace Umbraco.Tests.Services
             Assert.Throws<ArgumentException>(() => ServiceContext.MemberTypeService.Save(memberType));
         }
 
+        [Test]
+        public void Empty_Description_Is_Always_Null_After_Saving_Member_Type()
+        {
+            var service = ServiceContext.MemberTypeService;
+            var memberType = MockedContentTypes.CreateSimpleMemberType();
+            memberType.Description = null;
+            service.Save(memberType);
+
+            var memberType2 = MockedContentTypes.CreateSimpleMemberType("memberType2", "Member Type 2");
+            memberType2.Description = string.Empty;
+            service.Save(memberType2);
+
+            Assert.IsNull(memberType.Description);
+            Assert.IsNull(memberType2.Description);
+        }
+
         //[Test]
         //public void Can_Save_MemberType_Structure_And_Create_A_Member_Based_On_It()
         //{

src/Umbraco.Tests/TestHelpers/Entities/MockedContentTypes.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Linq;
 using Umbraco.Core;
 using Umbraco.Core.Models;
 

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -57,8 +57,8 @@
     <Reference Include="AutoMapper.Net4, Version=3.3.1.0, Culture=neutral, PublicKeyToken=be96cd2c38ef1005, processorArchitecture=MSIL">
       <HintPath>..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.81.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.81\lib\net45\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.82.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.82\lib\net45\Examine.dll</HintPath>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
@@ -155,11 +155,13 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Dependencies\NuGet.cs" />
+    <Compile Include="Issues\U9560.cs" />
     <Compile Include="Migrations\MigrationIssuesTests.cs" />
     <Compile Include="Persistence\BulkDataReaderTests.cs" />
     <Compile Include="Persistence\Migrations\MigrationStartupHandlerTests.cs" />
     <Compile Include="Persistence\PetaPocoCachesTest.cs" />
     <Compile Include="Persistence\PetaPocoExpressionsTests.cs" />
+    <Compile Include="Persistence\Repositories\EntityRepositoryTest.cs" />
     <Compile Include="Persistence\Repositories\RedirectUrlRepositoryTests.cs" />
     <Compile Include="Routing\NiceUrlRoutesTests.cs" />
     <Compile Include="TestHelpers\Entities\MockedPropertyTypes.cs" />

src/Umbraco.Tests/packages.config

@@ -2,7 +2,7 @@
 <packages>
   <package id="AspNetWebApi.SelfHost" version="4.0.20710.0" targetFramework="net45" />
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
-  <package id="Examine" version="0.1.81" targetFramework="net45" />
+  <package id="Examine" version="0.1.82" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.Mvc" version="5.2.3" targetFramework="net45" />

src/Umbraco.Web.UI.Client/src/common/directives/components/events/events.directive.js

@@ -163,13 +163,13 @@ angular.module('umbraco.directives')
                 }
 
                 // ignore clicks on dialog from old dialog service
-                var oldDialog = $(el).parents("#old-dialog-service");
+                var oldDialog = $(event.target).parents("#old-dialog-service");
                 if (oldDialog.length === 1) {
                     return;
                 }
 
                 // ignore clicks in tinyMCE dropdown(floatpanel)
-                var floatpanel = $(el).parents(".mce-floatpanel");
+                var floatpanel = $(event.target).closest(".mce-floatpanel");
                 if (floatpanel.length === 1) {
                     return;
                 }

src/Umbraco.Web.UI.Client/src/common/directives/components/imaging/umbimagegravity.directive.js

@@ -105,12 +105,14 @@ angular.module("umbraco.directives")
 					});
 
 					//// INIT /////
-					$image.load(function(){
-						$timeout(function(){
-							setDimensions();
-							scope.loaded = true;
-							scope.onImageLoaded();
-						});
+					$image.load(function() {
+					    $timeout(function() {
+					        setDimensions();
+					        scope.loaded = true;
+					        if (angular.isFunction(scope.onImageLoaded)) {
+					            scope.onImageLoaded();
+					        }
+					});
 					});
 
 					$(window).on('resize.umbImageGravity', function(){

src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js

@@ -427,7 +427,9 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) {
           * Retrieves all media children with types used as folders.
           * Uses the convention of looking for media items with mediaTypes ending in
           * *Folder so will match "Folder", "bannerFolder", "secureFolder" etc,
-          *
+          * 
+          * NOTE: This will return a max of 500 folders, if more is required it needs to be paged
+          * 
           * ##usage
           * <pre>
           * mediaResource.getChildFolders(1234)
@@ -445,14 +447,15 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) {
                 parentId = -1;
             }
 
+            //NOTE: This will return a max of 500 folders, if more is required it needs to be paged
             return umbRequestHelper.resourcePromise(
                   $http.get(
                         umbRequestHelper.getApiUrl(
                               "mediaApiBaseUrl",
                               "GetChildFolders",
-                              [
-                                    { id: parentId }
-                              ])),
+                            {
+                                id: parentId
+                            })),
                   'Failed to retrieve child folders for media item ' + parentId);
         },
 

src/Umbraco.Web.UI.Client/src/common/resources/ourpackagerrepository.resource.js

@@ -5,15 +5,14 @@
     **/
 function ourPackageRepositoryResource($q, $http, umbDataFormatter, umbRequestHelper) {
 
-    //var baseurl = "http://localhost:24292/webapi/packages/v1";
-    var baseurl = "https://our.umbraco.org/webapi/packages/v1";
+    var baseurl = Umbraco.Sys.ServerVariables.umbracoUrls.packagesRestApiBaseUrl;
 
     return {
         
         getDetails: function (packageId) {
 
             return umbRequestHelper.resourcePromise(
-               $http.get(baseurl + "/" + packageId),
+               $http.get(baseurl + "/" + packageId + "?version=" + Umbraco.Sys.ServerVariables.application.version),
                'Failed to get package details');
         },
 

src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js

@@ -19,15 +19,20 @@ angular.module('umbraco.security.interceptor')
                     return promise;
                 }, function(originalResponse) {
                     // Intercept failed requests
+                    
+                    // Make sure we have the configuration of the request (don't we always?)
+                    var config = originalResponse.config ? originalResponse.config : {};
+                    
+                    // Make sure we have an object for the headers of the request
+                    var headers = config.headers ? config.headers : {};
 
-                    //Here we'll check if we should ignore the error, this will be based on an original header set
-                    var headers = originalResponse.config ? originalResponse.config.headers : {};
-                    if (headers["x-umb-ignore-error"] === "ignore") {
+                    //Here we'll check if we should ignore the error (either based on the original header set or the request configuration)
+                    if (headers["x-umb-ignore-error"] === "ignore" || config.umbIgnoreErrors === true) {
                         //exit/ignore
                         return promise;
                     }
                     var filtered = _.find(requestInterceptorFilter(), function(val) {
-                        return originalResponse.config.url.indexOf(val) > 0;
+                        return config.url.indexOf(val) > 0;
                     });
                     if (filtered) {
                         return promise;
@@ -99,4 +104,4 @@ angular.module('umbraco.security.interceptor')
     // We have to add the interceptor to the queue as a string because the interceptor depends upon service instances that are not available in the config block.
     .config(['$httpProvider', function ($httpProvider) {
         $httpProvider.responseInterceptors.push('securityInterceptor');
-    }]);
+    }]);

src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js

@@ -7,6 +7,19 @@ function mediaTypeHelper(mediaTypeResource, $q) {
 
     var mediaTypeHelperService = {
 
+        isFolderType: function(mediaEntity) {
+            if (!mediaEntity) {
+                throw "mediaEntity is null";
+            }
+            if (!mediaEntity.contentTypeAlias) {
+                throw "mediaEntity.contentTypeAlias is null";
+            }
+
+            //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder"
+            //this is the exact same logic that is performed in MediaController.GetChildFolders
+            return mediaEntity.contentTypeAlias.endsWith("Folder");
+        },
+
         getAllowedImagetypes: function (mediaId){
 				
             // Get All allowedTypes

src/Umbraco.Web.UI.Client/src/less/canvas-designer.less

@@ -14,6 +14,7 @@ body {
     overflow: hidden;
     height: 100%;
     width: 100%;
+    width: ~"(calc(~'100%' - ~'80px'))"; // 80px is the fixed left menu for toggling the different browser sizes
     position: absolute;
     padding: 0;
     margin: 0;

src/Umbraco.Web.UI.Client/src/less/tree.less

@@ -49,7 +49,7 @@
 .umb-tree li.current > div i.icon,
 .umb-tree li.current > div ins {
 	color: white !important;
-	background: @blue;
+	background-color: @blue;
 	border-color: @blue;
 }
 

src/Umbraco.Web.UI.Client/src/views/common/dialogs/mediapicker.html

@@ -158,4 +158,4 @@
 
     </div>
 
-</div>
+</form>

src/Umbraco.Web.UI.Client/src/views/components/umb-media-grid.html

@@ -1,5 +1,5 @@
 <div class="umb-media-grid">
-    <div class="umb-media-grid__item" ng-click="clickItem(item, $event, $index)" ng-repeat="item in items | filter:filterBy" ng-style="item.flexStyle" ng-class="{'-selected': item.selected, '-file': !item.thumbnail, '-svg': item.extension == 'svg'}">
+    <div class="umb-media-grid__item" title="{{item.name}}" ng-click="clickItem(item, $event, $index)" ng-repeat="item in items | filter:filterBy" ng-style="item.flexStyle" ng-class="{'-selected': item.selected, '-file': !item.thumbnail, '-svg': item.extension == 'svg'}">
         <div>
             <i ng-show="item.selected" class="icon-check umb-media-grid__checkmark"></i>
 
@@ -12,10 +12,10 @@
             <div class="umb-media-grid__image-background" ng-if="item.thumbnail || item.extension == 'svg'"></div>
 
             <!-- Image thumbnail -->
-            <img class="umb-media-grid__item-image" width="{{item.width}}" height="{{item.height}}" ng-if="item.thumbnail" ng-src="{{item.thumbnail}}" alt="{{item.name}}" title="{{item.name}}" draggable="false" />
+            <img class="umb-media-grid__item-image" width="{{item.width}}" height="{{item.height}}" ng-if="item.thumbnail" ng-src="{{item.thumbnail}}" alt="{{item.name}}" draggable="false" />
 
             <!-- SVG -->
-            <img class="umb-media-grid__item-image" width="{{item.width}}" height="{{item.height}}" ng-if="!item.thumbnail && item.extension == 'svg'" ng-src="{{item.file}}" alt="{{item.name}}" title="{{item.name}}" draggable="false" />
+            <img class="umb-media-grid__item-image" width="{{item.width}}" height="{{item.height}}" ng-if="!item.thumbnail && item.extension == 'svg'" ng-src="{{item.file}}" alt="{{item.name}}" draggable="false" />
 
             <!-- Transparent image - fallback - used to keep image proportions on wrapper-->
             <img class="umb-media-grid__item-image-placeholder" ng-if="!item.thumbnail && item.extension != 'svg'" src="assets/img/transparent.png" alt="{{item.name}}" draggable="false" />

src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html

@@ -3,232 +3,285 @@
     <h3>Examine Management</h3>
 
     <div ng-show="loading">
-        Loading...
+        <div class="umb-healthcheck-group__details-status-overlay"></div>
+        <umb-load-indicator></umb-load-indicator>
     </div>
 
-    <h4>Indexers</h4>
+    <div ng-hide="loading" class="umb-healthcheck-group__details">
 
-    <ul ng-hide="loading">
-        <li class="provider" ng-repeat="indexer in indexerDetails">
+        <div class="umb-healthcheck-group__details-group-title">
+            <div class="umb-healthcheck-group__details-group-name">Indexers</div>
+        </div>
 
-            <a class="btn-link -underline" href="" ng-click="toggle(indexer, 'showProperties')">
-                {{indexer.name}}
-            </a>
+        <div class="umb-healthcheck-group__details-checks">
+            <div class="umb-healthcheck-group__details-check">
+                <div class="umb-healthcheck-group__details-check-title">
+                    <div class="umb-healthcheck-group__details-check-name">Manage Examine's indexes</div>
+                    <div class="umb-healthcheck-group__details-check-description">Allows you to view the details of each index and provides some tools for managing the indexes</div>
+                </div>
 
-            <ul ng-show="indexer.showProperties">
+                <div class="umb-healthcheck-group__details-status" ng-repeat="indexer in indexerDetails">
 
-                <li>
+                    <div class="umb-healthcheck-group__details-status-icon-container">
+                        <i class="umb-healthcheck-status-icon" ng-class="{'icon-check color-green' : indexer.isHealthy, 'icon-delete color-red' : !indexer.isHealthy}"></i>
+                    </div>
 
-                    <a href="" ng-click="toggle(indexer, 'showTools')">Index info & tools</a>
-
-                    <div ng-show="indexer.showTools && indexer.isLuceneIndex">
-                        <div>
-                            <br />
-                            <div ng-show="!indexer.isProcessing && (!indexer.processingAttempts || indexer.processingAttempts < 100)" class="btn-group">
-
-                                <button type="button" class="btn btn-warning" ng-click="rebuildIndex(indexer)">Rebuild index</button>
-                                <button type="button" class="btn btn-warning" ng-click="optimizeIndex(indexer)" ng-show="indexer.documentCount > 0">Optimize index</button>
+                    <div class="umb-healthcheck-group__details-status-content">
+                        <div class="umb-healthcheck-group__details-status-text">
+                            <div ng-show="!indexer.isHealthy">
+                                {{indexer.name}}
                             </div>
-
-                            <div ng-show="indexer.isProcessing" class="umb-loader-wrapper" ng-show="actionInProgress">
-                                <div class="umb-loader"></div>
-                            </div>
-
-                            <div class="error" ng-show="indexer.processingAttempts >= 100">
-                                The process is taking longer than expected, check the umbraco log to see if there have been any errors during this operation
+                            <a class="btn-link -underline" href="" ng-click="toggle(indexer, 'showProperties')" ng-show="indexer.isHealthy">
+                                {{indexer.name}}
+                            </a>
+                            <div ng-if="!indexer.isHealthy" class="text-error">
+                                The index cannot be read and will need to be rebuilt
                             </div>
                         </div>
-                        <table class="table table-bordered table-condensed">
-                            <caption>&nbsp;</caption>
-                            <tr>
-                                <th>Documents in index</th>
-                                <td>{{indexer.documentCount}}</td>
-                            </tr>
-                            <tr>
-                                <th>Fields in index</th>
-                                <td>{{indexer.fieldCount}}</td>
-                            </tr>
-                            <tr>
-                                <th>Has deletions?</th>
-                                <td>
-                                    <span>{{indexer.deletionCount > 0}}</span>
-                                    (<span>{{indexer.deletionCount}}</span>)
-                                </td>
-                            </tr>
-                            <tr>
-                                <th>Optimized?</th>
-                                <td>
-                                    <span>{{indexer.isOptimized}}</span>
-                                </td>
-                            </tr>
 
-                        </table>
-                    </div>
-                </li>
-
-                <li ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0 || indexer.indexCriteria.ExcludeNodeTypes.length > 0 || indexer.indexCriteria.ParentNodeId">
-                    <a href="" ng-click="toggle(indexer, 'showNodeTypes')">Node types</a>
-                    <table ng-show="indexer.showNodeTypes" class="table table-bordered table-condensed">
-                        <tr ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0">
-                            <th>Include node types</th>
-                            <td>{{indexer.indexCriteria.IncludeNodeTypes | json}}</td>
-                        </tr>
-                        <tr ng-show="indexer.indexCriteria.ExcludeNodeTypes.length > 0">
-                            <th>Exclude node types</th>
-                            <td>{{indexer.indexCriteria.ExcludeNodeTypes | json}}</td>
-                        </tr>
-                        <tr ng-show="indexer.indexCriteria.ParentNodeId">
-                            <th>Parent node id</th>
-                            <td>{{indexer.indexCriteria.ParentNodeId}}</td>
-                        </tr>
-                    </table>
-                </li>
-
-                <li ng-show="indexer.indexCriteria.StandardFields.length > 0">
-                    <a href="" ng-click="toggle(indexer, 'showSystemFields')">System fields</a>
-                    <table ng-show="indexer.showSystemFields" class="table table-bordered table-condensed">
-                        <caption>&nbsp;</caption>
-                        <thead>
-                            <tr>
-                                <th>Name</th>
-                                <th>Enable sorting</th>
-                                <th>Type</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                            <tr ng-repeat="field in indexer.indexCriteria.StandardFields">
-                                <th>{{field.Name}}</th>
-                                <td>{{field.EnableSorting}}</td>
-                                <td>{{field.Type}}</td>
-                            </tr>
-                        </tbody>
-                    </table>
-                </li>
-
-                <li ng-show="indexer.indexCriteria.UserFields.length > 0">
-                    <a href="" ng-click="toggle(indexer, 'showUserFields')">User fields</a>
-                    <table ng-show="indexer.showUserFields" class="table table-bordered table-condensed">
-                        <caption>&nbsp;</caption>
-                        <thead>
-                            <tr>
-                                <th>Name</th>
-                                <th>Enable sorting</th>
-                                <th>Type</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                            <tr ng-repeat="field in indexer.indexCriteria.UserFields">
-                                <th>{{field.Name}}</th>
-                                <td>{{field.EnableSorting}}</td>
-                                <td>{{field.Type}}</td>
-                            </tr>
-                        </tbody>
-                    </table>
-                </li>
-
-                <li>
-                    <a href="" ng-click="toggle(indexer, 'showProviderProperties')">Provider properties</a>
-                    <table ng-show="indexer.showProviderProperties" class="table table-bordered table-condensed">
-                        <caption>&nbsp;</caption>
-                        <tr ng-repeat="(key, val) in indexer.providerProperties track by $index">
-                            <th>{{key}}</th>
-                            <td>{{val}}</td>
-                        </tr>
-                    </table>
-                </li>
-            </ul>
-        </li>
-
-
-
-    </ul>
-
-    <h4>Searchers</h4>
-
-    <ul ng-hide="loading">
-        <li class="provider" ng-repeat="searcher in searcherDetails">
-            <a class="btn-link -underline" href="" ng-click="toggle(searcher, 'showProperties')">
-                {{searcher.name}}
-            </a>
-
-            <ul ng-show="searcher.showProperties">
-
-                <li class="search-tools">
-
-                    <a href="" ng-click="toggle(searcher, 'showTools')">Search tools</a>
-
-                    <div ng-show="searcher.showTools">
-                        <a class="hide" href="" ng-click="closeSearch(searcher)" ng-show="searcher.isSearching">Hide search results</a>
-
-                        <br />
-
-                        <form>
-
-                            <div class="row form-search">
-                                <div class="span8 input-append">
-                                    <input type="text" class="search-query" ng-model="searcher.searchText" no-dirty-check />
-                                    <button type="button" class="btn btn-info" ng-click="search(searcher)" ng-disabled="searcher.isProcessing">Search</button>
-                                </div>
-                            </div>
-                            <div class="row">
-                                <label for="{{searcher.name}}-textSearch" class="radio inline">
-                                    <input type="radio" name="searchType" id="{{searcher.name}}-textSearch" value="text" ng-model="searcher.searchType" no-dirty-check />
-                                    Text Search
-                                </label>
-                                <label for="{{searcher.name}}-luceneSearch" class="radio inline">
-                                    <input type="radio" name="searchType" id="{{searcher.name}}-luceneSearch" value="lucene" ng-model="searcher.searchType" no-dirty-check />
-                                    Lucene Search
-                                </label>
+                        <div class="umb-healthcheck-group__details-status-actions" ng-if="!indexer.isHealthy">
+                            <div class="umb-healthcheck-group__details-status-action">
+                                <button type="button" class="umb-era-button -blue"
+                                        ng-show="!indexer.isProcessing && (!indexer.processingAttempts || indexer.processingAttempts < 100)"
+                                        ng-click="rebuildIndex(indexer)">
+                                    Rebuild index
+                                </button>
                             </div>
+                        </div>
 
+                        <div class="umb-healthcheck-group__details-status-actions" ng-show="indexer.isHealthy && indexer.showProperties">
+                            <ul>
+                                <li>
+                                    <a href="" ng-click="toggle(indexer, 'showTools')">Index info & tools</a>
 
-                        </form>
+                                    <div ng-show="indexer.showTools && indexer.isLuceneIndex">
+                                        <div>
+                                            <br />
 
-                        <div class="search-results" ng-show="searcher.isSearching">
+                                            <div ng-show="!indexer.isProcessing && (!indexer.processingAttempts || indexer.processingAttempts < 100)"
+                                                 class="umb-healthcheck-group__details-status-action">
+                                                <button type="button" class="umb-era-button -blue" ng-click="rebuildIndex(indexer)">Rebuild index</button>
+                                            </div>
 
-                            <div ng-show="indexer.isProcessing" class="umb-loader-wrapper" ng-show="indexer.isProcessing">
-                                <div class="umb-loader"></div>
-                            </div>
+                                            <div ng-show="indexer.processingAttempts >= 100">
+                                                The process is taking longer than expected, check the umbraco log to see if there have been any errors during this operation
+                                            </div>
+                                        </div>
+                                        <table class="table table-bordered table-condensed">
+                                            <caption>&nbsp;</caption>
+                                            <tr>
+                                                <th>Documents in index</th>
+                                                <td>{{indexer.documentCount}}</td>
+                                            </tr>
+                                            <tr>
+                                                <th>Fields in index</th>
+                                                <td>{{indexer.fieldCount}}</td>
+                                            </tr>
+                                            <tr>
+                                                <th>Has deletions?</th>
+                                                <td>
+                                                    <span>{{indexer.deletionCount > 0}}</span>
+                                                    (<span>{{indexer.deletionCount}}</span>)
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <th>Optimized?</th>
+                                                <td>
+                                                    <span>{{indexer.isOptimized}}</span>
+                                                </td>
+                                            </tr>
 
-                            <table ng-hide="searcher.isProcessing" class="table table-bordered table-condensed">
-                                <thead>
-                                    <tr>
-                                        <th class="score">Score</th>
-                                        <th class="id">Id</th>
-                                        <th>Values</th>
-                                    </tr>
-                                </thead>
-                                <tbody>
-                                    <tr ng-repeat="result in searcher.searchResults">
-                                        <td>{{result.Score}}</td>
-                                        <td>{{result.Id}}</td>
-                                        <td>
-                                            <span ng-repeat="(key,val) in result.Fields track by $index">
-                                                <span class=""><em>{{key}}</em>:</span>
-                                                <span class="text-info">{{val}}</span>
-                                            </span>
-                                        </td>
-                                    </tr>
-                                </tbody>
-                            </table>
+                                        </table>
+                                    </div>
+                                </li>
+                                <li ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0 || indexer.indexCriteria.ExcludeNodeTypes.length > 0 || indexer.indexCriteria.ParentNodeId">
+                                    <a href="" ng-click="toggle(indexer, 'showNodeTypes')">Node types</a>
+                                    <table ng-show="indexer.showNodeTypes" class="table table-bordered table-condensed">
+                                        <tr ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0">
+                                            <th>Include node types</th>
+                                            <td>{{indexer.indexCriteria.IncludeNodeTypes | json}}</td>
+                                        </tr>
+                                        <tr ng-show="indexer.indexCriteria.ExcludeNodeTypes.length > 0">
+                                            <th>Exclude node types</th>
+                                            <td>{{indexer.indexCriteria.ExcludeNodeTypes | json}}</td>
+                                        </tr>
+                                        <tr ng-show="indexer.indexCriteria.ParentNodeId">
+                                            <th>Parent node id</th>
+                                            <td>{{indexer.indexCriteria.ParentNodeId}}</td>
+                                        </tr>
+                                    </table>
+                                </li>
+                                <li ng-show="indexer.indexCriteria.StandardFields.length > 0">
+                                    <a href="" ng-click="toggle(indexer, 'showSystemFields')">System fields</a>
+                                    <table ng-show="indexer.showSystemFields" class="table table-bordered table-condensed">
+                                        <caption>&nbsp;</caption>
+                                        <thead>
+                                            <tr>
+                                                <th>Name</th>
+                                                <th>Enable sorting</th>
+                                                <th>Type</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody>
+                                            <tr ng-repeat="field in indexer.indexCriteria.StandardFields">
+                                                <th>{{field.Name}}</th>
+                                                <td>{{field.EnableSorting}}</td>
+                                                <td>{{field.Type}}</td>
+                                            </tr>
+                                        </tbody>
+                                    </table>
+                                </li>
+                                <li ng-show="indexer.indexCriteria.UserFields.length > 0">
+                                    <a href="" ng-click="toggle(indexer, 'showUserFields')">User fields</a>
+                                    <table ng-show="indexer.showUserFields" class="table table-bordered table-condensed">
+                                        <caption>&nbsp;</caption>
+                                        <thead>
+                                            <tr>
+                                                <th>Name</th>
+                                                <th>Enable sorting</th>
+                                                <th>Type</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody>
+                                            <tr ng-repeat="field in indexer.indexCriteria.UserFields">
+                                                <th>{{field.Name}}</th>
+                                                <td>{{field.EnableSorting}}</td>
+                                                <td>{{field.Type}}</td>
+                                            </tr>
+                                        </tbody>
+                                    </table>
+                                </li>
+                                <li>
+                                    <a href="" ng-click="toggle(indexer, 'showProviderProperties')">Provider properties</a>
+                                    <table ng-show="indexer.showProviderProperties" class="table table-bordered table-condensed">
+                                        <caption>&nbsp;</caption>
+                                        <tr ng-repeat="(key, val) in indexer.providerProperties track by $index">
+                                            <th>{{key}}</th>
+                                            <td>{{val}}</td>
+                                        </tr>
+                                    </table>
+                                </li>
+                            </ul>
                         </div>
                     </div>
-                </li>
 
-                <li>
-                    <a href="" ng-click="toggle(searcher, 'showProviderProperties')">Provider properties</a>
-                    <table ng-show="searcher.showProviderProperties" class="table table-bordered table-condensed">
-                        <caption>&nbsp;</caption>
-                        <tr ng-repeat="(key, val) in searcher.providerProperties track by $index">
-                            <th>{{key}}</th>
-                            <td>{{val}}</td>
-                        </tr>
-                    </table>
-                </li>
+                    <div ng-show="indexer.isProcessing">
+                        <div class="umb-healthcheck-group__details-status-overlay"></div>
+                        <umb-load-indicator></umb-load-indicator>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <br />
+
+        <div class="umb-healthcheck-group__details-group-title">
+            <div class="umb-healthcheck-group__details-group-name">Searchers</div>
+        </div>
+
+        <div class="umb-healthcheck-group__details-checks">
+            <div class="umb-healthcheck-group__details-check">
+                <div class="umb-healthcheck-group__details-check-title">
+                    <div class="umb-healthcheck-group__details-check-name">Search indexes</div>
+                    <div class="umb-healthcheck-group__details-check-description">Allows you to search the indexes and view the searcher properties</div>
+                </div>
+
+                <div class="umb-healthcheck-group__details-status" ng-repeat="searcher in searcherDetails">
+
+                    <div class="umb-healthcheck-group__details-status-icon-container">
+                        <i class="umb-healthcheck-status-icon icon-info"></i>
+                    </div>
+                    
+                    <div class="umb-healthcheck-group__details-status-content">
+                        <div class="umb-healthcheck-group__details-status-text">
+                            <a class="btn-link -underline" href="" ng-click="toggle(searcher, 'showProperties')">
+                                {{searcher.name}}
+                            </a>
+                        </div>
+                        
+                        <div class="umb-healthcheck-group__details-status-actions" ng-show="searcher.showProperties">
+                            <ul>
+                                <li class="search-tools">
+
+                                    <a href="" ng-click="toggle(searcher, 'showTools')">Search tools</a>
+
+                                    <div ng-show="searcher.showTools">
+                                        <a class="hide" href="" ng-click="closeSearch(searcher)" ng-show="searcher.isSearching">Hide search results</a>
+
+                                        <br />
+
+                                        <form>
+
+                                            <div class="row form-search">
+                                                <div class="span8 input-append">
+                                                    <input type="text" class="search-query" ng-model="searcher.searchText" no-dirty-check />
+                                                    <button type="button" class="btn btn-info" ng-click="search(searcher)" ng-disabled="searcher.isProcessing">Search</button>
+                                                </div>
+                                            </div>
+                                            <div class="row">
+                                                <label for="{{searcher.name}}-textSearch" class="radio inline">
+                                                    <input type="radio" name="searchType" id="{{searcher.name}}-textSearch" value="text" ng-model="searcher.searchType" no-dirty-check />
+                                                    Text Search
+                                                </label>
+                                                <label for="{{searcher.name}}-luceneSearch" class="radio inline">
+                                                    <input type="radio" name="searchType" id="{{searcher.name}}-luceneSearch" value="lucene" ng-model="searcher.searchType" no-dirty-check />
+                                                    Lucene Search
+                                                </label>
+                                            </div>
+
+
+                                        </form>
+
+                                        <div class="search-results" ng-show="searcher.isSearching">
+
+                                            <div ng-show="indexer.isProcessing" class="umb-loader-wrapper" ng-show="indexer.isProcessing">
+                                                <div class="umb-loader"></div>
+                                            </div>
+
+                                            <table ng-hide="searcher.isProcessing" class="table table-bordered table-condensed">
+                                                <thead>
+                                                    <tr>
+                                                        <th class="score">Score</th>
+                                                        <th class="id">Id</th>
+                                                        <th>Values</th>
+                                                    </tr>
+                                                </thead>
+                                                <tbody>
+                                                    <tr ng-repeat="result in searcher.searchResults">
+                                                        <td>{{result.Score}}</td>
+                                                        <td>{{result.Id}}</td>
+                                                        <td>
+                                                            <span ng-repeat="(key,val) in result.Fields track by $index">
+                                                                <span class=""><em>{{key}}</em>:</span>
+                                                                <span class="text-info">{{val}}</span>
+                                                            </span>
+                                                        </td>
+                                                    </tr>
+                                                </tbody>
+                                            </table>
+                                        </div>
+                                    </div>
+                                </li>
+                                <li>
+                                    <a href="" ng-click="toggle(searcher, 'showProviderProperties')">Provider properties</a>
+                                    <table ng-show="searcher.showProviderProperties" class="table table-bordered table-condensed">
+                                        <caption>&nbsp;</caption>
+                                        <tr ng-repeat="(key, val) in searcher.providerProperties track by $index">
+                                            <th>{{key}}</th>
+                                            <td>{{val}}</td>
+                                        </tr>
+                                    </table>
+                                </li>
+                            </ul>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+
+    </div>
 
-            </ul>
 
-        </li>
-    </ul>
 
 </div>

src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js

@@ -1,10 +1,17 @@
 angular.module("umbraco")
     .controller("Umbraco.PropertyEditors.Grid.MediaController",
-    function ($scope, $rootScope, $timeout) {
+    function ($scope, $rootScope, $timeout, userService) {
+
+        if (!$scope.model.config.startNodeId) {
+            userService.getCurrentUser().then(function (userData) {
+                $scope.model.config.startNodeId = userData.startMediaId;
+            });
+        }
 
         $scope.setImage = function(){
             $scope.mediaPickerOverlay = {};
             $scope.mediaPickerOverlay.view = "mediapicker";
+            $scope.mediaPickerOverlay.startNodeId = $scope.model.config && $scope.model.config.startNodeId ? $scope.model.config.startNodeId : undefined;
             $scope.mediaPickerOverlay.cropSize = $scope.control.editor.config && $scope.control.editor.config.size ? $scope.control.editor.config.size : undefined;
             $scope.mediaPickerOverlay.showDetails = true;
             $scope.mediaPickerOverlay.disableFolderSelect = true;

src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html

@@ -51,7 +51,7 @@
       </umb-file-dropzone>
 
       <umb-folder-grid
-         ng-if="folders.length > 0 && !vm.isRecycleBin"
+         ng-if="folders.length > 0"
          folders="folders"
          on-click="vm.selectFolder"
          on-click-name="vm.goToItem">

src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js

@@ -37,7 +37,8 @@
       function activate() {
           vm.itemsWithoutFolders = filterOutFolders($scope.items);
 
-          if($scope.entityType === 'media') {
+          //no need to make another REST/DB call if this data is not used when we are browsing the bin
+          if ($scope.entityType === 'media' && !vm.isRecycleBin) {
             mediaTypeHelper.getAllowedImagetypes(vm.nodeId).then(function (types) {
                 vm.acceptedMediatypes = types;
             });

src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js

@@ -53,7 +53,9 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
    $scope.isNew = false;
    $scope.actionInProgress = false;
    $scope.selection = [];
-   $scope.folders = [];
+   $scope.folders = [];   
+   //tracks if we've already loaded the folders for the current node
+   var foldersLoaded = false;
    $scope.listViewResultSet = {
       totalPages: 0,
       items: []
@@ -268,12 +270,13 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
             });
          }
 
-         if ($scope.entityType === 'media') {
-
+         if (!foldersLoaded && $scope.entityType === 'media') {
+            //The folders aren't loaded - we only need to do this once since we're never changing node ids
             mediaResource.getChildFolders($scope.contentId)
                     .then(function (folders) {
                        $scope.folders = folders;
                        $scope.viewLoaded = true;
+                       foldersLoaded = true;
                     });
 
          } else {