U4-9495 ImageProcessor high CPU usage when trimCache=true

Changes IndexFieldPolicies to be a StaticFieldCollection which is a k…

build/NuSpecs/UmbracoCms.Core.nuspec

@@ -32,9 +32,9 @@
 	  <!-- AutoMapper can not be updated due to: https://github.com/AutoMapper/AutoMapper/issues/373#issuecomment-127644405 -->
       <dependency id="AutoMapper" version="[3.3.1, 4.0.0)" />
       <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Examine" version="[0.1.70, 1.0.0)" />
-      <dependency id="ImageProcessor" version="[2.5.1, 3.0.0)" />
-      <dependency id="ImageProcessor.Web" version="[4.7.2, 5.0.0)" />
+      <dependency id="Examine" version="[0.1.80, 1.0.0)" />
+      <dependency id="ImageProcessor" version="[2.5.2, 3.0.0)" />
+      <dependency id="ImageProcessor.Web" version="[4.8.2, 5.0.0)" />
       <dependency id="semver" version="[1.1.2, 2.0.0)" />
       <dependency id="UrlRewritingNet" version="[2.0.7, 3.0.0)" />	  
 	  <!-- Markdown can not be updated due to: https://github.com/hey-red/markdownsharp/issues/71#issuecomment-233585487 -->

build/NuSpecs/UmbracoCms.nuspec

@@ -45,6 +45,7 @@
     <file src="tools\umbracoSettings.config.install.xdt" target="Content\config\umbracoSettings.config.install.xdt" />
     <file src="tools\Views.Web.config.install.xdt" target="Views\Web.config.install.xdt" />
     <file src="tools\processing.config.install.xdt" target="Content\Config\imageprocessor\processing.config.install.xdt" />
+    <file src="tools\cache.config.install.xdt" target="Content\Config\imageprocessor\cache.config.install.xdt" />
     <file src="build\**" target="build" />
   </files>
 </package>

build/NuSpecs/tools/cache.config.install.xdt

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<caching xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform">
+  <caches>
+    <cache name="DiskCache" trimCache="false" xdt:Transform="SetAttributes(trimCache)" xdt:Locator="Match(name)"  />
+  </caches>
+</caching>

build/UmbracoVersion.txt

@@ -1,2 +1,2 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
-7.5.7
+7.5.9

src/SolutionInfo.cs

@@ -11,5 +11,5 @@ using System.Resources;
 
 [assembly: AssemblyVersion("1.0.*")]
 
-[assembly: AssemblyFileVersion("7.5.7")]
-[assembly: AssemblyInformationalVersion("7.5.7")]
+[assembly: AssemblyFileVersion("7.5.9")]
+[assembly: AssemblyInformationalVersion("7.5.9")]

src/Umbraco.Core/Cache/CacheKeys.cs

@@ -64,7 +64,8 @@ namespace Umbraco.Core.Cache
         [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
         public const string ContentTypePropertiesCacheKey = "ContentType_PropertyTypes_Content:";
         
-        [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
+        [Obsolete("No longer used and will be removed in v8")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         public const string PropertyTypeCacheKey = "UmbracoPropertyTypeCache";
 
         [Obsolete("This is no longer used and will be removed from the codebase in the future")]

src/Umbraco.Core/Configuration/UmbracoVersion.cs

@@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration
 {
     public class UmbracoVersion
     {
-        private static readonly Version Version = new Version("7.5.7");
+        private static readonly Version Version = new Version("7.5.9");
 
         /// <summary>
         /// Gets the current version of Umbraco.

src/Umbraco.Core/CoreBootManager.cs

@@ -415,7 +415,7 @@ namespace Umbraco.Core
 
             if (currentTry == 5)
             {
-                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but the Umbraco cannot connect to the database.");
+                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but Umbraco cannot connect to the database.");
             }
         }
 

src/Umbraco.Core/DatabaseContext.cs

@@ -203,15 +203,10 @@ namespace Umbraco.Core
             var path = Path.Combine(GlobalSettings.FullpathToRoot, "App_Data", "Umbraco.sdf");
             if (File.Exists(path) == false)
             {
-                var engine = new SqlCeEngine(connectionString);
-                engine.CreateDatabase();
-
-                // SD: Pretty sure this should be in a using clause but i don't want to cause unknown side-effects here
-                // since it's been like this for quite some time
-                //using (var engine = new SqlCeEngine(connectionString))
-                //{
-                //    engine.CreateDatabase();    
-                //}
+                using (var engine = new SqlCeEngine(connectionString))
+                {
+                    engine.CreateDatabase();
+                }
             }
 
             Initialize(providerName);

src/Umbraco.Core/Models/IMediaType.cs

@@ -4,7 +4,7 @@ namespace Umbraco.Core.Models
 {
     /// <summary>
     /// Defines a ContentType, which Media is based on
-    /// </summary
+    /// </summary>
     public interface IMediaType : IContentTypeComposition
     {
 

src/Umbraco.Core/Persistence/Factories/ContentFactory.cs

@@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IContent,DocumentDto>
 
-        public IContent BuildEntity(DocumentDto dto)
+        public static IContent BuildEntity(DocumentDto dto, IContentType contentType)
         {
-            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, _contentType);
+            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, contentType);
 
             try
             {
                 content.DisableChangeTracking();
 
-                content.Id = _id;
+                content.Id = dto.NodeId;
                 content.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                 content.Name = dto.Text;
                 content.NodeName = dto.ContentVersionDto.ContentDto.NodeDto.Text;
@@ -49,8 +49,8 @@ namespace Umbraco.Core.Persistence.Factories
                 content.Published = dto.Published;
                 content.CreateDate = dto.ContentVersionDto.ContentDto.NodeDto.CreateDate;
                 content.UpdateDate = dto.ContentVersionDto.VersionDate;
-                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?) null;
-                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?) null;
+                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?)null;
+                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?)null;
                 content.Version = dto.ContentVersionDto.VersionId;
                 content.PublishedState = dto.Published ? PublishedState.Published : PublishedState.Unpublished;
                 content.PublishedVersionGuid = dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId;
@@ -64,6 +64,13 @@ namespace Umbraco.Core.Persistence.Factories
             {
                 content.EnableChangeTracking();
             }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IContent BuildEntity(DocumentDto dto)
+        {
+            return BuildEntity(dto, _contentType);
         }
 
         public DocumentDto BuildDto(IContent entity)

src/Umbraco.Core/Persistence/Factories/MediaFactory.cs

@@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IMedia,ContentVersionDto>
 
-        public IMedia BuildEntity(ContentVersionDto dto)
+        public static IMedia BuildEntity(ContentVersionDto dto, IMediaType contentType)
         {
-            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, _contentType);
+            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, contentType);
 
             try
             {
                 media.DisableChangeTracking();
 
-                media.Id = _id;
+                media.Id = dto.NodeId;
                 media.Key = dto.ContentDto.NodeDto.UniqueId;
                 media.Path = dto.ContentDto.NodeDto.Path;
                 media.CreatorId = dto.ContentDto.NodeDto.UserId.Value;
@@ -55,6 +55,13 @@ namespace Umbraco.Core.Persistence.Factories
             {
                 media.EnableChangeTracking();
             }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMedia BuildEntity(ContentVersionDto dto)
+        {
+            return BuildEntity(dto, _contentType);
         }
 
         public ContentVersionDto BuildDto(IMedia entity)

src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs

@@ -30,11 +30,11 @@ namespace Umbraco.Core.Persistence.Factories
             _updateDate = updateDate;
         }
 
-        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        public static IEnumerable<Property> BuildEntity(IReadOnlyCollection<PropertyDataDto> dtos, PropertyType[] compositionTypeProperties, DateTime createDate, DateTime updateDate)
         {
             var properties = new List<Property>();
 
-            foreach (var propertyType in _compositionTypeProperties)
+            foreach (var propertyType in compositionTypeProperties)
             {
                 var propertyDataDto = dtos.LastOrDefault(x => x.PropertyTypeId == propertyType.Id);
                 var property = propertyDataDto == null
@@ -47,8 +47,8 @@ namespace Umbraco.Core.Persistence.Factories
                     //on initial construction we don't want to have dirty properties tracked
                     property.DisableChangeTracking();
 
-                    property.CreateDate = _createDate;
-                    property.UpdateDate = _updateDate;
+                    property.CreateDate = createDate;
+                    property.UpdateDate = updateDate;
                     // http://issues.umbraco.org/issue/U4-1946
                     property.ResetDirtyProperties(false);
                     properties.Add(property);
@@ -57,12 +57,18 @@ namespace Umbraco.Core.Persistence.Factories
                 {
                     property.EnableChangeTracking();
                 }
-                
+
             }
 
             return properties;
         }
 
+        [Obsolete("Use the static method instead, there's no reason to allocate one of these classes everytime we want to map values")]
+        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        {
+            return BuildEntity(dtos, _compositionTypeProperties, _createDate, _updateDate);
+        }
+
         public IEnumerable<PropertyDataDto> BuildDto(IEnumerable<Property> properties)
         {
             var propertyDataDtos = new List<PropertyDataDto>();

src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs

@@ -20,6 +20,7 @@ namespace Umbraco.Core.Persistence.Querying
             _mapper = mapper;
         }
 
+        [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public ModelToSqlExpressionVisitor()
             : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T)))
         { }

src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs

@@ -18,14 +18,13 @@ namespace Umbraco.Core.Persistence.Querying
         public PocoToSqlExpressionVisitor(ISqlSyntaxProvider syntaxProvider)
             : base(syntaxProvider)
         {
-            
+            _pd = new Database.PocoData(typeof(T));
         }
 
         [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public PocoToSqlExpressionVisitor()
-            : base(SqlSyntaxContext.SqlSyntaxProvider)
-        {
-            _pd = new Database.PocoData(typeof(T));
+            : this(SqlSyntaxContext.SqlSyntaxProvider)
+        {            
         }
 
         protected override string VisitMemberAccess(MemberExpression m)

src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Persistence.Repositories
+{
+    internal enum BaseQueryType
+    {
+        Full,
+        Ids,
+        Count
+    }
+}

src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs

@@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = GetBaseQuery(false)
                 .Where(GetBaseWhereClause(), new { Id = id })
-                .Where<DocumentDto>(x => x.Newest)
+                .Where<DocumentDto>(x => x.Newest, SqlSyntax)
                 .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(SqlSyntax.SelectTop(sql, 1)).FirstOrDefault();
@@ -70,63 +70,90 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected override IEnumerable<IContent> PerformGetAll(params int[] ids)
         {
-            var sql = GetBaseQuery(false);
-            if (ids.Any())
+            Func<Sql, Sql> translate = s =>
             {
-                sql.Where("umbracoNode.id in (@ids)", new { ids });
-            }
+                if (ids.Any())
+                {
+                    s.Where("umbracoNode.id in (@ids)", new { ids });
+                }
+                //we only want the newest ones with this method
+                s.Where<DocumentDto>(x => x.Newest, SqlSyntax);
+                return s;
+            };
+            
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
 
-            //we only want the newest ones with this method
-            sql.Where<DocumentDto>(x => x.Newest);
-
-            return ProcessQuery(sql);
+            return ProcessQuery(translate(sqlBaseFull), translate(sqlBaseIds));
         }
 
         protected override IEnumerable<IContent> PerformGetByQuery(IQuery<IContent> query)
         {
-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Newest)
-                                .OrderByDescending<ContentVersionDto>(x => x.VersionDate)
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
 
-            return ProcessQuery(sql);
+            Func<SqlTranslator<IContent>, Sql> translate = (translator) =>
+            {
+                return translator.Translate()
+                    .Where<DocumentDto>(x => x.Newest, SqlSyntax)
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
+            var translatorFull = new SqlTranslator<IContent>(sqlBaseFull, query);
+            var translatorIds = new SqlTranslator<IContent>(sqlBaseIds, query);
+
+            return ProcessQuery(translate(translatorFull), translate(translatorIds));
         }
 
         #endregion        
 
         #region Overrides of PetaPocoRepositoryBase<IContent>
 
-
-        protected override Sql GetBaseQuery(bool isCount)
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
         {
-            var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsDocument.nodeId" : "*"))
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<DocumentDto, ContentVersionDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            if (queryType == BaseQueryType.Full)
+            {
+                //The only reason we apply this left outer join is to be able to pull back the DocumentPublishedReadOnlyDto
+                //information with the entire data set, so basically this will get both the latest document and also it's published
+                //version if it has one. When performing a count or when just retrieving Ids like in paging, this is unecessary 
+                //and causes huge performance overhead for the SQL server, especially when sorting the result. 
+                //To fix this perf overhead we'd need another index on :
+                // CREATE NON CLUSTERED INDEX ON cmsDocument.node + cmsDocument.published
+
+                var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
                 SqlSyntax.GetQuotedTableName("cmsDocument"),
                 SqlSyntax.GetQuotedTableName("cmsDocument2"),
                 SqlSyntax.GetQuotedColumnName("nodeId"),
                 SqlSyntax.GetQuotedColumnName("published"));
 
-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<DocumentDto, ContentVersionDto>(left => left.VersionId, right => right.VersionId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-
                 // cannot do this because PetaPoco does not know how to alias the table
                 //.LeftOuterJoin<DocumentPublishedReadOnlyDto>()
                 //.On<DocumentDto, DocumentPublishedReadOnlyDto>(left => left.NodeId, right => right.NodeId)
                 // so have to rely on writing our own SQL
-                .Append(sqlx/*, new { @published = true }*/)
+                sql.Append(sqlx /*, new { @published = true }*/);
+            }
+
+            sql.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
 
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
             return sql;
         }
 
+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
+        }
+
         protected override string GetBaseWhereClause()
         {
             return "umbracoNode.id = @Id";
@@ -173,20 +200,32 @@ namespace Umbraco.Core.Persistence.Repositories
             // not bring much safety - so this reverts to updating each record individually,
             // and it may be slower in the end, but should be more resilient.
 
-            var baseId = 0;
             var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
+
+            Func<int, Sql, Sql> translate = (bId, sql) =>
+            {
+                if (contentTypeIdsA.Length > 0)
+                {
+                    sql.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
+
+                sql
+                    .Where<NodeDto>(x => x.NodeId > bId && x.Trashed == false, SqlSyntax)
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
+
+                return sql;
+            };
+
+            var baseId = 0;
+            
             while (true)
             {
                 // get the next group of nodes
-                var query = GetBaseQuery(false);
-                if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
-                query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId && x.Trashed == false)
-                    .Where<DocumentDto>(x => x.Published)
-                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
-                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
+                var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.Full));
+                var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids));
+
+                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), SqlSyntax.SelectTop(sqlIds, groupSize))
                     .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
                     .ToList();
 
@@ -212,17 +251,23 @@ namespace Umbraco.Core.Persistence.Repositories
 
         public override IEnumerable<IContent> GetAllVersions(int id)
         {
-            var sql = GetBaseQuery(false)
-                .Where(GetBaseWhereClause(), new { Id = id })
-                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
-            return ProcessQuery(sql, true);
+            Func<Sql, Sql> translate = s =>
+            {
+                return s.Where(GetBaseWhereClause(), new {Id = id})
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
+            };
+
+            var sqlFull = translate(GetBaseQuery(BaseQueryType.Full));
+            var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
+            
+            return ProcessQuery(sqlFull, sqlIds, true);
         }
 
         public override IContent GetByVersion(Guid versionId)
         {
             var sql = GetBaseQuery(false);
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql).FirstOrDefault();
 
@@ -238,10 +283,10 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = new Sql()
                 .Select("*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
-                .Where<ContentVersionDto>(x => x.VersionId == versionId)
-                .Where<DocumentDto>(x => x.Newest != true);
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax).On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .Where<ContentVersionDto>(x => x.VersionId == versionId, SqlSyntax)
+                .Where<DocumentDto>(x => x.Newest != true, SqlSyntax);
             var dto = Database.Fetch<DocumentDto, ContentVersionDto>(sql).FirstOrDefault();
 
             if (dto == null) return;
@@ -616,21 +661,27 @@ namespace Umbraco.Core.Persistence.Repositories
 
         public IEnumerable<IContent> GetByPublishedVersion(IQuery<IContent> query)
         {
+            Func<SqlTranslator<IContent>, Sql> translate = t =>
+            {
+                return t.Translate()
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
             // we WANT to return contents in top-down order, ie parents should come before children
             // ideal would be pure xml "document order" which can be achieved with:
             // ORDER BY substring(path, 1, len(path) - charindex(',', reverse(path))), sortOrder
             // but that's probably an overkill - sorting by level,sortOrder should be enough
 
-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Published)
-                                .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
-                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            var sqlFull = GetBaseQuery(BaseQueryType.Full);
+            var translatorFull = new SqlTranslator<IContent>(sqlFull, query);
+            var sqlIds = GetBaseQuery(BaseQueryType.Ids);
+            var translatorIds = new SqlTranslator<IContent>(sqlIds, query);
 
-            return ProcessQuery(sql, true);
+            return ProcessQuery(translate(translatorFull), translate(translatorIds), true);
         }
-
+        
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -662,10 +713,14 @@ namespace Umbraco.Core.Persistence.Repositories
             parent.Attributes.Append(pIdAtt);
             xmlDoc.AppendChild(parent);
 
-            const string sql = @"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.xml, umbracoNode.level from umbracoNode
+            //Ensure that only nodes that have published versions are selected
+            var sql = string.Format(@"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.{0}, umbracoNode.{1} from umbracoNode
 inner join cmsContentXml on cmsContentXml.nodeId = umbracoNode.id and umbracoNode.nodeObjectType = @type
 where umbracoNode.id in (select cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1)
-order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
+order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
+                SqlSyntax.GetQuotedColumnName("xml"),
+                SqlSyntax.GetQuotedColumnName("level"),
+                SqlSyntax.GetQuotedColumnName("level"));
 
             XmlElement last = null;
 
@@ -802,9 +857,9 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
             
             Func<Tuple<string, object[]>> filterCallback = () => new Tuple<string, object[]>(filterSql.SQL, filterSql.Arguments);
 
-            return GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<DocumentDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsDocument", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
 
         }
@@ -835,16 +890,32 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
             return base.GetDatabaseFieldNameForOrderBy(orderBy);
         }
 
-        private IEnumerable<IContent> ProcessQuery(Sql sql, bool withCache = false)
+        /// <summary>
+        /// This is the underlying method that processes most queries for this repository
+        /// </summary>
+        /// <param name="sqlFull">
+        /// The full SQL with the outer join to return all data required to create an IContent
+        /// </param>
+        /// <param name="sqlIds">
+        /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
+        /// </param>
+        /// <param name="withCache"></param>
+        /// <returns></returns>
+        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false)
         {
             // fetch returns a list so it's ok to iterate it in this method
-            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql);
+            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sqlFull);
             if (dtos.Count == 0) return Enumerable.Empty<IContent>();
 
             var content = new IContent[dtos.Count];
             var defs = new List<DocumentDefinition>();
             var templateIds = new List<int>();
-
+            
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IContentType>();
+            
             for (var i = 0; i < dtos.Count; i++)
             {
                 var dto = dtos[i];
@@ -863,9 +934,19 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
 
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
-                var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
+
+                IContentType contentType;
+                if (contentTypes.ContainsKey(dto.ContentVersionDto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
+                }
+                
+                content[i] = ContentFactory.BuildEntity(dto, contentType);
 
                 // need template
                 if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
@@ -886,7 +967,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
                 .ToDictionary(x => x.Id, x => x);
 
             // load all properties for all documents from database in 1 query
-            var propertyData = GetPropertyCollection(sql, defs);
+            var propertyData = GetPropertyCollection(sqlIds, defs);
 
             // assign
             var dtoIndex = 0;
@@ -906,7 +987,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
             return content;
@@ -923,8 +1004,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
         {
             var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
-            var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var content = factory.BuildEntity(dto);
+            var content = ContentFactory.BuildEntity(dto, contentType);
 
             //Check if template id is set on DocumentDto, and get ITemplate if it is.
             if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)

src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs

@@ -62,17 +62,27 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected override IEnumerable<EntityContainer> PerformGetAll(params int[] ids)
         {
-            //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
-            return ids.InGroupsOf(2000).SelectMany(@group =>
+            if (ids.Any())
+            {
+                //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
+                return ids.InGroupsOf(2000).SelectMany(@group =>
+                {
+                    var sql = GetBaseQuery(false)
+                        .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId})
+                        .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new {ids = @group});
+
+                    sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
+
+                    return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
+                });
+            }
+            else
             {
                 var sql = GetBaseQuery(false)
-                    .Where("nodeObjectType=@umbracoObjectTypeId", new { umbracoObjectTypeId = NodeObjectTypeId })
-                    .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new { ids = @group });
-
+                    .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId});
                 sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
-
                 return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
-            });
+            }
         }
 
         protected override IEnumerable<EntityContainer> PerformGetByQuery(IQuery<EntityContainer> query)

src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs

@@ -96,5 +96,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IContent> GetPagedResultsByQuery(IQuery<IContent> query, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter = null);
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs

@@ -38,15 +38,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <returns>An Enumerable list of <see cref="IMedia"/> objects</returns>
         IEnumerable<IMedia> GetPagedResultsByQuery(IQuery<IMedia> query, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter = "");
-
-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords);
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs

@@ -73,6 +73,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="content"></param>
         /// <param name="xml"></param>
         void AddOrUpdatePreviewXml(IMember content, Func<IMember, XElement> xml);
-
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs

@@ -67,5 +67,16 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="id">Id of the <see cref="TEntity"/> object to delete versions from</param>
         /// <param name="versionDate">Latest version date</param>
         void DeleteVersions(int id, DateTime versionDate);
+
+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords);
     }
 }

src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs

@@ -76,7 +76,7 @@ namespace Umbraco.Core.Persistence.Repositories
             var sqlClause = GetBaseQuery(false);
             var translator = new SqlTranslator<IMedia>(sqlClause, query);
             var sql = translator.Translate()
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
 
             return ProcessQuery(sql);
         }
@@ -84,18 +84,23 @@ namespace Umbraco.Core.Persistence.Repositories
         #endregion
 
         #region Overrides of PetaPocoRepositoryBase<int,IMedia>
+        
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
+        {
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsContentVersion.contentId" : "*"))
+                .From<ContentVersionDto>(SqlSyntax)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax)
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
+            return sql;
+        }
 
         protected override Sql GetBaseQuery(bool isCount)
         {
-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<ContentVersionDto>()
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
-            return sql;
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
         }
 
         protected override string GetBaseWhereClause()
@@ -148,6 +153,11 @@ namespace Umbraco.Core.Persistence.Repositories
             var content = new IMedia[dtos.Count];
             var defs = new List<DocumentDefinition>();
 
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IMediaType>();
+
             for (var i = 0; i < dtos.Count; i++)
             {
                 var dto = dtos[i];
@@ -165,9 +175,19 @@ namespace Umbraco.Core.Persistence.Repositories
 
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
-                var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
-                var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
+
+                IMediaType contentType;
+                if (contentTypes.ContainsKey(dto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentDto.ContentTypeId] = contentType;
+                }
+                
+                content[i] = MediaFactory.BuildEntity(dto, contentType);
 
                 // need properties
                 defs.Add(new DocumentDefinition(
@@ -195,7 +215,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
             return content;
@@ -205,26 +225,16 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = GetBaseQuery(false);
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sql).FirstOrDefault();
 
             if (dto == null)
                 return null;
 
-            var mediaType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+            var content = CreateMediaFromDto(dto, versionId, sql);
 
-            var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
-
-            var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) });
-
-            media.Properties = properties[dto.NodeId];
-
-            //on initial construction we don't want to have dirty properties tracked
-            // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            return content;
         }
 
         public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
@@ -245,7 +255,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     query = query
                         .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
                 query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId)
+                    .Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
                     .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
                 var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
                     .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
@@ -495,37 +505,13 @@ namespace Umbraco.Core.Persistence.Repositories
                 filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
             }
 
-            return GetPagedResultsByQuery<ContentVersionDto, Models.Media>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<ContentVersionDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsContentVersion", "contentId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
 
         }
 
-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        public IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords)
-        {
-            Sql query;
-            if (path == "-1")
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE nodeObjectType = @0)", Guid.Parse(Constants.ObjectTypes.Media)).OrderBy("nodeId");
-            }
-            else
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE path LIKE @0)", path.EnsureEndsWith(",%")).OrderBy("nodeId");
-            }
-            var pagedResult = Database.Page<ContentXmlDto>(pageIndex+1, pageSize, query);
-            totalRecords = pagedResult.TotalItems;
-            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
-        }
-
         /// <summary>
         /// Private method to create a media object from a ContentDto
         /// </summary>
@@ -536,9 +522,8 @@ namespace Umbraco.Core.Persistence.Repositories
         private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
         {
             var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
-
-            var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            
+            var media = MediaFactory.BuildEntity(dto, contentType);
 
             var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
 

src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs

@@ -107,24 +107,29 @@ namespace Umbraco.Core.Persistence.Repositories
 
         #region Overrides of PetaPocoRepositoryBase<int,IMembershipUser>
 
-        protected override Sql GetBaseQuery(bool isCount)
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
         {
             var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<MemberDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<ContentVersionDto, MemberDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsMember.nodeId" : "*"))
+                .From<MemberDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, MemberDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 //We're joining the type so we can do a query against the member type - not sure if this adds much overhead or not?
                 // the execution plan says it doesn't so we'll go with that and in that case, it might be worth joining the content
                 // types by default on the document and media repo's so we can query by content type there too.
-                .InnerJoin<ContentTypeDto>().On<ContentTypeDto, ContentDto>(left => left.NodeId, right => right.ContentTypeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
+                .InnerJoin<ContentTypeDto>(SqlSyntax)
+                .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
             return sql;
+        }
 
+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
         }
 
         protected override string GetBaseWhereClause()
@@ -381,7 +386,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Where(GetBaseWhereClause(), new { Id = id })
                 .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
             return ProcessQuery(sql, true);
-        }
+        }        
 
         public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
         {
@@ -617,9 +622,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
             }
 
-            return GetPagedResultsByQuery<MemberDto, Member>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<MemberDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsMember", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
         }
 

src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.Logging;
@@ -25,8 +27,6 @@ using Umbraco.Core.IO;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
-    using SqlSyntax;
-
     internal abstract class VersionableRepositoryBase<TId, TEntity> : PetaPocoRepositoryBase<TId, TEntity>
         where TEntity : class, IAggregateRoot
     {
@@ -137,6 +137,40 @@ namespace Umbraco.Core.Persistence.Repositories
 
         #endregion
 
+        /// <summary>
+        /// Gets paged document descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public virtual IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords)
+        {
+            var query = new Sql().Select(string.Format("umbracoNode.id, cmsContentXml.{0}", SqlSyntax.GetQuotedColumnName("xml")))
+                .From("umbracoNode")
+                .InnerJoin("cmsContentXml").On("cmsContentXml.nodeId = umbracoNode.id");
+
+            if (path == "-1")
+            {
+                query.Where("umbracoNode.nodeObjectType = @type", new { type = NodeObjectTypeId });
+            }
+            else
+            {
+                query.Where(string.Format("umbracoNode.{0} LIKE (@0)", SqlSyntax.GetQuotedColumnName("path")), path.EnsureEndsWith(",%"));
+            }
+
+            //each order by param needs to be in a bracket! see: https://github.com/toptensoftware/PetaPoco/issues/177
+            query.OrderBy(orderBy == null
+                ? "(umbracoNode.id)"
+                : string.Join(",", orderBy.Select(x => string.Format("({0})", SqlSyntax.GetQuotedColumnName(x)))));
+
+            var pagedResult = Database.Page<ContentXmlDto>(pageIndex + 1, pageSize, query);
+            totalRecords = pagedResult.TotalItems;
+            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
+        }
+
         public int CountDescendants(int parentId, string contentTypeAlias = null)
         {
             var pathMatch = parentId == -1
@@ -348,12 +382,8 @@ namespace Umbraco.Core.Persistence.Repositories
                                 ON CustomPropData.CustomPropValContentId = umbracoNode.id
                 ", sortedInt, sortedDecimal, sortedDate, sortedString, nodeIdSelect.Item2, nodeIdSelect.Item1, versionQuery, sortedSql.Arguments.Length, newestQuery);
 
-                //insert this just above the first LEFT OUTER JOIN (for cmsDocument) or the last WHERE (everything else)
-                string newSql;
-                if (nodeIdSelect.Item1 == "cmsDocument")
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.IndexOf("LEFT OUTER JOIN"), outerJoinTempTable);
-                else
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);
+                //insert this just above the last WHERE
+                string newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);
 
                 var newArgs = sortedSql.Arguments.ToList();
                 newArgs.Add(orderBy);
@@ -370,11 +400,14 @@ namespace Umbraco.Core.Persistence.Repositories
                 }
             }
 
-            //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
-            // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
-            // is empty for many nodes)
-            // see: http://issues.umbraco.org/issue/U4-8831
-            sortedSql.OrderBy("umbracoNode.id");
+            if (orderBySystemField && orderBy != "umbracoNode.id")
+            {
+                //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
+                // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
+                // is empty for many nodes)
+                // see: http://issues.umbraco.org/issue/U4-8831
+                sortedSql.OrderBy("umbracoNode.id");
+            }
 
             return sortedSql;
 
@@ -384,7 +417,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// A helper method for inheritors to get the paged results by query in a way that minimizes queries
         /// </summary>
         /// <typeparam name="TDto">The type of the d.</typeparam>
-        /// <typeparam name="TContentBase">The 'true' entity type (i.e. Content, Member, etc...)</typeparam>
         /// <param name="query">The query.</param>
         /// <param name="pageIndex">Index of the page.</param>
         /// <param name="pageSize">Size of the page.</param>
@@ -397,34 +429,30 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
         /// <returns></returns>
         /// <exception cref="System.ArgumentNullException">orderBy</exception>
-        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto, TContentBase>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
+        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
             Tuple<string, string> nodeIdSelect,
-            Func<Sql, IEnumerable<TEntity>> processQuery,
+            Func<Sql, Sql, IEnumerable<TEntity>> processQuery,
             string orderBy,
             Direction orderDirection,
             bool orderBySystemField,
             Func<Tuple<string, object[]>> defaultFilter = null)
-            where TContentBase : class, IAggregateRoot, TEntity
         {
             if (orderBy == null) throw new ArgumentNullException("orderBy");
 
-            // Get base query
-            var sqlBase = GetBaseQuery(false);
+            // Get base query for returning IDs
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
+            // Get base query for returning all data
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
 
             if (query == null) query = new Query<TEntity>();
-            var translator = new SqlTranslator<TEntity>(sqlBase, query);
-            var sqlQuery = translator.Translate();
-
-            // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, 
-            // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
-            // So we'll modify the SQL.
-            var sqlNodeIds = new Sql(
-                sqlQuery.SQL.Replace("SELECT *", string.Format("SELECT {0}.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2)),
-                sqlQuery.Arguments);
+            var translatorIds = new SqlTranslator<TEntity>(sqlBaseIds, query);
+            var sqlQueryIds = translatorIds.Translate();
+            var translatorFull = new SqlTranslator<TEntity>(sqlBaseFull, query);
+            var sqlQueryFull = translatorFull.Translate();
 
             //get sorted and filtered sql
             var sqlNodeIdsWithSort = GetSortedSqlForPagedResults(
-                GetFilteredSqlForPagedResults(sqlNodeIds, defaultFilter),
+                GetFilteredSqlForPagedResults(sqlQueryIds, defaultFilter),
                 orderDirection, orderBy, orderBySystemField, nodeIdSelect);
 
             // Get page of results and total count
@@ -433,7 +461,7 @@ namespace Umbraco.Core.Persistence.Repositories
             totalRecords = Convert.ToInt32(pagedResult.TotalItems);
 
             //NOTE: We need to check the actual items returned, not the 'totalRecords', that is because if you request a page number
-            // that doesn't actually have any data on it, the totalRecords will still indicate there are records but there are none in 
+            // that doesn't actually have any data on it, the totalRecords will still indicate there are records but there are none in
             // the pageResult, then the GetAll will actually return ALL records in the db.
             if (pagedResult.Items.Any())
             {
@@ -442,29 +470,23 @@ namespace Umbraco.Core.Persistence.Repositories
                 string sqlStringCount, sqlStringPage;
                 Database.BuildPageQueries<TDto>(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage);
 
-                //if this is for sql server, the sqlPage will start with a SELECT * but we don't want that, we only want to return the nodeId                
-                sqlStringPage = sqlStringPage
-                    .Replace("SELECT *",
-                        //This ensures we only take the field name of the node id select and not the table name - since the resulting select
-                        // will ony work with the field name.
-                        "SELECT " + nodeIdSelect.Item2);
-
-                //We need to make this an inner join on the paged query
-                var splitQuery = sqlQuery.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
-                var withInnerJoinSql = new Sql(splitQuery[0])
+                //We need to make this FULL query an inner join on the paged ID query
+                var splitQuery = sqlQueryFull.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
+                var fullQueryWithPagedInnerJoin = new Sql(splitQuery[0])
                     .Append("INNER JOIN (")
                     //join the paged query with the paged query arguments
                     .Append(sqlStringPage, args)
                     .Append(") temp ")
                     .Append(string.Format("ON {0}.{1} = temp.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2))
                     //add the original where clause back with the original arguments
-                    .Where(splitQuery[1], sqlQuery.Arguments);
+                    .Where(splitQuery[1], sqlQueryIds.Arguments);
 
                 //get sorted and filtered sql
                 var fullQuery = GetSortedSqlForPagedResults(
-                    GetFilteredSqlForPagedResults(withInnerJoinSql, defaultFilter),
+                    GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter),
                     orderDirection, orderBy, orderBySystemField, nodeIdSelect);
-                return processQuery(fullQuery);
+
+                return processQuery(fullQuery, sqlNodeIdsWithSort);
             }
             else
             {
@@ -476,11 +498,11 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected IDictionary<int, PropertyCollection> GetPropertyCollection(
             Sql docSql,
-            IEnumerable<DocumentDefinition> documentDefs)
+            IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
-            if (documentDefs.Any() == false) return new Dictionary<int, PropertyCollection>();
+            if (documentDefs.Count == 0) return new Dictionary<int, PropertyCollection>();
 
-            //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use 
+            //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use
             // the statement to go get the property data for all of the items by using an inner join
             var parsedOriginalSql = "SELECT {0} " + docSql.SQL.Substring(docSql.SQL.IndexOf("FROM", StringComparison.Ordinal));
             //now remove everything from an Orderby clause and beyond
@@ -489,23 +511,12 @@ namespace Umbraco.Core.Persistence.Repositories
                 parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
             }
 
-            var propSql = new Sql(@"SELECT cmsPropertyData.*
-FROM cmsPropertyData
-INNER JOIN cmsPropertyType
-ON cmsPropertyData.propertytypeid = cmsPropertyType.id
-INNER JOIN 
-	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
-ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-LEFT OUTER JOIN cmsDataTypePreValues
-ON cmsPropertyType.dataTypeId = cmsDataTypePreValues.datatypeNodeId", docSql.Arguments);
-
-            var allPropertyData = Database.Fetch<PropertyDataDto>(propSql);
-
-            //This is a lazy access call to get all prevalue data for the data types that make up all of these properties which we use
-            // below if any property requires tag support
-            var allPreValues = new Lazy<IEnumerable<DataTypePreValueDto>>(() =>
-            {
-                var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
+            //This retrieves all pre-values for all data types that are referenced for all property types
+            // that exist in the data set.
+            //Benchmarks show that eagerly loading these so that we can lazily read the property data
+            // below (with the use of Query intead of Fetch) go about 30% faster, so we'll eagerly load
+            // this now since we cannot execute another reader inside of reading the property data.
+            var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
 FROM cmsDataTypePreValues a
 WHERE EXISTS(
     SELECT DISTINCT b.id as preValIdInner
@@ -517,25 +528,78 @@ WHERE EXISTS(
     ON cmsPropertyType.contentTypeId = docData.contentType
     WHERE a.id = b.id)", docSql.Arguments);
 
-                return Database.Fetch<DataTypePreValueDto>(preValsSql);
-            });
+            var allPreValues = Database.Fetch<DataTypePreValueDto>(preValsSql);
+
+            //It's Important with the sort order here! We require this to be sorted by node id,
+            // this is required because this data set can be huge depending on the page size. Due
+            // to it's size we need to be smart about iterating over the property values to build
+            // the document. Before we used to use Linq to get the property data for a given content node
+            // and perform a Distinct() call. This kills performance because that would mean if we had 7000 nodes
+            // and on each iteration we will perform a lookup on potentially 100,000 property rows against the node
+            // id which turns out to be a crazy amount of iterations. Instead we know it's sorted by this value we'll
+            // keep an index stored of the rows being read so we never have to re-iterate the entire data set
+            // on each document iteration.
+            var propSql = new Sql(@"SELECT cmsPropertyData.*
+FROM cmsPropertyData
+INNER JOIN cmsPropertyType
+ON cmsPropertyData.propertytypeid = cmsPropertyType.id
+INNER JOIN 
+	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
+ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
+ORDER BY contentNodeId, propertytypeid
+", docSql.Arguments);
+
+            //This does NOT fetch all data into memory in a list, this will read
+            // over the records as a data reader, this is much better for performance and memory,
+            // but it means that during the reading of this data set, nothing else can be read
+            // from SQL server otherwise we'll get an exception.
+            var allPropertyData = Database.Query<PropertyDataDto>(propSql);
 
             var result = new Dictionary<int, PropertyCollection>();
-
             var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
+            //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
+            var resolvedCompositionProperties = new Dictionary<int, PropertyType[]>();
 
-            //iterate each definition grouped by it's content type - this will mean less property type iterations while building 
-            // up the property collections
-            foreach (var compositionGroup in documentDefs.GroupBy(x => x.Composition))
+            //keep track of the current property data item being enumerated
+            var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
+            var hasCurrent = false; // initially there is no enumerator.Current
+
+            try
             {
-                var compositionProperties = compositionGroup.Key.CompositionPropertyTypes.ToArray();
-
-                foreach (var def in compositionGroup)
+                //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
+                // which allows us to more efficiently iterate over the large data set of property values
+                foreach (var def in documentDefs.OrderBy(x => x.Id))
                 {
-                    var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct();
+                    // get the resolved properties from our local cache, or resolve them and put them in cache
+                    PropertyType[] compositionProperties;
+                    if (resolvedCompositionProperties.ContainsKey(def.Composition.Id))
+                    {
+                        compositionProperties = resolvedCompositionProperties[def.Composition.Id];
+                    }
+                    else
+                    {
+                        compositionProperties = def.Composition.CompositionPropertyTypes.ToArray();
+                        resolvedCompositionProperties[def.Composition.Id] = compositionProperties;
+                    }
 
-                    var propertyFactory = new PropertyFactory(compositionProperties, def.Version, def.Id, def.CreateDate, def.VersionDate);
-                    var properties = propertyFactory.BuildEntity(propertyDataDtos.ToArray()).ToArray();
+                    // assemble the dtos for this def
+                    // use the available enumerator.Current if any else move to next
+                    var propertyDataDtos = new List<PropertyDataDto>();
+                    while (hasCurrent || propertyDataSetEnumerator.MoveNext())
+                    {
+                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        {
+                            hasCurrent = false; // enumerator.Current is not available
+                            propertyDataDtos.Add(propertyDataSetEnumerator.Current);
+                        }
+                        else
+                        {
+                            hasCurrent = true; // enumerator.Current is available for another def
+                            break; // no more propertyDataDto for this def
+                        }
+                    }
+
+                    var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray();
 
                     foreach (var property in properties)
                     {
@@ -552,7 +616,7 @@ WHERE EXISTS(
                             propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport;
 
                             //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up
-                            var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
+                            var preValData = allPreValues.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
                                 .Distinct()
                                 .ToArray();
 
@@ -575,8 +639,13 @@ WHERE EXISTS(
                     result[def.Id] = new PropertyCollection(properties);
                 }
             }
+            finally
+            {
+                propertyDataSetEnumerator.Dispose();
+            }
 
             return result;
+
         }
 
         public class DocumentDefinition
@@ -688,5 +757,12 @@ WHERE EXISTS(
 
             return allsuccess;
         }
+
+        /// <summary>
+        /// For Paging, repositories must support returning different query for the query type specified
+        /// </summary>
+        /// <param name="queryType"></param>
+        /// <returns></returns>
+        protected abstract Sql GetBaseQuery(BaseQueryType queryType);
     }
 }

src/Umbraco.Core/Services/ContentService.cs

@@ -585,7 +585,7 @@ namespace Umbraco.Core.Services
 
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
-        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             long total;
             var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter);
@@ -604,7 +604,7 @@ namespace Umbraco.Core.Services
         /// <param name="orderDirection">Direction to order by</param>
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns> 
-        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter);
         }
@@ -1764,6 +1764,31 @@ namespace Umbraco.Core.Services
             return true;
         }
 
+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords)
+        {
+            Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
+            Mandate.ParameterCondition(pageSize > 0, "pageSize");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateContentRepository(uow))
+            {
+                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize,
+                    //This order by is VERY important! This allows us to figure out what is implicitly not published, see ContentRepository.BuildXmlCache and
+                    // UmbracoContentIndexer.PerformIndexAll which uses the logic based on this sort order
+                    new[] {"level", "parentID", "sortOrder"},
+                    out totalRecords);
+                return contents;
+            }
+        }
+
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -2192,6 +2217,11 @@ namespace Umbraco.Core.Services
                 }
             }
 
+            if (string.IsNullOrWhiteSpace(content.Name))
+            {
+                throw new ArgumentException("Cannot save content with empty name.");
+            }
+
             using (new WriteLock(Locker))
             {
                 var uow = UowProvider.GetUnitOfWork();

src/Umbraco.Core/Services/ContentTypeService.cs

@@ -719,8 +719,13 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Optional id of the user saving the ContentType</param>
         public void Save(IContentType contentType, int userId = 0)
         {
-	        if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
-				return;
+            if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
+                return;
+
+            if (string.IsNullOrWhiteSpace(contentType.Name))
+            {
+                throw new ArgumentException("Cannot save content type with empty name.");
+            }
 
             using (new WriteLock(Locker))
             {

src/Umbraco.Core/Services/DataTypeService.cs

@@ -361,8 +361,13 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Id of the user issueing the save</param>
         public void Save(IDataTypeDefinition dataTypeDefinition, int userId = 0)
         {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
-				return;
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
+                return;
+
+            if (string.IsNullOrWhiteSpace(dataTypeDefinition.Name))
+            {
+                throw new ArgumentException("Cannot save datatype with empty name.");
+            }
 
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateDataTypeDefinitionRepository(uow))

src/Umbraco.Core/Services/IContentService.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Xml;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -95,6 +96,19 @@ namespace Umbraco.Core.Services
     /// </summary>
     public interface IContentService : IService
     {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table based on the given path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        /// <remarks>
+        /// If -1 is passed, then this will return all content xml entries, otherwise will return all descendents from the path
+        /// </remarks>
+        IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords);
+
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -245,7 +259,7 @@ namespace Umbraco.Core.Services
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IContent"/> objects by Parent Id
@@ -259,7 +273,7 @@ namespace Umbraco.Core.Services
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IContent"/> objects by Parent Id

src/Umbraco.Core/Services/IMediaService.cs

@@ -169,7 +169,7 @@ namespace Umbraco.Core.Services
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
@@ -183,7 +183,7 @@ namespace Umbraco.Core.Services
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IMedia"/> objects by Parent Id

src/Umbraco.Core/Services/IMemberService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -13,6 +14,15 @@ namespace Umbraco.Core.Services
     /// </summary>
     public interface IMemberService : IMembershipMemberService
     {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table
+        /// </summary>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>        
+        IEnumerable<XElement> GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords);
+
         /// <summary>
         /// Rebuilds all xml content in the cmsContentXml table for all documents
         /// </summary>

src/Umbraco.Core/Services/MacroService.cs

@@ -142,19 +142,24 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Optional Id of the user deleting the macro</param>
         public void Save(IMacro macro, int userId = 0)
         {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this)) 
-				return;
-	        
-			var uow = UowProvider.GetUnitOfWork();
-	        using (var repository = RepositoryFactory.CreateMacroRepository(uow))
-	        {
-		        repository.AddOrUpdate(macro);
-		        uow.Commit();
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this))
+                return;
 
-		        Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
-	        }
+            if (string.IsNullOrWhiteSpace(macro.Name))
+            {
+                throw new ArgumentException("Cannot save macro with empty name.");
+            }
 
-	        Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMacroRepository(uow))
+            {
+                repository.AddOrUpdate(macro);
+                uow.Commit();
+
+                Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
+            }
+
+            Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
         }
 
         ///// <summary>

src/Umbraco.Core/Services/MediaService.cs

@@ -451,7 +451,7 @@ namespace Umbraco.Core.Services
 
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
-        public IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             long total;
             var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter);
@@ -470,7 +470,7 @@ namespace Umbraco.Core.Services
         /// <param name="orderDirection">Direction to order by</param>
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
-        public IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter);
         }
@@ -837,7 +837,11 @@ namespace Umbraco.Core.Services
                 {
                     return OperationStatus.Cancelled(evtMsgs);
                 }
+            }
 
+            if (string.IsNullOrWhiteSpace(media.Name))
+            {
+                throw new ArgumentException("Cannot save media with empty name.");
             }
 
             var uow = UowProvider.GetUnitOfWork();
@@ -1224,7 +1228,7 @@ namespace Umbraco.Core.Services
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateMediaRepository(uow))
             {
-                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, out totalRecords);
+                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, null, out totalRecords);
                 return contents;
             }
         }

src/Umbraco.Core/Services/MemberService.cs

@@ -612,6 +612,26 @@ namespace Umbraco.Core.Services
             Audit(AuditType.Publish, "MemberService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
         }
 
+        /// <summary>
+        /// Gets paged member descendants as XML by path
+        /// </summary>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of member items</returns>
+        public IEnumerable<XElement> GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords)
+        {
+            Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
+            Mandate.ParameterCondition(pageSize > 0, "pageSize");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMemberRepository(uow))
+            {
+                var contents = repository.GetPagedXmlEntriesByPath("-1", pageIndex, pageSize, null, out totalRecords);
+                return contents;
+            }
+        }
+
         #endregion
 
         #region IMembershipMemberService Implementation
@@ -978,6 +998,11 @@ namespace Umbraco.Core.Services
                 }
             }
 
+            if (string.IsNullOrWhiteSpace(entity.Name))
+            {
+                throw new ArgumentException("Cannot save member with empty name.");
+            }
+
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateMemberRepository(uow))
             {

src/Umbraco.Core/Services/MemberTypeService.cs

@@ -78,6 +78,11 @@ namespace Umbraco.Core.Services
             if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMemberType>(memberType), this))
                 return;
 
+            if (string.IsNullOrWhiteSpace(memberType.Name))
+            {
+                throw new ArgumentException("Cannot save MemberType with empty name.");
+            }
+
             using (new WriteLock(Locker))
             {
                 var uow = UowProvider.GetUnitOfWork();

src/Umbraco.Core/Services/UserService.cs

@@ -125,6 +125,11 @@ namespace Umbraco.Core.Services
         {
             if (userType == null) throw new ArgumentNullException("userType");
 
+            if (string.IsNullOrWhiteSpace(username))
+            {
+                throw new ArgumentException("Cannot create user with empty username.");
+            }
+
             //TODO: PUT lock here!!
 
             var uow = UowProvider.GetUnitOfWork();
@@ -312,6 +317,15 @@ namespace Umbraco.Core.Services
                     return;
             }
 
+            if (string.IsNullOrWhiteSpace(entity.Username))
+            {
+                throw new ArgumentException("Cannot save user with empty username.");
+            }
+            if (string.IsNullOrWhiteSpace(entity.Name))
+            {
+                throw new ArgumentException("Cannot save user with empty name.");
+            }
+
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateUserRepository(uow))
             {
@@ -353,6 +367,14 @@ namespace Umbraco.Core.Services
             {
                 foreach (var member in entities)
                 {
+                    if (string.IsNullOrWhiteSpace(member.Username))
+                    {
+                        throw new ArgumentException("Cannot save user with empty username.");
+                    }
+                    if (string.IsNullOrWhiteSpace(member.Name))
+                    {
+                        throw new ArgumentException("Cannot save user with empty name.");
+                    }
                     repository.AddOrUpdate(member);
                 }
                 //commit the whole lot in one go

src/Umbraco.Core/Umbraco.Core.csproj

@@ -49,8 +49,9 @@
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
     </Reference>
-    <Reference Include="ImageProcessor, Version=2.5.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\ImageProcessor.2.5.1\lib\net45\ImageProcessor.dll</HintPath>
+    <Reference Include="ImageProcessor, Version=2.5.2.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\ImageProcessor.2.5.2\lib\net45\ImageProcessor.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="log4net, Version=1.2.11.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\log4net-mediumtrust.2.0.0\lib\log4net.dll</HintPath>
@@ -466,6 +467,7 @@
     <Compile Include="Persistence\RecordPersistenceType.cs" />
     <Compile Include="Persistence\Relators\AccessRulesRelator.cs" />
     <Compile Include="Persistence\Repositories\AuditRepository.cs" />
+    <Compile Include="Persistence\Repositories\BaseQueryType.cs" />
     <Compile Include="Persistence\Repositories\EntityContainerRepository.cs" />
     <Compile Include="Persistence\Repositories\DomainRepository.cs" />
     <Compile Include="Persistence\Repositories\ExternalLoginRepository.cs" />

src/Umbraco.Core/packages.config

@@ -2,7 +2,7 @@
 <packages>
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
   <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net45" />
-  <package id="ImageProcessor" version="2.5.1" targetFramework="net45" />
+  <package id="ImageProcessor" version="2.5.2" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Microsoft.AspNet.Identity.Core" version="2.2.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.Identity.Owin" version="2.2.1" targetFramework="net45" />

src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs

@@ -67,6 +67,66 @@ namespace Umbraco.Tests.Persistence.Repositories
             return repository;
         }
 
+        /// <summary>
+        /// This tests the regression issue of U4-9438
+        /// </summary>
+        /// <remarks>
+        /// The problem was the iteration of the property data in VersionableRepositoryBase when a content item
+        /// in the list actually doesn't have any property types, it would still skip over a property row.
+        /// To test, we have 3 content items, the first has properties, the second doesn't and the third does.
+        /// </remarks>
+        [Test]
+        public void Property_Data_Assigned_Correctly()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+
+            var allContent = new List<IContent>();
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var emptyContentType = MockedContentTypes.CreateBasicContentType();
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                var content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+                var content2 = MockedContent.CreateBasicContent(emptyContentType);
+                var content3 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                // Act
+                contentTypeRepository.AddOrUpdate(emptyContentType);
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                repository.AddOrUpdate(content2);
+                repository.AddOrUpdate(content3);
+                unitOfWork.Commit();
+
+                allContent.Add(content1);
+                allContent.Add(content2);
+                allContent.Add(content3);
+            }
+
+            // Assert
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                //this will cause the GetPropertyCollection to execute and we need to ensure that 
+                // all of the properties and property types are all correct
+                var result = repository.GetAll(allContent.Select(x => x.Id).ToArray()).ToArray();
+
+                foreach (var content in result)
+                {
+                    foreach (var contentProperty in content.Properties)
+                    {
+                        //prior to the fix, the 2nd document iteration in the GetPropertyCollection would have caused
+                        //the enumerator to move forward past the first property of the 3rd document which would have 
+                        //ended up not assiging a property to the 3rd document. This would have ended up with the 3rd
+                        //document still having 3 properties but the last one would not have been assigned an identity
+                        //because the property data would not have been assigned.
+                        Assert.IsTrue(contentProperty.HasIdentity);
+                    }
+                }
+            }
+        }
+
         [Test]
         public void Rebuild_Xml_Structures_With_Non_Latest_Version()
         {

src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs

@@ -172,6 +172,38 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Can_Get_All_Containers()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            EntityContainer container1, container2, container3;
+            using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid))
+            {
+                container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container1" };
+                containerRepository.AddOrUpdate(container1);
+                container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container2" };
+                containerRepository.AddOrUpdate(container2);
+                container3 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container3" };
+                containerRepository.AddOrUpdate(container3);
+                unitOfWork.Commit();
+                Assert.That(container1.Id, Is.GreaterThan(0));
+                Assert.That(container2.Id, Is.GreaterThan(0));
+                Assert.That(container3.Id, Is.GreaterThan(0));
+            }
+            using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid))
+            {
+                var found1 = containerRepository.Get(container1.Id);
+                Assert.IsNotNull(found1);
+                var found2 = containerRepository.Get(container2.Id);
+                Assert.IsNotNull(found2);
+                var found3 = containerRepository.Get(container3.Id);
+                Assert.IsNotNull(found3);
+                var allContainers = containerRepository.GetAll();
+                Assert.AreEqual(3, allContainers.Count());
+            }
+        }
+
         [Test]
         public void Can_Delete_Container()
         {

src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
+using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Documents;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using Moq;
 using NUnit.Framework;
@@ -31,12 +33,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Children_Are_Sorted()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
 
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
                 Assert.IsNotNull(result);
                 Assert.AreEqual(1, result.TotalItemCount);
@@ -60,12 +65,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Result_Has_All_Values()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
 
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
                 Assert.IsNotNull(result);
                 Assert.AreEqual(1, result.TotalItemCount);

src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs

@@ -27,6 +27,7 @@ using UmbracoExamine;
 using UmbracoExamine.DataServices;
 using umbraco.BusinessLogic;
 using System.Linq;
+using Lucene.Net.Index;
 
 namespace Umbraco.Tests.PublishedContent
 {
@@ -106,11 +107,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Children_Sorted_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -135,11 +139,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Do_Not_Find_In_Recycle_Bin()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -175,11 +182,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Children_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -197,11 +207,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Descendants_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -219,11 +232,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void DescendantsOrSelf_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -241,12 +257,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ancestors_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
                 var ctx = GetUmbracoContext("/test", 1234);
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
                 //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace
@@ -260,12 +279,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void AncestorsOrSelf_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
                 var ctx = GetUmbracoContext("/test", 1234);
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
                 //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace

src/Umbraco.Tests/Services/ContentServiceTests.cs

@@ -668,6 +668,17 @@ namespace Umbraco.Tests.Services
             Assert.Throws<Exception>(() => contentService.CreateContent("Test", -1, "umbAliasDoesntExist"));
         }
 
+        [Test]
+        public void Cannot_Save_Content_With_Empty_Name()
+        {
+            // Arrange
+            var contentService = ServiceContext.ContentService;
+            var content = new Content(string.Empty, -1, ServiceContext.ContentTypeService.GetContentType("umbTextpage"));
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => contentService.Save(content));
+        }
+
         [Test]
         public void Can_Get_Content_By_Id()
         {

src/Umbraco.Tests/Services/ContentTypeServiceTests.cs

@@ -304,6 +304,42 @@ namespace Umbraco.Tests.Services
             Assert.IsNull(deletedContentType);
         }
 
+        [Test]
+        public void Can_Create_Container()
+        {
+            // Arrange
+            var cts = ServiceContext.ContentTypeService;
+
+            // Act
+            var container = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container.Name = "container1";
+            cts.SaveContentTypeContainer(container);
+
+            // Assert
+            var createdContainer = cts.GetContentTypeContainer(container.Id);
+            Assert.IsNotNull(createdContainer);
+        }
+
+        [Test]
+        public void Can_Get_All_Containers()
+        {
+            // Arrange
+            var cts = ServiceContext.ContentTypeService;
+
+            // Act
+            var container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container1.Name = "container1";
+            cts.SaveContentTypeContainer(container1);
+
+            var container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container2.Name = "container2";
+            cts.SaveContentTypeContainer(container2);
+
+            // Assert
+            var containers = cts.GetContentTypeContainers(new int[0]);
+            Assert.AreEqual(2, containers.Count());
+        }
+
         [Test]
         public void Deleting_ContentType_Sends_Correct_Number_Of_DeletedEntities_In_Events()
         {
@@ -717,6 +753,16 @@ namespace Umbraco.Tests.Services
             Assert.DoesNotThrow(() => service.GetContentType("advancedPage"));
         }
 
+        [Test]
+        public void Cannot_Save_ContentType_With_Empty_Name()
+        {
+            // Arrange
+            var contentType = MockedContentTypes.CreateSimpleContentType("contentType", string.Empty);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.ContentTypeService.Save(contentType));
+        }
+
         [Test]
         public void Cannot_Rename_PropertyType_Alias_On_Composition_Which_Would_Cause_Conflict_In_Other_Composition()
         {

src/Umbraco.Tests/Services/DataTypeServiceTests.cs

@@ -221,5 +221,18 @@ namespace Umbraco.Tests.Services
             Assert.AreEqual("preVal1", preVals.PreValuesAsArray.First().Value);
             Assert.AreEqual("preVal2", preVals.PreValuesAsArray.Last().Value);            
         }
+
+        [Test]
+        public void Cannot_Save_DataType_With_Empty_Name()
+        {
+            // Arrange
+            var dataTypeService = ServiceContext.DataTypeService;
+
+            // Act
+            var dataTypeDefinition = new DataTypeDefinition(-1, "Test.TestEditor") { Name = string.Empty, DatabaseType = DataTypeDatabaseType.Ntext };
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => dataTypeService.Save(dataTypeDefinition));
+        }
     }
 }

src/Umbraco.Tests/Services/MacroServiceTests.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System;
+using System.Linq;
 using Moq;
 using NUnit.Framework;
 using Umbraco.Core;
@@ -217,6 +218,17 @@ namespace Umbraco.Tests.Services
 
         }
 
+        [Test]
+        public void Cannot_Save_Macro_With_Empty_Name()
+        {
+            // Arrange
+            var macroService = ServiceContext.MacroService;
+            var macro = new Macro("test", string.Empty, scriptPath: "~/Views/MacroPartials/Test.cshtml", cacheDuration: 1234);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => macroService.Save(macro));
+        }
+
         //[Test]
         //public void Can_Get_Many_By_Alias()
         //{

src/Umbraco.Tests/Services/MediaServiceTests.cs

@@ -81,6 +81,19 @@ namespace Umbraco.Tests.Services
             Assert.That(mediaChild.Trashed, Is.False);
         }
 
+        [Test]
+        public void Cannot_Save_Media_With_Empty_Name()
+        {
+            // Arrange
+            var mediaService = ServiceContext.MediaService;
+            var mediaType = MockedContentTypes.CreateVideoMediaType();
+            ServiceContext.ContentTypeService.Save(mediaType);
+            var media = mediaService.CreateMedia(string.Empty, -1, "video");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => mediaService.Save(media));
+        }
+
         [Test]
         public void Ensure_Content_Xml_Created()
         {

src/Umbraco.Tests/Services/MemberServiceTests.cs

@@ -185,6 +185,18 @@ namespace Umbraco.Tests.Services
             Assert.AreEqual(2, membersInRole.Count());
         }
 
+        [Test]
+        public void Cannot_Save_Member_With_Empty_Name()
+        {
+            IMemberType memberType = MockedContentTypes.CreateSimpleMemberType();
+            ServiceContext.MemberTypeService.Save(memberType);
+            IMember member = MockedMember.CreateSimpleMember(memberType, string.Empty, "test@test.com", "pass", "test");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.MemberService.Save(member));
+            
+        }
+
         [TestCase("MyTestRole1", "test1", StringPropertyMatchType.StartsWith, 1)]
         [TestCase("MyTestRole1", "test", StringPropertyMatchType.StartsWith, 3)]
         [TestCase("MyTestRole1", "test1", StringPropertyMatchType.Exact, 1)]

src/Umbraco.Tests/Services/MemberTypeServiceTests.cs

@@ -173,6 +173,16 @@ namespace Umbraco.Tests.Services
             }
         }
 
+        [Test]
+        public void Cannot_Save_MemberType_With_Empty_Name()
+        {
+            // Arrange
+            IMemberType memberType = MockedContentTypes.CreateSimpleMemberType("memberTypeAlias", string.Empty);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.MemberTypeService.Save(memberType));
+        }
+
         //[Test]
         //public void Can_Save_MemberType_Structure_And_Create_A_Member_Based_On_It()
         //{

src/Umbraco.Tests/Services/UserServiceTests.cs

@@ -490,6 +490,43 @@ namespace Umbraco.Tests.Services
             Assert.IsTrue(result4.AllowedSections.Contains("test"));
         }
 
+        [Test]
+        public void Cannot_Create_User_With_Empty_Username()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.CreateUserWithIdentity(string.Empty, "john@umbraco.io", userType));
+        }
+
+        [Test]
+        public void Cannot_Save_User_With_Empty_Username()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+            var user = userService.CreateUserWithIdentity("John Doe", "john@umbraco.io", userType);
+            user.Username = string.Empty;
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.Save(user));
+        }
+
+        [Test]
+        public void Cannot_Save_User_With_Empty_Name()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+            var user = userService.CreateUserWithIdentity("John Doe", "john@umbraco.io", userType);
+            user.Name = string.Empty;
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.Save(user));
+        }
+
         [Test]
         public void Get_By_Profile_Username()
         {

src/Umbraco.Tests/TestHelpers/Entities/MockedContent.cs

@@ -6,9 +6,18 @@ namespace Umbraco.Tests.TestHelpers.Entities
 {
     public class MockedContent
     {
+        public static Content CreateBasicContent(IContentType contentType)
+        {
+            var content = new Content("Home", -1, contentType) { Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
+           
+            content.ResetDirtyProperties(false);
+
+            return content;
+        }
+
         public static Content CreateSimpleContent(IContentType contentType)
         {
-            var content = new Content("Home", -1, contentType) { Language = "en-US", Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
+            var content = new Content("Home", -1, contentType) { Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
             object obj =
                 new
                     {
@@ -26,7 +35,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
 
         public static Content CreateSimpleContent(IContentType contentType, string name, int parentId)
         {
-            var content = new Content(name, parentId, contentType) { Language = "en-US", CreatorId = 0, WriterId = 0 };
+            var content = new Content(name, parentId, contentType) { CreatorId = 0, WriterId = 0 };
             object obj =
                 new
                 {
@@ -44,7 +53,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
 
 		public static Content CreateSimpleContent(IContentType contentType, string name, IContent parent)
 		{
-			var content = new Content(name, parent, contentType) { Language = "en-US", CreatorId = 0, WriterId = 0 };
+			var content = new Content(name, parent, contentType) { CreatorId = 0, WriterId = 0 };
 			object obj =
 				new
 				{
@@ -62,7 +71,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
 
         public static Content CreateTextpageContent(IContentType contentType, string name, int parentId)
         {
-            var content = new Content(name, parentId, contentType) { Language = "en-US", CreatorId = 0, WriterId = 0};
+            var content = new Content(name, parentId, contentType) { CreatorId = 0, WriterId = 0};
             object obj =
                 new
                 {
@@ -81,7 +90,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
 
         public static Content CreateSimpleContentWithSpecialDatabaseTypes(IContentType contentType, string name, int parentId, string decimalValue, string intValue, DateTime datetimeValue)
         {
-            var content = new Content(name, parentId, contentType) { Language = "en-US", CreatorId = 0, WriterId = 0 };
+            var content = new Content(name, parentId, contentType) { CreatorId = 0, WriterId = 0 };
             object obj = new
             {
                 decimalProperty = decimalValue,
@@ -96,7 +105,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
 
         public static Content CreateAllTypesContent(IContentType contentType, string name, int parentId)
         {
-            var content = new Content("Random Content Name", parentId, contentType) { Language = "en-US", Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
+            var content = new Content("Random Content Name", parentId, contentType) { Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
 
             content.SetValue("isTrue", true);
             content.SetValue("number", 42);
@@ -130,7 +139,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
             for (int i = 0; i < amount; i++)
             {
                 var name = "Textpage No-" + i;
-                var content = new Content(name, parentId, contentType) { Language = "en-US", CreatorId = 0, WriterId = 0 };
+                var content = new Content(name, parentId, contentType) { CreatorId = 0, WriterId = 0 };
                 object obj =
                     new
                     {

src/Umbraco.Tests/TestHelpers/Entities/MockedContentTypes.cs

@@ -7,6 +7,13 @@ namespace Umbraco.Tests.TestHelpers.Entities
 {
     public class MockedContentTypes
     {
+        /// <summary>
+        /// Creates a content type without any properties
+        /// </summary>
+        /// <param name="alias"></param>
+        /// <param name="name"></param>
+        /// <param name="parent"></param>
+        /// <returns></returns>
         public static ContentType CreateBasicContentType(string alias = "basePage", string name = "Base Page",
             ContentType parent = null)
         {

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -57,8 +57,8 @@
     <Reference Include="AutoMapper.Net4, Version=3.3.1.0, Culture=neutral, PublicKeyToken=be96cd2c38ef1005, processorArchitecture=MSIL">
       <HintPath>..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.81.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.81\lib\net45\Examine.dll</HintPath>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
@@ -163,6 +163,8 @@
     <Compile Include="Routing\NiceUrlRoutesTests.cs" />
     <Compile Include="TestHelpers\Entities\MockedPropertyTypes.cs" />
     <Compile Include="TryConvertToTests.cs" />
+    <Compile Include="UmbracoExamine\RandomIdRAMDirectory.cs" />
+    <Compile Include="UmbracoExamine\UmbracoContentIndexerTests.cs" />
     <Compile Include="Web\AngularIntegration\AngularAntiForgeryTests.cs" />
     <Compile Include="Web\AngularIntegration\ContentModelSerializationTests.cs" />
     <Compile Include="Web\AngularIntegration\JsInitializationTests.cs" />

src/Umbraco.Tests/UmbracoExamine/EventsTest.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
 using Examine;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using NUnit.Framework;
 using Umbraco.Tests.TestHelpers;
@@ -15,53 +17,39 @@ namespace Umbraco.Tests.UmbracoExamine
 		[Test]
 		public void Events_Ignoring_Node()
 		{
-			//change the parent id so that they are all ignored
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				999); //change to 999
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+		    using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+		    {
+                //change the parent id so that they are all ignored
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    999); //change to 999
 
-			var isIgnored = false;
+                var isIgnored = false;
 
-			EventHandler<IndexingNodeDataEventArgs> ignoringNode = (s, e) =>
-				{
-					isIgnored = true;
-				};
+                EventHandler<IndexingNodeDataEventArgs> ignoringNode = (s, e) =>
+                {
+                    isIgnored = true;
+                };
 
-			_indexer.IgnoringNode += ignoringNode;
+                indexer.IgnoringNode += ignoringNode;
 
-			//get a node from the data repo
-			var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
-			                          .Root
-			                          .Elements()
-			                          .First();
+                //get a node from the data repo
+                var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
+                                          .Root
+                                          .Elements()
+                                          .First();
 
-			_indexer.ReIndexNode(node, IndexTypes.Content);
+                indexer.ReIndexNode(node, IndexTypes.Content);
 
 
-			Assert.IsTrue(isIgnored);
-
+                Assert.IsTrue(isIgnored);
+            }
 		}
 
 		private readonly TestContentService _contentService = new TestContentService();
-		private static UmbracoExamineSearcher _searcher;
-		private static UmbracoContentIndexer _indexer;
-		private Lucene.Net.Store.Directory _luceneDir;
-
-	    public override void Initialize()
-	    {
-	        base.Initialize();
-
-            _luceneDir = new RAMDirectory();
-            _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
-            _indexer.RebuildIndex();
-            _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
-        }
-
-	    public override void TearDown()
-	    {
-	        base.TearDown();
-            _luceneDir.Dispose();
-        }
         
 	}
 }

src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs

@@ -7,6 +7,7 @@ using Examine.LuceneEngine.Config;
 using Examine.LuceneEngine.Providers;
 using Lucene.Net.Analysis;
 using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using Moq;
 using Umbraco.Core;
@@ -33,7 +34,7 @@ namespace Umbraco.Tests.UmbracoExamine
     internal static class IndexInitializer
     {
         public static UmbracoContentIndexer GetUmbracoIndexer(
-                Directory luceneDir,
+                IndexWriter writer,
                 Analyzer analyzer = null,
                 IDataService dataService = null,
                 IContentService contentService = null,
@@ -41,7 +42,8 @@ namespace Umbraco.Tests.UmbracoExamine
                 IDataTypeService dataTypeService = null,
                 IMemberService memberService = null,
                 IUserService userService = null,
-                IContentTypeService contentTypeService = null)
+                IContentTypeService contentTypeService = null, 
+                bool supportUnpublishedContent = false)
         {
             if (dataService == null)
             {
@@ -177,15 +179,17 @@ namespace Umbraco.Tests.UmbracoExamine
             var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies);
 
             var i = new UmbracoContentIndexer(indexCriteria,
-                                              luceneDir, //custom lucene directory
-                                                  dataService,
-                                                  contentService,
-                                                  mediaService,
-                                                  dataTypeService,
-                                                  userService,
-                                                  contentTypeService,
-                                              analyzer,
-                                              false);
+                writer, 
+                dataService,
+                contentService,
+                mediaService,
+                dataTypeService,
+                userService,
+                contentTypeService,
+                false)
+            {
+                SupportUnpublishedContent = supportUnpublishedContent
+            };
 
             //i.IndexSecondsInterval = 1;
 
@@ -193,13 +197,14 @@ namespace Umbraco.Tests.UmbracoExamine
 
             return i;
         }
-        public static UmbracoExamineSearcher GetUmbracoSearcher(Directory luceneDir, Analyzer analyzer = null)
+
+        public static UmbracoExamineSearcher GetUmbracoSearcher(IndexWriter writer, Analyzer analyzer = null)
         {
             if (analyzer == null)
             {
                 analyzer = new StandardAnalyzer(Version.LUCENE_29);
             }
-            return new UmbracoExamineSearcher(luceneDir, analyzer);
+            return new UmbracoExamineSearcher(writer, analyzer);
         }
 
         public static LuceneSearcher GetLuceneSearcher(Directory luceneDir)

src/Umbraco.Tests/UmbracoExamine/IndexTest.cs

@@ -1,11 +1,11 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using Examine;
 using Examine.LuceneEngine;
 using Examine.LuceneEngine.Providers;
 using Examine.LuceneEngine.SearchCriteria;
 using Examine.SearchCriteria;
+using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Index;
 using Lucene.Net.Search;
 using Lucene.Net.Store;
@@ -15,218 +15,230 @@ using UmbracoExamine;
 
 namespace Umbraco.Tests.UmbracoExamine
 {
-
     /// <summary>
     /// Tests the standard indexing capabilities
     /// </summary>
     [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
     [TestFixture, RequiresSTA]
     public class IndexTest : ExamineBaseTest
-	{
+    {
+        /// <summary>
+        /// Check that the node signalled as protected in the content service is not present in the index.
+        /// </summary>
+        [Test]
+        public void Index_Protected_Content_Not_Indexed()
+        {
 
-		///// <summary>
-		/// <summary>
-		/// Check that the node signalled as protected in the content service is not present in the index.
-		/// </summary>
-		[Test]
-		public void Index_Protected_Content_Not_Indexed()
-		{
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			var protectedQuery = new BooleanQuery();
-			protectedQuery.Add(
-				new BooleanClause(
-					new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)),
-					BooleanClause.Occur.MUST));
+                var protectedQuery = new BooleanQuery();
+                protectedQuery.Add(
+                    new BooleanClause(
+                        new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)),
+                        BooleanClause.Occur.MUST));
 
-			protectedQuery.Add(
-				new BooleanClause(
-					new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, TestContentService.ProtectedNode.ToString())),
-					BooleanClause.Occur.MUST));
+                protectedQuery.Add(
+                    new BooleanClause(
+                        new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, TestContentService.ProtectedNode.ToString())),
+                        BooleanClause.Occur.MUST));
 
-			var collector = new AllHitsCollector(false, true);
-			var s = _searcher.GetSearcher();
-			s.Search(protectedQuery, collector);
+                var collector = new AllHitsCollector(false, true);
+                var s = searcher.GetSearcher();
+                s.Search(protectedQuery, collector);
 
-			Assert.AreEqual(0, collector.Count, "Protected node should not be indexed");
+                Assert.AreEqual(0, collector.Count, "Protected node should not be indexed");
+            }
 
-		}
+        }
 
-		[Test]
-		public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
-		{
-			//change parent id to 1116
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				1116);
-			
-			//rebuild so it excludes children unless they are under 1116
-			_indexer.RebuildIndex();
+        [Test]
+        public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//ensure that node 2112 doesn't exist
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(0, results.Count());
+                var mediaService = new TestMediaService();
 
-			//get a node from the data repo (this one exists underneath 2222)
-			var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
-			                        .Root
-			                        .Elements()
-			                        .Where(x => (int)x.Attribute("id") == 2112)
-			                        .First();
+                //change parent id to 1116
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    1116);
 
-            var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
-            Assert.AreEqual("-1,1111,2222,2112", currPath);
+                //rebuild so it excludes children unless they are under 1116
+                indexer.RebuildIndex();
 
-			//now mimic moving 2112 to 1116
-			//node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
-            node.SetAttributeValue("path", "-1,1116,2112");
-			node.SetAttributeValue("parentID", "1116");
+                //ensure that node 2112 doesn't exist
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
 
-			//now reindex the node, this should first delete it and then WILL add it because of the parent id constraint
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                //get a node from the data repo (this one exists underneath 2222)
+                var node = mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
+                                        .Root
+                                        .Elements()
+                                        .First(x => (int)x.Attribute("id") == 2112);
 
-			//RESET the parent id
-			existingCriteria = ((IndexCriteria)_indexer.IndexerData);
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				null);
+                var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
+                Assert.AreEqual("-1,1111,2222,2112", currPath);
 
-			//now ensure it's deleted
-			var newResults = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(1, newResults.Count());
-		}
+                //now mimic moving 2112 to 1116
+                //node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
+                node.SetAttributeValue("path", "-1,1116,2112");
+                node.SetAttributeValue("parentID", "1116");
 
-		[Test]
-        [Ignore]
-		public void Index_Move_Media_To_Non_Indexable_ParentID()
-		{
-			//get a node from the data repo (this one exists underneath 2222)
-			var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
-			                        .Root
-			                        .Elements()
-			                        .Where(x => (int)x.Attribute("id") == 2112)
-			                        .First();
+                //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint
+                indexer.ReIndexNode(node, IndexTypes.Media);
 
-            var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
-            Assert.AreEqual("-1,1111,2222,2112", currPath);
+                //RESET the parent id
+                existingCriteria = ((IndexCriteria)indexer.IndexerData);
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    null);
 
-			//ensure it's indexed
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                //now ensure it's deleted
+                var newResults = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(1, newResults.TotalItemCount);
+            }
 
-			//change the parent node id to be the one it used to exist under
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				2222);
+            
+        }
 
-			//now mimic moving the node underneath 1116 instead of 2222
-			node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
-			node.SetAttributeValue("parentID", "1116");
+        [Test]
+        public void Index_Move_Media_To_Non_Indexable_ParentID()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//now reindex the node, this should first delete it and then NOT add it because of the parent id constraint
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                var mediaService = new TestMediaService();
 
-			//RESET the parent id
-			existingCriteria = ((IndexCriteria)_indexer.IndexerData);
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				null);
+                //get a node from the data repo (this one exists underneath 2222)
+                var node = mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
+                                    .Root
+                                    .Elements()
+                                    .First(x => (int)x.Attribute("id") == 2112);
 
-			//now ensure it's deleted
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(0, results.Count());
+                var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
+                Assert.AreEqual("-1,1111,2222,2112", currPath);
 
-		}
+                //ensure it's indexed
+                indexer.ReIndexNode(node, IndexTypes.Media);
+
+                //change the parent node id to be the one it used to exist under
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    2222);
+
+                //now mimic moving the node underneath 1116 instead of 2222
+                node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
+                node.SetAttributeValue("parentID", "1116");
+
+                //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint
+                indexer.ReIndexNode(node, IndexTypes.Media);
+
+                //RESET the parent id
+                existingCriteria = ((IndexCriteria)indexer.IndexerData);
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    null);
+
+                //now ensure it's deleted
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
+            }
+        }
 
 
-		/// <summary>
-		/// This will ensure that all 'Content' (not media) is cleared from the index using the Lucene API directly.
-		/// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly.
-		/// </summary>
-		[Test]
-		public void Index_Reindex_Content()
-		{
-			var s = (IndexSearcher)_searcher.GetSearcher();
+        /// <summary>
+        /// This will ensure that all 'Content' (not media) is cleared from the index using the Lucene API directly.
+        /// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly.
+        /// </summary>
+        [Test]
+        public void Index_Reindex_Content()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, supportUnpublishedContent: true))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine!
-			
-			var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content);
-		    var writer = _indexer.GetIndexWriter();
-            writer.DeleteDocuments(contentTerm);
-            writer.Commit();
-			
+                var s = (IndexSearcher)searcher.GetSearcher();
 
-			//make sure the content is gone. This is done with lucene APIs, not examine!
-			var collector = new AllHitsCollector(false, true);
-			var query = new TermQuery(contentTerm);
-			s = (IndexSearcher)_searcher.GetSearcher(); //make sure the searcher is up do date.
-			s.Search(query, collector);
-			Assert.AreEqual(0, collector.Count);
+                //first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine!
 
-			//call our indexing methods
-			_indexer.IndexAll(IndexTypes.Content);
+                var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content);
+                writer.DeleteDocuments(contentTerm);
+                writer.Commit();
 
-			collector = new AllHitsCollector(false, true);
-			s = (IndexSearcher)_searcher.GetSearcher(); //make sure the searcher is up do date.
-			s.Search(query, collector);
-            //var ids = new List<string>();
-            //for (var i = 0; i < collector.Count;i++)
-            //{
-            //    ids.Add(s.Doc(collector.GetDocId(i)).GetValues("__NodeId")[0]);
-            //}
-			Assert.AreEqual(20, collector.Count);
-		}
+                //make sure the content is gone. This is done with lucene APIs, not examine!
+                var collector = new AllHitsCollector(false, true);
+                var query = new TermQuery(contentTerm);
+                s = (IndexSearcher)searcher.GetSearcher(); //make sure the searcher is up do date.
+                s.Search(query, collector);
+                Assert.AreEqual(0, collector.Count);
 
-		/// <summary>
-		/// This will delete an item from the index and ensure that all children of the node are deleted too!
-		/// </summary>
-		[Test]
-        [Ignore]
-		public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
-		{
+                //call our indexing methods
+                indexer.IndexAll(IndexTypes.Content);
 
-			//now delete a node that has children
+                collector = new AllHitsCollector(false, true);
+                s = (IndexSearcher)searcher.GetSearcher(); //make sure the searcher is up do date.
+                s.Search(query, collector);
+                //var ids = new List<string>();
+                //for (var i = 0; i < collector.Count;i++)
+                //{
+                //    ids.Add(s.Doc(collector.GetDocId(i)).GetValues("__NodeId")[0]);
+                //}
+                Assert.AreEqual(21, collector.Count);
+            }
+        }
 
-			_indexer.DeleteFromIndex(1140.ToString());
-			//this node had children: 1141 & 1142, let's ensure they are also removed
+        /// <summary>
+        /// This will delete an item from the index and ensure that all children of the node are deleted too!
+        /// </summary>
+        [Test]
+        public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
+        {
 
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(1141).Compile());
-			Assert.AreEqual(0, results.Count());
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			results = _searcher.Search(_searcher.CreateSearchCriteria().Id(1142).Compile());
-			Assert.AreEqual(0, results.Count());
+                //now delete a node that has children
 
-		}
+                indexer.DeleteFromIndex(1140.ToString());
+                //this node had children: 1141 & 1142, let's ensure they are also removed
 
-		#region Private methods and properties
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
 
-		private readonly TestContentService _contentService = new TestContentService();
-		private readonly TestMediaService _mediaService = new TestMediaService();
-
-		private static UmbracoExamineSearcher _searcher;
-		private static UmbracoContentIndexer _indexer;
-
-		#endregion
-
-		#region Initialize and Cleanup
-
-		private Lucene.Net.Store.Directory _luceneDir;
-
-	    public override void TearDown()
-	    {
-	        base.TearDown();
-            _luceneDir.Dispose();
+                results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
+            }
+        }
+        
+        #region Initialize and Cleanup
+            
+        public override void TearDown()
+        {
+            base.TearDown();
+        
             UmbracoExamineSearcher.DisableInitializationCheck = null;
             BaseUmbracoIndexer.DisableInitializationCheck = null;
         }
         
-
-	    public override void Initialize()
-	    {
-	        base.Initialize();
-            _luceneDir = new RAMDirectory();
-            _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
-            _indexer.RebuildIndex();
-            _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
-        }
-        
-
-		#endregion
-	}
+        #endregion
+    }
 }

src/Umbraco.Tests/UmbracoExamine/RandomIdRAMDirectory.cs

@@ -0,0 +1,14 @@
+using System;
+using Lucene.Net.Store;
+
+namespace Umbraco.Tests.UmbracoExamine
+{
+    public class RandomIdRAMDirectory : RAMDirectory
+    {
+        private readonly string _lockId = Guid.NewGuid().ToString();
+        public override string GetLockID()
+        {
+            return _lockId;
+        }
+    }
+}

src/Umbraco.Tests/UmbracoExamine/SearchTests.cs

@@ -8,6 +8,8 @@ using Examine.LuceneEngine.Providers;
 using Lucene.Net.Store;
 using NUnit.Framework;
 using Examine.LuceneEngine.SearchCriteria;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Umbraco.Tests.TestHelpers;
 
 namespace Umbraco.Tests.UmbracoExamine
@@ -20,19 +22,17 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Test_Sort_Order_Sorting()
         {
-            //var newIndexFolder = new DirectoryInfo(Path.Combine("App_Data\\SearchTests", Guid.NewGuid().ToString()));
-            //System.IO.Directory.CreateDirectory(newIndexFolder.FullName);
-
-            using (var luceneDir = new RAMDirectory())
-            //using (var luceneDir = new SimpleFSDirectory(newIndexFolder))
-            {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, null,
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, null,
                     new TestDataService()
-                        {
-                            ContentService = new TestContentService(TestFiles.umbraco_sort)
-                        });
-                indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                    {
+                        ContentService = new TestContentService(TestFiles.umbraco_sort)
+                    },
+                    supportUnpublishedContent: true))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {                
+                indexer.RebuildIndex();                
 
                 var s = (LuceneSearcher)searcher;
                 var luceneSearcher = s.GetSearcher();
@@ -69,25 +69,7 @@ namespace Umbraco.Tests.UmbracoExamine
                 currentSort = sort;
             }
             return true;
-        }
-
-        //[Test]
-        //public void Test_Index_Type_With_German_Analyzer()
-        //{
-        //    using (var luceneDir = new RAMDirectory())
-        //    {
-        //        var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir,
-        //            new GermanAnalyzer());
-        //        indexer.RebuildIndex();
-        //        var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);    
-        //    }
-        //}
-
-        //private readonly TestContentService _contentService = new TestContentService();
-        //private readonly TestMediaService _mediaService = new TestMediaService();
-        //private static UmbracoExamineSearcher _searcher;
-        //private static UmbracoContentIndexer _indexer;
-        //private Lucene.Net.Store.Directory _luceneDir;
+        }        
 
     }
 }

src/Umbraco.Tests/UmbracoExamine/UmbracoContentIndexerTests.cs

@@ -0,0 +1,107 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Xml.Linq;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using UmbracoExamine;
+
+namespace Umbraco.Tests.UmbracoExamine
+{
+    [TestFixture]
+    public class UmbracoContentIndexerTests : ExamineBaseTest
+    {
+        [Test]
+        public void Get_Serialized_Content_No_Published_Content()
+        {
+            var contentSet = new List<IContent>
+            {
+                Mock.Of<IContent>(c => c.Id == 1 && c.Path == "-1,1" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 2 && c.Path == "-1,2" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 3 && c.Path == "-1,3" && c.Published == false && c.Level == 1),      // no
+                Mock.Of<IContent>(c => c.Id == 4 && c.Path == "-1,4" && c.Published == false && c.Level == 1),      // no
+
+                Mock.Of<IContent>(c => c.Id == 5 && c.Path == "-1,1,5" && c.Published && c.Level == 2),
+                Mock.Of<IContent>(c => c.Id == 6 && c.Path == "-1,2,6" && c.Published == false && c.Level == 2),    // no
+                Mock.Of<IContent>(c => c.Id == 7 && c.Path == "-1,3,7" && c.Published && c.Level == 2),             // no
+                Mock.Of<IContent>(c => c.Id == 8 && c.Path == "-1,4,8" && c.Published && c.Level == 2),             // no
+                Mock.Of<IContent>(c => c.Id == 9 && c.Path == "-1,4,9" && c.Published && c.Level == 2),             // no
+
+                Mock.Of<IContent>(c => c.Id == 10 && c.Path == "-1,1,5,10" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 15 && c.Path == "-1,1,5,15" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 11 && c.Path == "-1,2,6,11" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 16 && c.Path == "-1,2,6,16" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 12 && c.Path == "-1,3,7,12" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 17 && c.Path == "-1,3,7,17" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 13 && c.Path == "-1,4,8,13" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 18 && c.Path == "-1,4,8,18" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 14 && c.Path == "-1,4,9,14" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 19 && c.Path == "-1,4,9,19" && c.Published && c.Level == 3),         // no
+            };
+
+            //ensure the rest of the required values are populted
+            foreach (var content in contentSet)
+            {
+                var mock = Mock.Get(content);
+                mock.Setup(x => x.ContentType).Returns(Mock.Of<IContentType>(type => type.Icon == "hello"));
+            }
+
+            contentSet.Sort((a, b) => Comparer<int>.Default.Compare(a.Level, b.Level));
+
+            var published = new HashSet<string>();
+
+            var result = UmbracoContentIndexer.GetSerializedContent(false, content => new XElement("test"), contentSet, published)
+                .WhereNotNull()
+                .ToArray();
+
+            Assert.AreEqual(5, result.Length);
+        }
+
+        [Test]
+        public void Get_Serialized_Content_With_Published_Content()
+        {
+            var contentSet = new List<IContent>
+            {
+                Mock.Of<IContent>(c => c.Id == 1 && c.Path == "-1,1" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 2 && c.Path == "-1,2" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 3 && c.Path == "-1,3" && c.Published == false && c.Level == 1),      
+                Mock.Of<IContent>(c => c.Id == 4 && c.Path == "-1,4" && c.Published == false && c.Level == 1),      
+
+                Mock.Of<IContent>(c => c.Id == 5 && c.Path == "-1,1,5" && c.Published && c.Level == 2),
+                Mock.Of<IContent>(c => c.Id == 6 && c.Path == "-1,2,6" && c.Published == false && c.Level == 2),    
+                Mock.Of<IContent>(c => c.Id == 7 && c.Path == "-1,3,7" && c.Published && c.Level == 2),             
+                Mock.Of<IContent>(c => c.Id == 8 && c.Path == "-1,4,8" && c.Published && c.Level == 2),             
+                Mock.Of<IContent>(c => c.Id == 9 && c.Path == "-1,4,9" && c.Published && c.Level == 2),             
+
+                Mock.Of<IContent>(c => c.Id == 10 && c.Path == "-1,1,5,10" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 15 && c.Path == "-1,1,5,15" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 11 && c.Path == "-1,2,6,11" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 16 && c.Path == "-1,2,6,16" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 12 && c.Path == "-1,3,7,12" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 17 && c.Path == "-1,3,7,17" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 13 && c.Path == "-1,4,8,13" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 18 && c.Path == "-1,4,8,18" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 14 && c.Path == "-1,4,9,14" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 19 && c.Path == "-1,4,9,19" && c.Published && c.Level == 3),          
+            };
+
+            //ensure the rest of the required values are populted
+            foreach (var content in contentSet)
+            {
+                var mock = Mock.Get(content);
+                mock.Setup(x => x.ContentType).Returns(Mock.Of<IContentType>(type => type.Icon == "hello"));
+            }
+
+            contentSet.Sort((a, b) => Comparer<int>.Default.Compare(a.Level, b.Level));
+
+            var published = new HashSet<string>();
+
+            var result = UmbracoContentIndexer.GetSerializedContent(true, content => new XElement("test"), contentSet, published)
+                .WhereNotNull()
+                .ToArray();
+
+            Assert.AreEqual(19, result.Length);
+        }
+    }
+}

src/Umbraco.Tests/packages.config

@@ -2,7 +2,7 @@
 <packages>
   <package id="AspNetWebApi.SelfHost" version="4.0.20710.0" targetFramework="net45" />
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.81" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.Mvc" version="5.2.3" targetFramework="net45" />

src/Umbraco.Web.UI.Client/src/common/services/user.service.js

@@ -204,7 +204,7 @@ angular.module('umbraco.services')
                         //when it's successful, return the user data
                         setCurrentUser(data);
 
-                        var result = { user: data, authenticated: true, lastUserId: lastUserId };
+                        var result = { user: data, authenticated: true, lastUserId: lastUserId, loginType: "credentials" };
 
                         //broadcast a global event
                         eventsService.emit("app.authenticated", result);
@@ -232,7 +232,7 @@ angular.module('umbraco.services')
                     authResource.getCurrentUser()
                         .then(function (data) {
 
-                            var result = { user: data, authenticated: true, lastUserId: lastUserId };
+                            var result = { user: data, authenticated: true, lastUserId: lastUserId, loginType: "implicit" };
 
                             //TODO: This is a mega backwards compatibility hack... These variables SHOULD NOT exist in the server variables
                             // since they are not supposed to be dynamic but I accidentally added them there in 7.1.5 IIRC so some people might

src/Umbraco.Web.UI.Client/src/controllers/main.controller.js

@@ -8,7 +8,7 @@
  * The main application controller
  * 
  */
-function MainController($scope, $rootScope, $location, $routeParams, $timeout, $http, $log, appState, treeService, notificationsService, userService, navigationService, historyService, updateChecker, assetsService, eventsService, umbRequestHelper, tmhDynamicLocale) {
+function MainController($scope, $rootScope, $location, $routeParams, $timeout, $http, $log, appState, treeService, notificationsService, userService, navigationService, historyService, updateChecker, assetsService, eventsService, umbRequestHelper, tmhDynamicLocale, localStorageService) {
 
     //the null is important because we do an explicit bool check on this in the view
     //the avatar is by default the umbraco logo    
@@ -81,6 +81,14 @@ function MainController($scope, $rootScope, $location, $routeParams, $timeout, $
             $location.path("/").search("");
             historyService.removeAll();
             treeService.clearCache();
+
+            //if the user changed, clearout local storage too - could contain sensitive data
+            localStorageService.clearAll();
+        }
+
+        //if this is a new login (i.e. the user entered credentials), then clear out local storage - could contain sensitive data
+        if (data.loginType === "credentials") {            
+            localStorageService.clearAll();
         }
 
         //Load locale file

src/Umbraco.Web.UI.Client/src/init.js

@@ -1,6 +1,6 @@
 /** Executed when the application starts, binds to events and set global state */
-app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache',
-    function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache) {
+app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache', 'localStorageService',
+    function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache, localStorageService) {
 
         //This sets the default jquery ajax headers to include our csrf token, we
         // need to user the beforeSend method because our token changes per user/login so
@@ -13,6 +13,7 @@ app.run(['userService', '$log', '$rootScope', '$location', 'navigationService',
 
         /** Listens for authentication and checks if our required assets are loaded, if/once they are we'll broadcast a ready event */
         eventsService.on("app.authenticated", function(evt, data) {
+            
             assetsService._loadInitAssets().then(function() {
                 appState.setGlobalState("isReady", true);
 

src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js

@@ -1,7 +1,7 @@
 //used for the media picker dialog
 angular.module("umbraco")
     .controller("Umbraco.Overlays.MediaPickerController",
-        function($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, $cookieStore, localizationService) {
+        function ($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, localStorageService, localizationService) {
 
             if (!$scope.model.title) {
                 $scope.model.title = localizationService.localize("defaultdialogs_selectMedia");
@@ -15,7 +15,7 @@ angular.module("umbraco")
             $scope.multiPicker = (dialogOptions.multiPicker && dialogOptions.multiPicker !== "0") ? true : false;
             $scope.startNodeId = dialogOptions.startNodeId ? dialogOptions.startNodeId : -1;
             $scope.cropSize = dialogOptions.cropSize;
-            $scope.lastOpenedNode = $cookieStore.get("umbLastOpenedMediaNodeId");
+            $scope.lastOpenedNode = localStorageService.get("umbLastOpenedMediaNodeId");
             if ($scope.onlyImages) {
                 $scope.acceptedFileTypes = mediaHelper
                     .formatFileTypes(Umbraco.Sys.ServerVariables.umbracoSettings.imageFileTypes);
@@ -133,8 +133,7 @@ angular.module("umbraco")
                     });
                 $scope.currentFolder = folder;
 
-                // for some reason i cannot set cookies with cookieStore
-                document.cookie = "umbLastOpenedMediaNodeId=" + folder.id;
+                localStorageService.set("umbLastOpenedMediaNodeId", folder.id);
 
             };
 

src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js

@@ -34,18 +34,24 @@
                 fields: {},
                 file: file
             }).progress(function (evt) {
+                
+                // hack: in some browsers the progress event is called after success
+                // this prevents the UI from going back to a uploading state
+                if(vm.zipFile.uploadStatus !== "done" && vm.zipFile.uploadStatus !== "error") {
 
-                // set view state to uploading
-                vm.state = 'uploading';
+                    // set view state to uploading
+                    vm.state = 'uploading';
 
-                // calculate progress in percentage
-                var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10);
+                    // calculate progress in percentage
+                    var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10);
 
-                // set percentage property on file
-                vm.zipFile.uploadProgress = progressPercentage;
+                    // set percentage property on file
+                    vm.zipFile.uploadProgress = progressPercentage;
 
-                // set uploading status on file
-                vm.zipFile.uploadStatus = "uploading";
+                    // set uploading status on file
+                    vm.zipFile.uploadStatus = "uploading";
+
+                }
 
             }).success(function (data, status, headers, config) {
 

src/Umbraco.Web.UI.Client/src/views/packager/views/installed.html

@@ -12,8 +12,8 @@
                 <div class="umb-package-list__item" ng-repeat="installedPackage in vm.installedPackages">
 
                     <div class="umb-package-list__item-icon">
-                        <i ng-if="!installedPackage.icon" class="icon-box"></i>
-                        <img ng-if="installedPackage.icon" ng-src="{{installedPackage.icon}}" />
+                        <i ng-if="!installedPackage.iconUrl" class="icon-box"></i>
+                        <img ng-if="installedPackage.iconUrl" ng-src="{{installedPackage.iconUrl}}" />
                     </div>
 
                     <div class="umb-package-list__item-content">

src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js

@@ -2,7 +2,7 @@ angular.module("umbraco")
 .controller("Umbraco.PropertyEditors.GoogleMapsController",
     function ($element, $rootScope, $scope, notificationsService, dialogService, assetsService, $log, $timeout) {
 
-        assetsService.loadJs('http://www.google.com/jsapi')
+        assetsService.loadJs('https://www.google.com/jsapi')
             .then(function () {
                 google.load("maps", "3",
                             {

src/Umbraco.Web.UI/Umbraco.Web.UI.csproj

@@ -127,19 +127,20 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\packages\dotless.1.4.1.0\lib\dotless.Core.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
-      <Private>True</Private>
+    <Reference Include="Examine, Version=0.1.81.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.81\lib\net45\Examine.dll</HintPath>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
     </Reference>
-    <Reference Include="ImageProcessor, Version=2.5.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\ImageProcessor.2.5.1\lib\net45\ImageProcessor.dll</HintPath>
+    <Reference Include="ImageProcessor, Version=2.5.2.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\ImageProcessor.2.5.2\lib\net45\ImageProcessor.dll</HintPath>
+      <Private>True</Private>
     </Reference>
-    <Reference Include="ImageProcessor.Web, Version=4.7.2.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\ImageProcessor.Web.4.7.2\lib\net45\ImageProcessor.Web.dll</HintPath>
+    <Reference Include="ImageProcessor.Web, Version=4.8.2.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\ImageProcessor.Web.4.8.2\lib\net45\ImageProcessor.Web.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="log4net, Version=1.2.11.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -167,7 +168,8 @@
     </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="Microsoft.IO.RecyclableMemoryStream, Version=1.2.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\Microsoft.IO.RecyclableMemoryStream.1.2.0\lib\net45\Microsoft.IO.RecyclableMemoryStream.dll</HintPath>
+      <HintPath>..\packages\Microsoft.IO.RecyclableMemoryStream.1.2.1\lib\net45\Microsoft.IO.RecyclableMemoryStream.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Owin">
       <HintPath>..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll</HintPath>
@@ -2421,9 +2423,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\"
         <WebProjectProperties>
           <UseIIS>True</UseIIS>
           <AutoAssignPort>True</AutoAssignPort>
-          <DevelopmentServerPort>7570</DevelopmentServerPort>
+          <DevelopmentServerPort>7590</DevelopmentServerPort>
           <DevelopmentServerVPath>/</DevelopmentServerVPath>
-          <IISUrl>http://localhost:7570</IISUrl>
+          <IISUrl>http://localhost:7590</IISUrl>
           <NTLMAuthentication>False</NTLMAuthentication>
           <UseCustomServer>False</UseCustomServer>
           <CustomServerUrl>

src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx

@@ -13,6 +13,7 @@
     
    <umb:JsInclude ID="JsInclude1" runat="server" FilePath="Application/NamespaceManager.js" PathNameAlias="UmbracoClient" Priority="0" />
    <umb:JsInclude ID="JsInclude3" runat="server" FilePath="ui/jquery.js" PathNameAlias="UmbracoClient" Priority="1" />
+   <umb:JsInclude ID="JsInclude4" runat="server" FilePath="lib/jquery-migrate/jquery-migrate.min.js" PathNameAlias="UmbracoRoot" Priority="1" />
    <umb:JsInclude ID="JsInclude6" runat="server" FilePath="ui/base2.js" PathNameAlias="UmbracoClient" Priority="1" />
    <umb:JsInclude ID="JsInclude11" runat="server" FilePath="UI/knockout.js" PathNameAlias="UmbracoClient" Priority="3" />
    <umb:JsInclude ID="JsInclude12" runat="server" FilePath="UI/knockout.mapping.js" PathNameAlias="UmbracoClient" Priority="4" />

src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs

@@ -48,6 +48,15 @@ namespace Umbraco.Web.UI.Umbraco.Dashboard {
         /// </remarks>
         protected global::ClientDependency.Core.Controls.JsInclude JsInclude3;
         
+        /// <summary>
+        /// JsInclude4 control.
+        /// </summary>
+        /// <remarks>
+        /// Auto-generated field.
+        /// To modify move field declaration from designer file to code-behind file.
+        /// </remarks>
+        protected global::ClientDependency.Core.Controls.JsInclude JsInclude4;
+        
         /// <summary>
         /// JsInclude6 control.
         /// </summary>

src/Umbraco.Web.UI/config/imageprocessor/cache.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <caching currentCache="DiskCache">
   <caches>
-    <cache name="DiskCache" type="ImageProcessor.Web.Caching.DiskCache, ImageProcessor.Web" maxDays="365" browserMaxDays="7">
+    <cache name="DiskCache" type="ImageProcessor.Web.Caching.DiskCache, ImageProcessor.Web" maxDays="365" browserMaxDays="7" trimCache="false">
       <settings>
         <setting key="VirtualCachePath" value="~/app_data/cache" />
       </settings>

src/Umbraco.Web.UI/packages.config

@@ -4,9 +4,9 @@
   <package id="ClientDependency" version="1.9.2" targetFramework="net45" />
   <package id="ClientDependency-Mvc5" version="1.8.0.0" targetFramework="net45" />
   <package id="dotless" version="1.4.1.0" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
-  <package id="ImageProcessor" version="2.5.1" targetFramework="net45" />
-  <package id="ImageProcessor.Web" version="4.7.2" targetFramework="net45" />
+  <package id="Examine" version="0.1.81" targetFramework="net45" />
+  <package id="ImageProcessor" version="2.5.2" targetFramework="net45" />
+  <package id="ImageProcessor.Web" version="4.8.2" targetFramework="net45" />
   <package id="ImageProcessor.Web.Config" version="2.3.0" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
@@ -22,7 +22,7 @@
   <package id="Microsoft.CodeAnalysis.Analyzers" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.Common" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.CodeAnalysis.CSharp" version="1.0.0" targetFramework="net45" />
-  <package id="Microsoft.IO.RecyclableMemoryStream" version="1.2.0" targetFramework="net45" />
+  <package id="Microsoft.IO.RecyclableMemoryStream" version="1.2.1" targetFramework="net45" />
   <package id="Microsoft.Owin" version="3.0.1" targetFramework="net45" />
   <package id="Microsoft.Owin.Host.SystemWeb" version="3.0.1" targetFramework="net45" />
   <package id="Microsoft.Owin.Security" version="3.0.1" targetFramework="net45" />

src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml

@@ -1142,7 +1142,7 @@ To manage your website, simply open the Umbraco back office and start adding con
 
   <area alias="modelsBuilder">
       <key alias="buildingModels">Building models</key>
-      <key alias="waitingMessage">this can take abit of time, don't worry</key>
+      <key alias="waitingMessage">this can take a bit of time, don't worry</key>
       <key alias="modelsGenerated">Models generated</key>
       <key alias="modelsGeneratedError">Models could not be generated</key>
       <key alias="modelsExceptionInUlog">Models generation has failed, see exception in U log</key>

src/Umbraco.Web.UI/umbraco/config/lang/ru.xml

@@ -27,6 +27,9 @@
     <key alias="publish">Опубликовать</key>
     <key alias="refreshNode">Обновить узлы</key>
     <key alias="republish">Опубликовать весь сайт</key>
+    <key alias="SetPermissionsForThePage">Установить разрешения для страницы '%0%'</key>
+    <key alias="chooseWhereToMove">Выберите, куда переместить</key>
+    <key alias="toInTheTreeStructureBelow">В структуре документов ниже</key>
     <key alias="restore" version="7.3.0">Восстановить</key>
     <key alias="rights">Разрешения</key>
     <key alias="rollback">Откатить</key>
@@ -189,7 +192,7 @@
     <key alias="memberrole">Роль участника</key>
     <key alias="membertype">Тип участника</key>
     <key alias="noDate">Дата не указана</key>
-    <key alias="nodeName">Заголовок страницы</key>
+    <key alias="nodeName">Заголовок ссылки</key>
     <key alias="notmemberof">Не является членом групп(ы)</key>
     <key alias="otherElements">Свойства</key>
     <key alias="parentNotPublished">Этот документ опубликован, но скрыт, потому что его родительский документ '%0%' не опубликован</key>
@@ -341,9 +344,10 @@
     <key alias="viewCacheItem">Просмотр элемента кэша</key>
     <key alias="createFolder">Создать папку...</key>
     <key alias="relateToOriginalLabel">Связать с оригиналом</key>
+    <key alias="includeDescendants">Включая все дочерние</key>
     <key alias="theFriendliestCommunity">Самое дружелюбное сообщество</key>
     <key alias="linkToPage">Ссылка на страницу</key>
-    <key alias="openInNewWindow">Открывает документ по ссылке в новом окне или вкладке браузера</key>
+    <key alias="openInNewWindow">Открывать ссылку в новом окне или вкладке браузера</key>
     <key alias="linkToMedia">Ссылка на медиа-файл</key>
     <key alias="selectMedia">Выбрать медиа</key>
     <key alias="selectIcon">Выбрать значок</key>

src/Umbraco.Web/Cache/ContentTypeCacheRefresher.cs

@@ -131,9 +131,7 @@ namespace Umbraco.Web.Cache
             ClearAllIsolatedCacheByEntityType<IMediaType>();
             ClearAllIsolatedCacheByEntityType<IMember>();
             ClearAllIsolatedCacheByEntityType<IMemberType>();
-
-            //all property type cache
-            ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheByKeySearch(CacheKeys.PropertyTypeCacheKey);
+            
             //all content type property cache
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheByKeySearch(CacheKeys.ContentTypePropertiesCacheKey);     
             //all content type cache
@@ -266,12 +264,6 @@ namespace Umbraco.Web.Cache
         /// </returns>
         private static void ClearContentTypeCache(JsonPayload payload)
         {
-            //clears the cache for each property type associated with the content type
-            foreach (var pid in payload.PropertyTypeIds)
-            {
-                ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(CacheKeys.PropertyTypeCacheKey + pid);
-            }
-
             //clears the cache associated with the Content type itself
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(string.Format("{0}{1}", CacheKeys.ContentTypeCacheKey, payload.Id));
             //clears the cache associated with the content type properties collection

src/Umbraco.Web/Cache/PageCacheRefresher.cs

@@ -74,7 +74,7 @@ namespace Umbraco.Web.Cache
         public override void Remove(int id)
         {
             ApplicationContext.Current.ApplicationCache.ClearPartialViewCache();
-            content.Instance.ClearDocumentCache(id);
+            content.Instance.ClearDocumentCache(id, false);
             DistributedCache.Instance.ClearAllMacroCacheOnCurrentServer();
             DistributedCache.Instance.ClearXsltCacheOnCurrentServer();
             ClearAllIsolatedCacheByEntityType<PublicAccessEntry>();
@@ -95,7 +95,7 @@ namespace Umbraco.Web.Cache
         public override void Remove(IContent instance)
         {
             ApplicationContext.Current.ApplicationCache.ClearPartialViewCache();
-            content.Instance.ClearDocumentCache(new Document(instance));
+            content.Instance.ClearDocumentCache(new Document(instance), false);
             XmlPublishedContent.ClearRequest();
             DistributedCache.Instance.ClearAllMacroCacheOnCurrentServer();
             DistributedCache.Instance.ClearXsltCacheOnCurrentServer();

src/Umbraco.Web/Editors/EntityController.cs

@@ -614,7 +614,7 @@ namespace Umbraco.Web.Editors
                     .Select(Mapper.Map<EntityBasic>);
 
                 // entities are in "some" order, put them back in order
-                var xref = entities.ToDictionary(x => x.Id);
+                var xref = entities.ToDictionary(x => x.Key);
                 var result = keysArray.Select(x => xref.ContainsKey(x) ? xref[x] : null).Where(x => x != null);
 
                 return result;

src/Umbraco.Web/Editors/MacroController.cs

@@ -4,21 +4,26 @@ using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Web.Http;
+using System.Web.SessionState;
 using AutoMapper;
 using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Mvc;
 using umbraco;
+using Umbraco.Core;
 
 namespace Umbraco.Web.Editors
 {
     /// <summary>
     /// API controller to deal with Macro data
     /// </summary>
+    /// <remarks>
+    /// Note that this implements IRequiresSessionState which will enable HttpContext.Session - generally speaking we don't normally
+    /// enable this for webapi controllers, however since this controller is used to render macro content and macros can access
+    /// Session, we don't want it to throw null reference exceptions.
+    /// </remarks>
     [PluginController("UmbracoApi")]
-    public class MacroController : UmbracoAuthorizedJsonController
+    public class MacroController : UmbracoAuthorizedJsonController, IRequiresSessionState
     {
-        
-
         /// <summary>
         /// Gets the macro parameters to be filled in for a particular macro
         /// </summary>
@@ -124,6 +129,6 @@ namespace Umbraco.Web.Editors
                 "text/html");
             return result;
         }
-
+        
     }
 }

src/Umbraco.Web/Editors/MediaController.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.IO;
 using System.Net;
 using System.Net.Http;
@@ -27,6 +28,7 @@ using Umbraco.Web.Mvc;
 using Umbraco.Web.WebApi;
 using System.Linq;
 using System.Runtime.Serialization;
+using System.Web.Http.Controllers;
 using Umbraco.Web.WebApi.Binders;
 using Umbraco.Web.WebApi.Filters;
 using umbraco;
@@ -36,6 +38,7 @@ using Umbraco.Core.Configuration;
 using Umbraco.Core.Persistence.FaultHandling;
 using Umbraco.Web.UI;
 using Notification = Umbraco.Web.Models.ContentEditing.Notification;
+using Umbraco.Core.Persistence;
 
 namespace Umbraco.Web.Editors
 {
@@ -44,9 +47,22 @@ namespace Umbraco.Web.Editors
     /// access to ALL of the methods on this controller will need access to the media application.
     /// </remarks>
     [PluginController("UmbracoApi")]
-    [UmbracoApplicationAuthorizeAttribute(Constants.Applications.Media)]
+    [UmbracoApplicationAuthorize(Constants.Applications.Media)]
+    [MediaControllerControllerConfiguration]
     public class MediaController : ContentControllerBase
     {
+        /// <summary>
+        /// Configures this controller with a custom action selector
+        /// </summary>
+        private class MediaControllerControllerConfigurationAttribute : Attribute, IControllerConfiguration
+        {
+            public void Initialize(HttpControllerSettings controllerSettings, HttpControllerDescriptor controllerDescriptor)
+            {
+                controllerSettings.Services.Replace(typeof(IHttpActionSelector), new ParameterSwapControllerActionSelector(
+                    new ParameterSwapControllerActionSelector.ParameterSwapInfo("GetChildren", "id", typeof(int), typeof(Guid), typeof(string))));
+            }
+        }
+
         /// <summary>
         /// Constructor
         /// </summary>
@@ -172,7 +188,7 @@ namespace Umbraco.Web.Editors
         }
 
         /// <summary>
-        /// Returns the child media objects
+        /// Returns the child media objects - using the entity INT id
         /// </summary>
         [FilterAllowedOutgoingMedia(typeof(IEnumerable<ContentItemBasic<ContentPropertyBasic, IMedia>>), "Items")]
         public PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(int id,
@@ -209,6 +225,58 @@ namespace Umbraco.Web.Editors
             return pagedResult;
         }
 
+        /// <summary>
+        /// Returns the child media objects - using the entity GUID id
+        /// </summary>
+        /// <param name="id"></param>
+        /// <param name="pageNumber"></param>
+        /// <param name="pageSize"></param>
+        /// <param name="orderBy"></param>
+        /// <param name="orderDirection"></param>
+        /// <param name="orderBySystemField"></param>
+        /// <param name="filter"></param>
+        /// <returns></returns>
+        [FilterAllowedOutgoingMedia(typeof(IEnumerable<ContentItemBasic<ContentPropertyBasic, IMedia>>), "Items")]
+        public PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(Guid id,
+           int pageNumber = 0,
+           int pageSize = 0,
+           string orderBy = "SortOrder",
+           Direction orderDirection = Direction.Ascending,
+           bool orderBySystemField = true,
+           string filter = "")
+        {            
+            var entity = Services.EntityService.GetByKey(id);
+            if (entity != null)
+            {
+                return GetChildren(entity.Id, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter);
+            }
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+
+        [Obsolete("Do not use this method, use either the overload with INT or GUID instead, this will be removed in future versions")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)]
+        public PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(string id,
+           int pageNumber = 0,
+           int pageSize = 0,
+           string orderBy = "SortOrder",
+           Direction orderDirection = Direction.Ascending,
+           bool orderBySystemField = true,
+           string filter = "")
+        {
+            foreach (var type in new[] { typeof(int), typeof(Guid) })
+            {
+                var parsed = id.TryConvertTo(type);
+                if (parsed)
+                {
+                    //oooh magic! will auto select the right overload
+                    return GetChildren((dynamic)parsed.Result);
+                }
+            }
+
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+
         /// <summary>
         /// Moves an item to the recycle bin, if it is already there then it will permanently delete it
         /// </summary>
@@ -448,12 +516,37 @@ namespace Umbraco.Web.Editors
             }
 
             //get the string json from the request
-            int parentId;
-            if (int.TryParse(result.FormData["currentFolder"], out parentId) == false)
+            int parentId; bool entityFound;
+            string currentFolderId = result.FormData["currentFolder"];
+            if (int.TryParse(currentFolderId, out parentId) == false)
             {
-                return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer");
+                // if a guid then try to look up the entity
+                Guid idGuid;
+                if (Guid.TryParse(currentFolderId, out idGuid))
+                {
+                    var entity = Services.EntityService.GetByKey(idGuid);
+                    if (entity != null)
+                    {
+                        entityFound = true;
+                        parentId = entity.Id;
+                    }
+                    else
+                    {
+                        throw new EntityNotFoundException(currentFolderId, "The passed id doesn't exist");
+                    }
+                }
+                else
+                {
+                    return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid");
+                }
+
+                if (entityFound == false)
+                {
+                    return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid");
+                }
             }
 
+
             //ensure the user has access to this folder by parent id!
             if (CheckPermissions(
                new Dictionary<string, object>(),

src/Umbraco.Web/Editors/MediaTypeController.cs

@@ -11,6 +11,11 @@ using System.Net;
 using System.Net.Http;
 using Umbraco.Web.WebApi;
 using Umbraco.Core.Services;
+using Umbraco.Core.Models.EntityBase;
+using System;
+using System.ComponentModel;
+using System.Web.Http.Controllers;
+using Umbraco.Core;
 
 namespace Umbraco.Web.Editors
 {
@@ -24,8 +29,21 @@ namespace Umbraco.Web.Editors
     [PluginController("UmbracoApi")]
     [UmbracoTreeAuthorize(Constants.Trees.MediaTypes)]
     [EnableOverrideAuthorization]
+    [MediaTypeControllerControllerConfigurationAttribute]
     public class MediaTypeController : ContentTypeControllerBase
     {
+        /// <summary>
+        /// Configures this controller with a custom action selector
+        /// </summary>
+        private class MediaTypeControllerControllerConfigurationAttribute : Attribute, IControllerConfiguration
+        {
+            public void Initialize(HttpControllerSettings controllerSettings, HttpControllerDescriptor controllerDescriptor)
+            {
+                controllerSettings.Services.Replace(typeof(IHttpActionSelector), new ParameterSwapControllerActionSelector(
+                    new ParameterSwapControllerActionSelector.ParameterSwapInfo("GetAllowedChildren", "contentId", typeof(int), typeof(Guid), typeof(string))));
+            }
+        }
+
         /// <summary>
         /// Constructor
         /// </summary>
@@ -170,7 +188,7 @@ namespace Umbraco.Web.Editors
 
 
         /// <summary>
-        /// Returns the allowed child content type objects for the content item id passed in
+        /// Returns the allowed child content type objects for the content item id passed in - based on an INT id
         /// </summary>
         /// <param name="contentId"></param>
         [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)]
@@ -214,6 +232,40 @@ namespace Umbraco.Web.Editors
             return basics;
         }
 
+        /// <summary>
+        /// Returns the allowed child content type objects for the content item id passed in - based on a GUID id
+        /// </summary>
+        /// <param name="contentId"></param>
+        [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)]
+        public IEnumerable<ContentTypeBasic> GetAllowedChildren(Guid contentId)
+        {
+            var entity = ApplicationContext.Services.EntityService.GetByKey(contentId);
+            if (entity != null)
+            {
+                return GetAllowedChildren(entity.Id);
+            }
+
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+        
+        [Obsolete("Do not use this method, use either the overload with INT or GUID instead, this will be removed in future versions")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)]        
+        public IEnumerable<ContentTypeBasic> GetAllowedChildren(string contentId)
+        {
+            foreach (var type in new[] { typeof(int), typeof(Guid) })
+            {
+                var parsed = contentId.TryConvertTo(type);
+                if (parsed)
+                {
+                    //oooh magic! will auto select the right overload
+                    return GetAllowedChildren((dynamic)parsed.Result);
+                }                    
+            }
+
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+
         /// <summary>
         /// Move the media type
         /// </summary>

src/Umbraco.Web/Editors/ParameterSwapControllerActionSelector.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Linq;
+using System.Web;
+using System.Web.Http.Controllers;
+using Umbraco.Core;
+
+namespace Umbraco.Web.Editors
+{
+    /// <summary>
+    /// This is used to auto-select specific actions on controllers that would otherwise be ambiguous based on a single parameter type
+    /// </summary>
+    /// <remarks>
+    /// As an example, lets say we have 2 methods: GetChildren(int id) and GetChildren(Guid id), by default Web Api won't allow this since
+    /// it won't know what to select, but if this Tuple is passed in new Tuple{string, string}("GetChildren", "id")
+    /// </remarks>
+    internal class ParameterSwapControllerActionSelector : ApiControllerActionSelector
+    {
+        private readonly ParameterSwapInfo[] _actions;
+
+        /// <summary>
+        /// Constructor accepting a list of action name + parameter name
+        /// </summary>
+        /// <param name="actions"></param>
+        public ParameterSwapControllerActionSelector(params ParameterSwapInfo[] actions)
+        {
+            _actions = actions;
+        }
+        public override HttpActionDescriptor SelectAction(HttpControllerContext controllerContext)
+        {
+            var found = _actions.FirstOrDefault(x => controllerContext.Request.RequestUri.GetLeftPart(UriPartial.Path).InvariantEndsWith(x.ActionName));
+
+            if (found != null)
+            {
+                var id = HttpUtility.ParseQueryString(controllerContext.Request.RequestUri.Query).Get(found.ParamName);
+
+                if (id != null)
+                {
+                    var idTypes = found.SupportedTypes;
+
+                    foreach (var idType in idTypes)
+                    {
+                        var converted = id.TryConvertTo(idType);
+                        if (converted)
+                        {
+                            var method = MatchByType(idType, controllerContext, found);
+                            if (method != null)
+                                return method;
+                        }
+                    }                   
+                }
+            }
+            return base.SelectAction(controllerContext);
+        }
+
+        private static ReflectedHttpActionDescriptor MatchByType(Type idType, HttpControllerContext controllerContext, ParameterSwapInfo found)
+        {
+            var controllerType = controllerContext.Controller.GetType();
+            var methods = controllerType.GetMethods().Where(info => info.Name == found.ActionName).ToArray();
+            if (methods.Length > 1)
+            {
+                //choose the one that has the parameter with the T type
+                var method = methods.FirstOrDefault(x => x.GetParameters().FirstOrDefault(p => p.Name == found.ParamName && p.ParameterType == idType) != null);
+
+                return new ReflectedHttpActionDescriptor(controllerContext.ControllerDescriptor, method);
+            }
+            return null;
+        }
+
+        internal class ParameterSwapInfo
+        {
+            public string ActionName { get; private set; }
+            public string ParamName { get; private set; }
+            public Type[] SupportedTypes { get; private set; }
+
+            public ParameterSwapInfo(string actionName, string paramName, params Type[] supportedTypes)
+            {
+                ActionName = actionName;
+                ParamName = paramName;
+                SupportedTypes = supportedTypes;
+            }
+        }
+
+    }
+}

src/Umbraco.Web/ImageCropperTemplateExtensions.cs

@@ -81,7 +81,7 @@ namespace Umbraco.Web
         /// Use focal point, to generate an output image using the focal point instead of the predefined crop
         /// </param>
         /// <param name="useCropDimensions">
-        /// Use crop dimensions to have the output image sized according to the predefined crop sizes, this will override the width and height parameters>.
+        /// Use crop dimensions to have the output image sized according to the predefined crop sizes, this will override the width and height parameters.
         /// </param>
         /// <param name="cacheBuster">
         /// Add a serialised date of the last edit of the item to ensure client cache refresh when updated

src/Umbraco.Web/Install/InstallSteps/NewInstallStep.cs

@@ -81,7 +81,7 @@ namespace Umbraco.Web.Install.InstallSteps
                 {
                     var client = new System.Net.WebClient();
                     var values = new NameValueCollection { { "name", admin.Name }, { "email", admin.Email} };
-                    client.UploadValues("https://umbraco.com/base/Ecom/SubmitEmail/installer.aspx", values);
+                    client.UploadValues("https://shop.umbraco.com/base/Ecom/SubmitEmail/installer.aspx", values);
                 }
                 catch { /* fail in silence */ }
             }
@@ -147,4 +147,4 @@ namespace Umbraco.Web.Install.InstallSteps
             }
         }
     }
-}
+}

src/Umbraco.Web/Mvc/AreaRegistrationExtensions.cs

@@ -3,8 +3,10 @@ using System.Collections.Generic;
 using System.Web.Http;
 using System.Web.Mvc;
 using System.Web.Routing;
+using System.Web.SessionState;
 using Umbraco.Core;
 using Umbraco.Core.Configuration;
+using Umbraco.Web.WebApi;
 
 namespace Umbraco.Web.Mvc
 {
@@ -93,6 +95,13 @@ namespace Umbraco.Web.Mvc
                 }
                 //look in this namespace to create the controller
                 controllerPluginRoute.DataTokens.Add("Namespaces", new[] {controllerType.Namespace});
+
+                //Special case! Check if the controller type implements IRequiresSessionState and if so use our
+                //custom webapi session handler
+                if (typeof(IRequiresSessionState).IsAssignableFrom(controllerType))
+                {
+                    controllerPluginRoute.RouteHandler = new SessionHttpControllerRouteHandler();
+                }
             }
 
             //Don't look anywhere else except this namespace!
@@ -100,9 +109,9 @@ namespace Umbraco.Web.Mvc
 
             //constraints: only match controllers ending with 'controllerSuffixName' and only match this controller's ID for this route            
             if (controllerSuffixName.IsNullOrWhiteSpace() == false)
-            {                
-            controllerPluginRoute.Constraints = new RouteValueDictionary(
-                new Dictionary<string, object>
+            {
+                controllerPluginRoute.Constraints = new RouteValueDictionary(
+                    new Dictionary<string, object>
                     {
                         {"controller", @"(\w+)" + controllerSuffixName}
                     });

src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs

@@ -92,7 +92,12 @@ namespace Umbraco.Web.PropertyEditors
                             //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect 
                             // the website. 
                         }
+                        catch (ArgumentException)
+                        {
+                            //swallow on purpose to prevent this error:
+                            // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
 
+                        }
                     }
                 }
             }

src/Umbraco.Web/Trees/ContentTreeController.cs

@@ -202,11 +202,13 @@ namespace Umbraco.Web.Trees
         /// <returns></returns>
         protected override bool HasPathAccess(string id, FormDataCollection queryStrings)
         {
-            var content = Services.ContentService.GetById(int.Parse(id));
-            if (content == null)
+            var entity = GetEntityFromId(id);
+            if (entity == null)
             {
                 return false;
             }
+
+            IContent content = Services.ContentService.GetById(entity.Id);
             return Security.CurrentUser.HasPathAccess(content);
         }
 

src/Umbraco.Web/Trees/ContentTreeControllerBase.cs

@@ -96,7 +96,7 @@ namespace Umbraco.Web.Trees
                     LogHelper.Warn<ContentTreeControllerBase>("The user " + Security.CurrentUser.Username + " does not have access to the tree node " + id);
                     return new TreeNodeCollection();
                 }
-                
+
                 // So there's an alt id specified, it's not the root node and the user has access to it, great! But there's one thing we
                 // need to consider: 
                 // If the tree is being rendered in a dialog view we want to render only the children of the specified id, but
@@ -110,7 +110,7 @@ namespace Umbraco.Web.Trees
                     id = Constants.System.Root.ToString(CultureInfo.InvariantCulture);
                 }
             }
-            
+
             var entities = GetChildEntities(id);
             nodes.AddRange(entities.Select(entity => GetSingleTreeNode(entity, id, queryStrings)).Where(node => node != null));
             return nodes;
@@ -122,23 +122,34 @@ namespace Umbraco.Web.Trees
 
         protected IEnumerable<IUmbracoEntity> GetChildEntities(string id)
         {
+            // use helper method to ensure we support both integer and guid lookups
             int iid;
+
+            // look up from GUID if it's not an integer
             if (int.TryParse(id, out iid) == false)
             {
-                throw new InvalidCastException("The id for the media tree must be an integer");
+
+                var idEntity = GetEntityFromId(id);
+                if (idEntity == null)
+                {
+                    throw new HttpResponseException(HttpStatusCode.NotFound);
+                }
+                iid = idEntity.Id;
             }
 
+
             //if a request is made for the root node data but the user's start node is not the default, then
             // we need to return their start node data
             if (iid == Constants.System.Root && UserStartNode != Constants.System.Root)
             {
                 //just return their single start node, it will show up under the 'Content' label
                 var startNode = Services.EntityService.Get(UserStartNode, UmbracoObjectType);
-                if (startNode == null)
+                if (startNode != null)
+                    return new[] { startNode };
+                else
                 {
                     throw new EntityNotFoundException(UserStartNode, "User's start content node could not be found");
                 }
-                return new[] { startNode };
             }
 
             return Services.EntityService.GetChildren(iid, UmbracoObjectType).ToArray();
@@ -176,9 +187,9 @@ namespace Umbraco.Web.Trees
                 {
                     id = altStartId;
                 }
-                
+
                 var nodes = GetTreeNodesInternal(id, queryStrings);
-                
+
                 //only render the recycle bin if we are not in dialog and the start id id still the root
                 if (IsDialog(queryStrings) == false && id == Constants.System.Root.ToInvariantString())
                 {
@@ -210,8 +221,9 @@ namespace Umbraco.Web.Trees
         /// </remarks>
         private TreeNodeCollection GetTreeNodesInternal(string id, FormDataCollection queryStrings)
         {
+            IUmbracoEntity current = GetEntityFromId(id);
+
             //before we get the children we need to see if this is a container node
-            var current = Services.EntityService.Get(int.Parse(id), UmbracoObjectType);
 
             //test if the parent is a listview / container
             if (current != null && current.IsContainer())
@@ -286,5 +298,32 @@ namespace Umbraco.Web.Trees
         {
             return allowedUserOptions.Select(x => x.Action).OfType<ActionBrowse>().Any();
         }
+
+        /// <summary>
+        /// Get an entity via an id that can be either an integer or a Guid
+        /// </summary>
+        /// <param name="id"></param>
+        /// <returns></returns>
+        internal IUmbracoEntity GetEntityFromId(string id)
+        {
+            IUmbracoEntity entity;
+            Guid idGuid = Guid.Empty;
+            int idInt;
+            if (Guid.TryParse(id, out idGuid))
+            {
+                entity = Services.EntityService.GetByKey(idGuid, UmbracoObjectType);
+
+            }
+            else if (int.TryParse(id, out idInt))
+            {
+                entity = Services.EntityService.Get(idInt, UmbracoObjectType);
+            }
+            else
+            {
+                return null;
+            }
+
+            return entity;
+        }
     }
 }

src/Umbraco.Web/Umbraco.Web.csproj

@@ -112,8 +112,8 @@
     <Reference Include="dotless.Core, Version=1.4.1.0, Culture=neutral, PublicKeyToken=96b446c9e63eae34, processorArchitecture=MSIL">
       <HintPath>..\packages\dotless.1.4.1.0\lib\dotless.Core.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.81.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.81\lib\net45\Examine.dll</HintPath>
     </Reference>
     <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
       <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>
@@ -277,6 +277,7 @@
     <Compile Include="Editors\EditorValidationResolver.cs" />
     <Compile Include="Editors\EditorValidator.cs" />
     <Compile Include="Editors\GravatarController.cs" />
+    <Compile Include="Editors\ParameterSwapControllerActionSelector.cs" />
     <Compile Include="HealthCheck\Checks\Config\AbstractConfigCheck.cs" />
     <Compile Include="HealthCheck\Checks\Config\AcceptableConfiguration.cs" />
     <Compile Include="HealthCheck\Checks\Config\ConfigurationService.cs" />
@@ -955,6 +956,7 @@
     <Compile Include="WebApi\NamespaceHttpControllerSelector.cs" />
     <Compile Include="WebApi\PrefixlessBodyModelValidator.cs" />
     <Compile Include="WebApi\PrefixlessBodyModelValidatorAttribute.cs" />
+    <Compile Include="WebApi\SessionHttpControllerRouteHandler.cs" />
     <Compile Include="WebApi\UmbracoApiController.cs" />
     <Compile Include="WebApi\UmbracoApiControllerBase.cs" />
     <Compile Include="WebApi\UmbracoApiControllerResolver.cs" />

src/Umbraco.Web/WebApi/SessionHttpControllerRouteHandler.cs

@@ -0,0 +1,31 @@
+using System.Web;
+using System.Web.Http.WebHost;
+using System.Web.Routing;
+using System.Web.SessionState;
+
+namespace Umbraco.Web.WebApi
+{
+    /// <summary>
+    /// A custom WebApi route handler that enables session on the HttpContext - use with caution! 
+    /// </summary>
+    /// <remarks>
+    /// WebApi controllers (and REST in general) shouldn't have session state enabled since it's stateless,
+    /// enabling session state puts additional locks on requests so only use this when absolutley needed 
+    /// </remarks>
+    internal class SessionHttpControllerRouteHandler : HttpControllerRouteHandler
+    {
+        protected override IHttpHandler GetHttpHandler(RequestContext requestContext)
+        {
+            return new SessionHttpControllerHandler(requestContext.RouteData);
+        }
+
+        /// <summary>
+        /// A custom WebApi handler that enables session on the HttpContext
+        /// </summary>        
+        private class SessionHttpControllerHandler : HttpControllerHandler, IRequiresSessionState
+        {
+            public SessionHttpControllerHandler(RouteData routeData) : base(routeData)
+            { }
+        }
+    }
+}

src/Umbraco.Web/WebServices/ExamineManagementApiController.cs

@@ -166,6 +166,8 @@ namespace Umbraco.Web.WebServices
             var msg = ValidateLuceneIndexer(indexerName, out indexer);
             if (msg.IsSuccessStatusCode)
             {
+                LogHelper.Info<ExamineManagementApiController>(string.Format("Rebuilding index '{0}'", indexerName));
+
                 //remove it in case there's a handler there alraedy
                 indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
                 //now add a single handler
@@ -201,6 +203,8 @@ namespace Umbraco.Web.WebServices
             //ensure it's not listening anymore
             indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
 
+            LogHelper.Info<ExamineManagementApiController>(string.Format("Rebuilding index '{0}' done, {1} items committed (can differ from the number of items in the index)", indexer.Name, indexer.CommitCount));
+
             var cacheKey = "temp_indexing_op_" + indexer.Name;
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey);
         }
@@ -266,7 +270,10 @@ namespace Umbraco.Web.WebServices
                 var val = p.GetValue(indexer, null);
                 if (val == null)
                 {
-                    LogHelper.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name);
+                    // Do not warn for new new attribute that is optional
+                    if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false)
+                        LogHelper.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name);
+
                     val = string.Empty;
                 }
                 indexerModel.ProviderProperties.Add(p.Name, val.ToString());

src/Umbraco.Web/packages.config

@@ -3,7 +3,7 @@
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
   <package id="ClientDependency" version="1.9.2" targetFramework="net45" />
   <package id="dotless" version="1.4.1.0" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.81" targetFramework="net45" />
   <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Markdown" version="1.14.7" targetFramework="net45" />

src/Umbraco.Web/umbraco.presentation/content.cs

@@ -394,6 +394,11 @@ namespace umbraco
         }
 
         public virtual void ClearDocumentCache(int documentId)
+        {
+            ClearDocumentCache(documentId, true);
+        }
+
+        internal virtual void ClearDocumentCache(int documentId, bool removeDbXmlEntry)
         {
             // Get the document
             Document d;
@@ -408,7 +413,7 @@ namespace umbraco
                 ClearDocumentXmlCache(documentId);
                 return;
             }
-            ClearDocumentCache(d);
+            ClearDocumentCache(d, removeDbXmlEntry);
         }
 
         /// <summary>
@@ -416,7 +421,8 @@ namespace umbraco
         /// This means the node gets unpublished from the website.
         /// </summary>
         /// <param name="doc">The document</param>
-        internal void ClearDocumentCache(Document doc)
+        /// <param name="removeDbXmlEntry"></param>
+        internal void ClearDocumentCache(Document doc, bool removeDbXmlEntry)
         {
             var e = new DocumentCacheEventArgs();
             FireBeforeClearDocumentCache(doc, e);
@@ -425,8 +431,13 @@ namespace umbraco
             {
                 XmlNode x;
 
-                // remove from xml db cache
-                doc.XmlRemoveFromDB();
+                //Hack: this is here purely for backwards compat if someone for some reason is using the 
+                // ClearDocumentCache(int documentId) method and expecting it to remove the xml
+                if (removeDbXmlEntry)
+                {
+                    // remove from xml db cache
+                    doc.XmlRemoveFromDB();
+                }
 
                 // clear xml cache
                 ClearDocumentXmlCache(doc.Id);

src/Umbraco.Web/umbraco.presentation/umbraco/controls/ContentTypeControl.cs

@@ -2,6 +2,7 @@ using System;
 using System.Web.UI.HtmlControls;
 using System.Web.UI.WebControls;
 using System.Collections;
+using System.ComponentModel;
 using System.IO;
 using Umbraco.Core.IO;
 using System.Linq;
@@ -11,6 +12,8 @@ namespace umbraco.controls
 	/// <summary>
 	/// Summary description for ContentTypeControl.
 	/// </summary>
+	[Obsolete("No longer used, will be removed in v8")]
+    [EditorBrowsable(EditorBrowsableState.Never)]
 	public class ContentTypeControl : uicontrols.TabView
 	{
 		public event System.EventHandler OnSave;

src/Umbraco.Web/umbraco.presentation/umbraco/users/EditUser.aspx.cs

@@ -3,6 +3,7 @@ using System.Collections;
 using System.Configuration.Provider;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Web;
 using System.Web.Security;
 using System.Web.UI;
@@ -320,13 +321,14 @@ namespace umbraco.cms.presentation.user
             }
 
             // Populate dropdowns
-            foreach (DocumentType dt in DocumentType.GetAllAsList())
-                cDocumentType.Items.Add(
-                    new ListItem(dt.Text, dt.Alias)
-                    );
+            var allContentTypes = Services.ContentTypeService.GetAllContentTypes().ToList();
+            foreach (var dt in allContentTypes)
+            {
+                cDocumentType.Items.Add(new ListItem(dt.Name, dt.Alias));
+            }
 
             // populate fields
-            ArrayList fields = new ArrayList();
+            var fields = new ArrayList();
             cDescription.ID = "cDescription";
             cCategories.ID = "cCategories";
             cExcerpt.ID = "cExcerpt";
@@ -334,9 +336,9 @@ namespace umbraco.cms.presentation.user
             cCategories.Items.Add(new ListItem(ui.Text("choose"), ""));
             cExcerpt.Items.Add(new ListItem(ui.Text("choose"), ""));
 
-            foreach (PropertyType pt in PropertyType.GetAll())
+            foreach (var pt in allContentTypes.SelectMany(x => x.PropertyTypes).OrderBy(x => x.Name))
             {
-                if (!fields.Contains(pt.Alias))
+                if (fields.Contains(pt.Alias) == false)
                 {
                     cDescription.Items.Add(new ListItem(string.Format("{0} ({1})", pt.Name, pt.Alias), pt.Alias));
                     cCategories.Items.Add(new ListItem(string.Format("{0} ({1})", pt.Name, pt.Alias), pt.Alias));

src/UmbracoExamine/BaseUmbracoIndexer.cs

@@ -49,6 +49,7 @@ namespace UmbracoExamine
         /// <param name="indexPath"></param>
         /// <param name="dataService"></param>
         /// <param name="analyzer"></param>
+        /// <param name="async"></param>
         protected BaseUmbracoIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async)
             : base(indexerData, indexPath, analyzer, async)
         {
@@ -61,6 +62,19 @@ namespace UmbracoExamine
 			DataService = dataService;
 		}
 
+        /// <summary>
+        /// Creates an NRT indexer
+        /// </summary>
+        /// <param name="indexerData"></param>
+        /// <param name="writer"></param>
+        /// <param name="async"></param>
+        /// <param name="dataService"></param>
+        protected BaseUmbracoIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService, bool async) 
+            : base(indexerData, writer, async)
+        {
+            DataService = dataService;
+        }
+
         #endregion
 
         /// <summary>
@@ -95,7 +109,7 @@ namespace UmbracoExamine
         /// Determines if the manager will call the indexing methods when content is saved or deleted as
         /// opposed to cache being updated.
         /// </summary>
-        public bool SupportUnpublishedContent { get; protected set; }
+        public bool SupportUnpublishedContent { get; protected internal set; }
 
         /// <summary>
         /// The data service used for retreiving and submitting data to the cms

src/UmbracoExamine/Config/IndexSetExtensions.cs

@@ -14,7 +14,7 @@ namespace UmbracoExamine.Config
     public static class IndexSetExtensions
     {
         internal static IIndexCriteria ToIndexCriteria(this IndexSet set, IDataService svc,
-            IEnumerable<StaticField> indexFieldPolicies)
+            StaticFieldCollection indexFieldPolicies)
         {
             return new LazyIndexCriteria(set, svc, indexFieldPolicies);
         }
@@ -29,7 +29,7 @@ namespace UmbracoExamine.Config
         /// <returns></returns>
         public static IIndexCriteria ToIndexCriteria(this IndexSet set, IDataService svc)
         {
-            return set.ToIndexCriteria(svc, Enumerable.Empty<StaticField>());
+            return set.ToIndexCriteria(svc, new StaticFieldCollection());
         }      
       
     }

src/UmbracoExamine/Config/LazyIndexCriteria.cs

@@ -12,7 +12,7 @@ namespace UmbracoExamine.Config
         public LazyIndexCriteria(
             IndexSet set,
             IDataService svc,
-            IEnumerable<StaticField> indexFieldPolicies)
+            StaticFieldCollection indexFieldPolicies)
         {
             if (set == null) throw new ArgumentNullException("set");
             if (indexFieldPolicies == null) throw new ArgumentNullException("indexFieldPolicies");
@@ -35,8 +35,9 @@ namespace UmbracoExamine.Config
                     foreach (var u in userProps)
                     {
                         var field = new IndexField() { Name = u };
-                        var policy = indexFieldPolicies.FirstOrDefault(x => x.Name == u);
-                        if (policy != null)
+
+                        StaticField policy;
+                        if (indexFieldPolicies.TryGetValue(u, out policy))
                         {
                             field.Type = policy.Type;
                             field.EnableSorting = policy.EnableSorting;
@@ -55,8 +56,9 @@ namespace UmbracoExamine.Config
                     foreach (var s in sysProps)
                     {
                         var field = new IndexField() { Name = s };
-                        var policy = indexFieldPolicies.FirstOrDefault(x => x.Name == s);
-                        if (policy != null)
+
+                        StaticField policy;
+                        if (indexFieldPolicies.TryGetValue(s, out policy))
                         {
                             field.Type = policy.Type;
                             field.EnableSorting = policy.EnableSorting;

src/UmbracoExamine/StaticFieldCollection.cs

@@ -0,0 +1,28 @@
+using System.Collections.ObjectModel;
+
+namespace UmbracoExamine
+{
+    internal class StaticFieldCollection : KeyedCollection<string, StaticField>
+    {
+        protected override string GetKeyForItem(StaticField item)
+        {
+            return item.Name;
+        }
+
+        /// <summary>
+        /// Implements TryGetValue using the underlying dictionary
+        /// </summary>
+        /// <param name="key"></param>
+        /// <param name="field"></param>
+        /// <returns></returns>
+        public bool TryGetValue(string key, out StaticField field)
+        {
+            if (Dictionary == null)
+            {
+                field = null;
+                return false;
+            }
+            return Dictionary.TryGetValue(key, out field);
+        }
+    }
+}

src/UmbracoExamine/UmbracoContentIndexer.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Xml;
 using System.Xml.Linq;
 using Examine;
 using Lucene.Net.Documents;
@@ -15,6 +17,7 @@ using Examine.LuceneEngine;
 using Examine.LuceneEngine.Config;
 using UmbracoExamine.Config;
 using Lucene.Net.Analysis;
+using Lucene.Net.Index;
 using Umbraco.Core.Persistence.Querying;
 using IContentService = Umbraco.Core.Services.IContentService;
 using IMediaService = Umbraco.Core.Services.IMediaService;
@@ -144,6 +147,34 @@ namespace UmbracoExamine
             _contentTypeService = contentTypeService;
         }
 
+        /// <summary>
+        /// Creates an NRT indexer
+        /// </summary>
+        /// <param name="indexerData"></param>
+        /// <param name="writer"></param>
+        /// <param name="dataService"></param>
+        /// <param name="contentTypeService"></param>
+        /// <param name="async"></param>
+        /// <param name="contentService"></param>
+        /// <param name="mediaService"></param>
+        /// <param name="dataTypeService"></param>
+        /// <param name="userService"></param>
+        public UmbracoContentIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService,
+            IContentService contentService,
+            IMediaService mediaService,
+            IDataTypeService dataTypeService,
+            IUserService userService,
+            IContentTypeService contentTypeService,
+            bool async) 
+            : base(indexerData, writer, dataService, async)
+        {
+            _contentService = contentService;
+            _mediaService = mediaService;
+            _dataTypeService = dataTypeService;
+            _userService = userService;
+            _contentTypeService = contentTypeService;
+        }
+
         #endregion
 
         #region Constants & Fields        
@@ -165,27 +196,27 @@ namespace UmbracoExamine
         /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene
         /// for retreival after searching.
         /// </summary>
-        internal static readonly List<StaticField> IndexFieldPolicies
-            = new List<StaticField>
+        internal static readonly StaticFieldCollection IndexFieldPolicies
+            = new StaticFieldCollection
             {
                 new StaticField("id", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
                 new StaticField("key", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "version", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "parentID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "level", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
-                new StaticField( "writerID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "creatorID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "nodeType", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "template", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "sortOrder", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
-                new StaticField( "createDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
-                new StaticField( "updateDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
-                new StaticField( "nodeName", FieldIndexTypes.ANALYZED, false, string.Empty),
-                new StaticField( "urlName", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
-                new StaticField( "writerName", FieldIndexTypes.ANALYZED, false, string.Empty),
-                new StaticField( "creatorName", FieldIndexTypes.ANALYZED, false, string.Empty),
-                new StaticField( "nodeTypeAlias", FieldIndexTypes.ANALYZED, false, string.Empty),
-                new StaticField( "path", FieldIndexTypes.NOT_ANALYZED, false, string.Empty)
+                new StaticField("version", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("parentID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("level", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
+                new StaticField("writerID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("creatorID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("nodeType", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("template", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("sortOrder", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
+                new StaticField("createDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
+                new StaticField("updateDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
+                new StaticField("nodeName", FieldIndexTypes.ANALYZED, false, string.Empty),
+                new StaticField("urlName", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
+                new StaticField("writerName", FieldIndexTypes.ANALYZED, false, string.Empty),
+                new StaticField("creatorName", FieldIndexTypes.ANALYZED, false, string.Empty),
+                new StaticField("nodeTypeAlias", FieldIndexTypes.ANALYZED, false, string.Empty),
+                new StaticField("path", FieldIndexTypes.NOT_ANALYZED, false, string.Empty)
             };
 
         #endregion
@@ -229,7 +260,13 @@ namespace UmbracoExamine
                 SupportProtectedContent = supportProtected;
             else
                 SupportProtectedContent = false;
-            
+
+            bool disableXmlDocLookup;
+            if (config["disableXmlDocLookup"] != null && bool.TryParse(config["disableXmlDocLookup"], out disableXmlDocLookup))
+                DisableXmlDocumentLookup = disableXmlDocLookup;
+            else
+                DisableXmlDocumentLookup = false;
+
             base.Initialize(name, config);
         }
 
@@ -237,6 +274,11 @@ namespace UmbracoExamine
 
         #region Properties
 
+        /// <summary>
+        /// Whether to use the cmsContentXml data to re-index when possible (i.e. for published content, media and members)
+        /// </summary>
+        public bool DisableXmlDocumentLookup { get; private set; }
+
         /// <summary>
         /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected.
         /// This property is ignored if SupportUnpublishedContent is set to true.
@@ -359,131 +401,250 @@ namespace UmbracoExamine
         }
         #endregion
 
-        #region Protected
-
-        /// <summary>
-        /// This is a static query, it's parameters don't change so store statically
-        /// </summary>
-        private IQuery<IContent> _publishedQuery;
+        #region Protected        
 
         protected override void PerformIndexAll(string type)
         {
+            if (SupportedTypes.Contains(type) == false)
+                return;
+
             const int pageSize = 10000;
             var pageIndex = 0;
 
-            switch (type)
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - Start data queries - {0}", type));
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            try
             {
-                case IndexTypes.Content:
-                    var contentParentId = -1;
-                    if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
-                    {
-                        contentParentId = IndexerData.ParentNodeId.Value;
-                    }
-                    IContent[] content;
-
-                    //used to track non-published entities so we can determine what items are implicitly not published
-                    var notPublished = new HashSet<string>();
-
-                    do
-                    {
-                        long total;
-
-                        IEnumerable<IContent> descendants;
-                        if (SupportUnpublishedContent)
+                switch (type)
+                {
+                    case IndexTypes.Content:
+                        var contentParentId = -1;
+                        if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
                         {
-                            descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
-                        }
-                        else
-                        {
-                            if (_publishedQuery == null)
-                            {
-                                _publishedQuery = Query<IContent>.Builder.Where(x => x.Published == true);
-                            }
-
-                            //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine
-                            // which descendent nodes are implicitly not published
-                            descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null);
-                        }                        
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (IndexerData.IncludeNodeTypes.Any())
-                        {
-                            content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
-                        }
-                        else
-                        {
-                            content = descendants.ToArray();
-                        }
-                        AddNodesToIndex(GetSerializedContent(content, notPublished).WhereNotNull(), type);
-                        pageIndex++;
-                    } while (content.Length == pageSize);
-
-                    notPublished.Clear();
-
-                    break;
-                case IndexTypes.Media:
-                    var mediaParentId = -1;
-
-                    if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
-                    {
-                        mediaParentId = IndexerData.ParentNodeId.Value;
-                    }
-                    
-                    XElement[] mediaXElements;
-                    
-                    var mediaTypes = _contentTypeService.GetAllMediaTypes().ToArray();
-                    var icons = mediaTypes.ToDictionary(x => x.Id, y => y.Icon);
-
-                    do
-                    {
-                        long total;
-                        if (mediaParentId == -1)
-                        {
-                            mediaXElements = _mediaService.GetPagedXmlEntries("-1", pageIndex, pageSize, out total).ToArray();
-                        }
-                        else
-                        {
-                            //Get the parent
-                            var parent = _mediaService.GetById(mediaParentId);
-                            if (parent == null)
-                                mediaXElements = new XElement[0];
-                            else
-                                mediaXElements = _mediaService.GetPagedXmlEntries(parent.Path, pageIndex, pageSize, out total).ToArray();
-                        }
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (IndexerData.IncludeNodeTypes.Any())
-                        {
-                            var includeNodeTypeIds = mediaTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id);
-                            mediaXElements = mediaXElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue<int>("nodeType"))).ToArray();
+                            contentParentId = IndexerData.ParentNodeId.Value;
                         }
                         
-                        foreach (var element in mediaXElements)
+                        if (SupportUnpublishedContent == false && DisableXmlDocumentLookup == false)
                         {
-                            element.Add(new XAttribute("icon", icons[element.AttributeValue<int>("nodeType")]));
+                            //get all node Ids that have a published version - this is a fail safe check, in theory
+                            // only document nodes that have a published version would exist in the cmsContentXml table
+                            var allNodesWithPublishedVersions = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                                "select DISTINCT cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1");
+
+                            XElement last = null;
+                            var trackedIds = new HashSet<string>();
+
+                            ReindexWithXmlEntries(type, contentParentId,
+                                () => _contentTypeService.GetAllContentTypes().ToArray(),
+                                (path, pIndex, pSize) =>
+                                {
+                                    long totalContent;
+
+                                    //sorted by: umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder
+                                    var result = _contentService.GetPagedXmlEntries(path, pIndex, pSize, out totalContent).ToArray();
+
+                                    //then like we do in the ContentRepository.BuildXmlCache we need to track what Parents have been processed
+                                    // already so that we can then exclude implicitly unpublished content items
+                                    var filtered = new List<XElement>();
+
+                                    foreach (var xml in result)
+                                    {
+                                        var id = xml.AttributeValue<int>("id");
+                                        
+                                        //don't include this if it doesn't have a published version
+                                        if (allNodesWithPublishedVersions.Contains(id) == false)
+                                            continue;
+
+                                        var parentId = xml.AttributeValue<string>("parentID");
+
+                                        if (parentId == null) continue; //this shouldn't happen
+
+                                        //if the parentid is changing
+                                        if (last != null && last.AttributeValue<string>("parentID") != parentId)
+                                        {
+                                            var found = trackedIds.Contains(parentId);
+                                            if (found == false)
+                                            {
+                                                //Need to short circuit here, if the parent is not there it means that the parent is unpublished
+                                                // and therefore the child is not published either so cannot be included in the xml cache
+                                                continue;
+                                            }                                            
+                                        }
+                                        
+                                        last = xml;
+                                        trackedIds.Add(xml.AttributeValue<string>("id"));
+
+                                        filtered.Add(xml);
+                                    }
+
+                                    return new Tuple<long, XElement[]>(totalContent, filtered.ToArray());
+                                },
+                                i => _contentService.GetById(i));
+                        }
+                        else
+                        {
+                            //used to track non-published entities so we can determine what items are implicitly not published
+                            //currently this is not in use apart form in tests
+                            var notPublished = new HashSet<string>();
+
+                            int currentPageSize;
+                            do
+                            {
+                                long total;
+
+                                IContent[] descendants;
+                                if (SupportUnpublishedContent)
+                                {
+                                    descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "umbracoNode.id").ToArray();
+                                }
+                                else
+                                {
+                                    //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine
+                                    // which descendent nodes are implicitly not published
+                                    descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null).ToArray();
+                                }
+
+                                // need to store decendants count before filtering, in order for loop to work correctly
+                                currentPageSize = descendants.Length;
+
+                                //if specific types are declared we need to post filter them
+                                //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
+                                IEnumerable<IContent> content;
+                                if (IndexerData.IncludeNodeTypes.Any())
+                                {
+                                    content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias));
+                                }
+                                else
+                                {
+                                    content = descendants;
+                                }
+
+                                AddNodesToIndex(GetSerializedContent(
+                                    SupportUnpublishedContent,
+                                    c => _serializer.Serialize(_contentService, _dataTypeService, _userService, c),
+                                    content, notPublished).WhereNotNull(), type);
+
+                                pageIndex++;
+                            } while (currentPageSize == pageSize);
                         }
 
-                        AddNodesToIndex(mediaXElements, type);
-                        pageIndex++;
-                    } while (mediaXElements.Length == pageSize);
+                        break;
+                    case IndexTypes.Media:
+                        var mediaParentId = -1;
 
-                    break;
+                        if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
+                        {
+                            mediaParentId = IndexerData.ParentNodeId.Value;
+                        }
+
+                        ReindexWithXmlEntries(type, mediaParentId,
+                            () => _contentTypeService.GetAllMediaTypes().ToArray(),
+                            (path, pIndex, pSize) =>
+                            {
+                                long totalMedia;
+                                var result = _mediaService.GetPagedXmlEntries(path, pIndex, pSize, out totalMedia).ToArray();
+                                return new Tuple<long, XElement[]>(totalMedia, result);
+                            },
+                            i => _mediaService.GetById(i));
+
+                        break;
+                }
             }
+            finally
+            {
+                stopwatch.Stop();
+            }
+            
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - End data queries - {0}, took {1}ms", type, stopwatch.ElapsedMilliseconds));
         }
 
-        private IEnumerable<XElement> GetSerializedContent(IEnumerable<IContent> content, ISet<string> notPublished)
+        /// <summary>
+        /// Performs a reindex of a type based on looking up entries from the cmsContentXml table - but using callbacks to get this data since
+        /// we don't have a common underlying service interface for the media/content stuff
+        /// </summary>
+        /// <param name="type"></param>
+        /// <param name="parentId"></param>
+        /// <param name="getContentTypes"></param>
+        /// <param name="getPagedXmlEntries"></param>
+        /// <param name="getContent"></param>
+        internal void ReindexWithXmlEntries<TContentType>(
+            string type, 
+            int parentId,
+            Func<TContentType[]> getContentTypes, 
+            Func<string, int, int, Tuple<long, XElement[]>> getPagedXmlEntries,
+            Func<int, IContentBase> getContent)
+            where TContentType: IContentTypeComposition
+        {
+            const int pageSize = 10000;
+            var pageIndex = 0;            
+
+            XElement[] xElements;
+
+            var contentTypes = getContentTypes();
+            var icons = contentTypes.ToDictionary(x => x.Id, y => y.Icon);
+
+            do
+            {
+                long total;
+                if (parentId == -1)
+                {
+                    var pagedElements = getPagedXmlEntries("-1", pageIndex, pageSize);
+                    total = pagedElements.Item1;
+                    xElements = pagedElements.Item2;
+                }
+                else
+                {
+                    //Get the parent
+                    var parent = getContent(parentId);
+                    if (parent == null)
+                        xElements = new XElement[0];
+                    else
+                    {
+                        var pagedElements = getPagedXmlEntries(parent.Path, pageIndex, pageSize);
+                        total = pagedElements.Item1;
+                        xElements = pagedElements.Item2;
+                    }
+                }
+
+                //if specific types are declared we need to post filter them
+                //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
+                if (IndexerData.IncludeNodeTypes.Any())
+                {
+                    var includeNodeTypeIds = contentTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id);
+                    xElements = xElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue<int>("nodeType"))).ToArray();
+                }
+
+                foreach (var element in xElements)
+                {
+                    if (element.Attribute("icon") == null)
+                    {
+                        element.Add(new XAttribute("icon", icons[element.AttributeValue<int>("nodeType")]));
+                    }
+                }
+
+                AddNodesToIndex(xElements, type);
+                pageIndex++;
+            } while (xElements.Length == pageSize);
+        }
+
+        internal static IEnumerable<XElement> GetSerializedContent(
+            bool supportUnpublishdContent, 
+            Func<IContent, XElement> serializer, 
+            IEnumerable<IContent> content, 
+            ISet<string> notPublished)
         {            
             foreach (var c in content)
             {
-                if (SupportUnpublishedContent == false)
+                if (supportUnpublishdContent == false)
                 {
                     //if we don't support published content and this is not published then track it and return null
                     if (c.Published == false)
                     {
                         notPublished.Add(c.Path);
                         yield return null;
+                        continue;
                     }
 
                     //if we don't support published content, check if this content item exists underneath any already tracked
@@ -491,14 +652,11 @@ namespace UmbracoExamine
                     if (notPublished.Any(path => c.Path.StartsWith(string.Format("{0},", path))))
                     {
                         yield return null;
+                        continue;
                     }
-                }
+                }                
 
-                var xml = _serializer.Serialize(
-                    _contentService,
-                    _dataTypeService,
-                    _userService,
-                    c);
+                var xml = serializer(c);                
 
                 //add a custom 'icon' attribute
                 xml.Add(new XAttribute("icon", c.ContentType.Icon));
@@ -520,7 +678,7 @@ namespace UmbracoExamine
 
         public override void RebuildIndex()
         {
-            DataService.LogService.AddVerboseLog(-1, "Rebuilding index");
+            DataService.LogService.AddInfoLog(-1, "Rebuilding index");
             base.RebuildIndex();
         }
 
@@ -551,15 +709,16 @@ namespace UmbracoExamine
             // Get all user data that we want to index and store into a dictionary 
             foreach (var field in IndexerData.UserFields)
             {
-                if (e.Fields.ContainsKey(field.Name))
+                string fieldVal;
+                if (e.Fields.TryGetValue(field.Name, out fieldVal))
                 {
                     //check if the field value has html
-                    if (XmlHelper.CouldItBeXml(e.Fields[field.Name]))
+                    if (XmlHelper.CouldItBeXml(fieldVal))
                     {
                         //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later
-                        e.Fields[RawFieldPrefix + field.Name] = e.Fields[field.Name];
+                        e.Fields[RawFieldPrefix + field.Name] = fieldVal;
                         //now replace the original value with the stripped html
-                        e.Fields[field.Name] = DataService.ContentService.StripHtml(e.Fields[field.Name]);
+                        e.Fields[field.Name] = DataService.ContentService.StripHtml(fieldVal);
                     }
                 }
             }
@@ -645,9 +804,13 @@ namespace UmbracoExamine
         /// <param name="fieldName"></param>
         /// <returns></returns>
         protected override FieldIndexTypes GetPolicy(string fieldName)
-        {
-            var def = IndexFieldPolicies.Where(x => x.Name == fieldName).ToArray();
-            return (def.Any() == false ? FieldIndexTypes.ANALYZED : def.Single().IndexType);
+        {            
+            StaticField def;
+            if (IndexFieldPolicies.TryGetValue(fieldName, out def))
+            {
+                return def.IndexType;
+            }
+            return FieldIndexTypes.ANALYZED;
         }
 
         /// <summary>
@@ -657,14 +820,18 @@ namespace UmbracoExamine
         /// </summary>
         protected override bool ValidateDocument(XElement node)
         {
-            var nodeId = int.Parse(node.Attribute("id").Value);
             // Test for access if we're only indexing published content
             // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content
-            if (!SupportUnpublishedContent
-                && (!SupportProtectedContent
-                && DataService.ContentService.IsProtected(nodeId, node.Attribute("path").Value)))
+            if (SupportUnpublishedContent == false
+                && SupportProtectedContent == false)
             {
-                return false;
+
+                var nodeId = int.Parse(node.Attribute("id").Value);
+
+                if (DataService.ContentService.IsProtected(nodeId, node.Attribute("path").Value))
+                {
+                    return false;
+                }               
             }
             return base.ValidateDocument(node);
         }

src/UmbracoExamine/UmbracoExamine.csproj

@@ -82,8 +82,8 @@
     <AssemblyOriginatorKeyFile>..\Solution Items\TheFARM-Public.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.81.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.81\lib\net45\Examine.dll</HintPath>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
@@ -133,6 +133,7 @@
     <Compile Include="LocalStorage\MultiIndexOutput.cs" />
     <Compile Include="LoggingLevel.cs" />
     <Compile Include="StaticField.cs" />
+    <Compile Include="StaticFieldCollection.cs" />
     <Compile Include="UmbracoMemberIndexer.cs" />
     <Compile Include="ContentExtensions.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />