Updates the paging logic so that the ContentRepository uses 2x queries: A full query and a query for getting Ids. This is important because the query for fetching ids is used for paging and for getting property data and it can be much much faster than the full query which was previously used for both. There's not really any changes to the media/members respositories since their full queries don't have certain outer joins that make them run really slow

Fixes: U4-9427 Unable to install local packages in various browsers

build/NuSpecs/UmbracoCms.Core.nuspec

@@ -32,9 +32,9 @@
 	  <!-- AutoMapper can not be updated due to: https://github.com/AutoMapper/AutoMapper/issues/373#issuecomment-127644405 -->
       <dependency id="AutoMapper" version="[3.3.1, 4.0.0)" />
       <dependency id="Newtonsoft.Json" version="[6.0.8, 10.0.0)" />
-      <dependency id="Examine" version="[0.1.70, 1.0.0)" />
+      <dependency id="Examine" version="[0.1.80, 1.0.0)" />
       <dependency id="ImageProcessor" version="[2.5.1, 3.0.0)" />
-      <dependency id="ImageProcessor.Web" version="[4.7.2, 5.0.0)" />
+      <dependency id="ImageProcessor.Web" version="[4.8.0, 5.0.0)" />
       <dependency id="semver" version="[1.1.2, 2.0.0)" />
       <dependency id="UrlRewritingNet" version="[2.0.7, 3.0.0)" />	  
 	  <!-- Markdown can not be updated due to: https://github.com/hey-red/markdownsharp/issues/71#issuecomment-233585487 -->

build/UmbracoVersion.txt

@@ -1,2 +1,2 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
-7.5.7
+7.5.8

src/SolutionInfo.cs

@@ -11,5 +11,5 @@ using System.Resources;
 
 [assembly: AssemblyVersion("1.0.*")]
 
-[assembly: AssemblyFileVersion("7.5.7")]
-[assembly: AssemblyInformationalVersion("7.5.7")]
+[assembly: AssemblyFileVersion("7.5.8")]
+[assembly: AssemblyInformationalVersion("7.5.8")]

src/Umbraco.Core/Cache/CacheKeys.cs

@@ -64,7 +64,8 @@ namespace Umbraco.Core.Cache
         [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
         public const string ContentTypePropertiesCacheKey = "ContentType_PropertyTypes_Content:";
         
-        [UmbracoWillObsolete("This cache key is only used for legacy business logic caching, remove in v8")]
+        [Obsolete("No longer used and will be removed in v8")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         public const string PropertyTypeCacheKey = "UmbracoPropertyTypeCache";
 
         [Obsolete("This is no longer used and will be removed from the codebase in the future")]

src/Umbraco.Core/Configuration/UmbracoVersion.cs

@@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration
 {
     public class UmbracoVersion
     {
-        private static readonly Version Version = new Version("7.5.7");
+        private static readonly Version Version = new Version("7.5.8");
 
         /// <summary>
         /// Gets the current version of Umbraco.

src/Umbraco.Core/CoreBootManager.cs

@@ -415,7 +415,7 @@ namespace Umbraco.Core
 
             if (currentTry == 5)
             {
-                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but the Umbraco cannot connect to the database.");
+                throw new UmbracoStartupFailedException("Umbraco cannot start. A connection string is configured but Umbraco cannot connect to the database.");
             }
         }
 

src/Umbraco.Core/DatabaseContext.cs

@@ -203,15 +203,10 @@ namespace Umbraco.Core
             var path = Path.Combine(GlobalSettings.FullpathToRoot, "App_Data", "Umbraco.sdf");
             if (File.Exists(path) == false)
             {
-                var engine = new SqlCeEngine(connectionString);
-                engine.CreateDatabase();
-
-                // SD: Pretty sure this should be in a using clause but i don't want to cause unknown side-effects here
-                // since it's been like this for quite some time
-                //using (var engine = new SqlCeEngine(connectionString))
-                //{
-                //    engine.CreateDatabase();    
-                //}
+                using (var engine = new SqlCeEngine(connectionString))
+                {
+                    engine.CreateDatabase();
+                }
             }
 
             Initialize(providerName);

src/Umbraco.Core/Models/IMediaType.cs

@@ -4,7 +4,7 @@ namespace Umbraco.Core.Models
 {
     /// <summary>
     /// Defines a ContentType, which Media is based on
-    /// </summary
+    /// </summary>
     public interface IMediaType : IContentTypeComposition
     {
 

src/Umbraco.Core/Persistence/Factories/ContentFactory.cs

@@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IContent,DocumentDto>
 
-        public IContent BuildEntity(DocumentDto dto)
+        public static IContent BuildEntity(DocumentDto dto, IContentType contentType)
         {
-            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, _contentType);
+            var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, contentType);
 
             try
             {
                 content.DisableChangeTracking();
 
-                content.Id = _id;
+                content.Id = dto.NodeId;
                 content.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId;
                 content.Name = dto.Text;
                 content.NodeName = dto.ContentVersionDto.ContentDto.NodeDto.Text;
@@ -49,8 +49,8 @@ namespace Umbraco.Core.Persistence.Factories
                 content.Published = dto.Published;
                 content.CreateDate = dto.ContentVersionDto.ContentDto.NodeDto.CreateDate;
                 content.UpdateDate = dto.ContentVersionDto.VersionDate;
-                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?) null;
-                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?) null;
+                content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?)null;
+                content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?)null;
                 content.Version = dto.ContentVersionDto.VersionId;
                 content.PublishedState = dto.Published ? PublishedState.Published : PublishedState.Unpublished;
                 content.PublishedVersionGuid = dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId;
@@ -64,6 +64,13 @@ namespace Umbraco.Core.Persistence.Factories
             {
                 content.EnableChangeTracking();
             }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IContent BuildEntity(DocumentDto dto)
+        {
+            return BuildEntity(dto, _contentType);
         }
 
         public DocumentDto BuildDto(IContent entity)

src/Umbraco.Core/Persistence/Factories/MediaFactory.cs

@@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories
 
         #region Implementation of IEntityFactory<IMedia,ContentVersionDto>
 
-        public IMedia BuildEntity(ContentVersionDto dto)
+        public static IMedia BuildEntity(ContentVersionDto dto, IMediaType contentType)
         {
-            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, _contentType);
+            var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, contentType);
 
             try
             {
                 media.DisableChangeTracking();
 
-                media.Id = _id;
+                media.Id = dto.NodeId;
                 media.Key = dto.ContentDto.NodeDto.UniqueId;
                 media.Path = dto.ContentDto.NodeDto.Path;
                 media.CreatorId = dto.ContentDto.NodeDto.UserId.Value;
@@ -55,6 +55,13 @@ namespace Umbraco.Core.Persistence.Factories
             {
                 media.EnableChangeTracking();
             }
+
+        }
+
+        [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")]
+        public IMedia BuildEntity(ContentVersionDto dto)
+        {
+            return BuildEntity(dto, _contentType);
         }
 
         public ContentVersionDto BuildDto(IMedia entity)

src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs

@@ -30,11 +30,11 @@ namespace Umbraco.Core.Persistence.Factories
             _updateDate = updateDate;
         }
 
-        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        public static IEnumerable<Property> BuildEntity(IReadOnlyCollection<PropertyDataDto> dtos, PropertyType[] compositionTypeProperties, DateTime createDate, DateTime updateDate)
         {
             var properties = new List<Property>();
 
-            foreach (var propertyType in _compositionTypeProperties)
+            foreach (var propertyType in compositionTypeProperties)
             {
                 var propertyDataDto = dtos.LastOrDefault(x => x.PropertyTypeId == propertyType.Id);
                 var property = propertyDataDto == null
@@ -47,8 +47,8 @@ namespace Umbraco.Core.Persistence.Factories
                     //on initial construction we don't want to have dirty properties tracked
                     property.DisableChangeTracking();
 
-                    property.CreateDate = _createDate;
-                    property.UpdateDate = _updateDate;
+                    property.CreateDate = createDate;
+                    property.UpdateDate = updateDate;
                     // http://issues.umbraco.org/issue/U4-1946
                     property.ResetDirtyProperties(false);
                     properties.Add(property);
@@ -57,12 +57,18 @@ namespace Umbraco.Core.Persistence.Factories
                 {
                     property.EnableChangeTracking();
                 }
-                
+
             }
 
             return properties;
         }
 
+        [Obsolete("Use the static method instead, there's no reason to allocate one of these classes everytime we want to map values")]
+        public IEnumerable<Property> BuildEntity(PropertyDataDto[] dtos)
+        {
+            return BuildEntity(dtos, _compositionTypeProperties, _createDate, _updateDate);
+        }
+
         public IEnumerable<PropertyDataDto> BuildDto(IEnumerable<Property> properties)
         {
             var propertyDataDtos = new List<PropertyDataDto>();

src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs

@@ -20,6 +20,7 @@ namespace Umbraco.Core.Persistence.Querying
             _mapper = mapper;
         }
 
+        [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public ModelToSqlExpressionVisitor()
             : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T)))
         { }

src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs

@@ -18,14 +18,13 @@ namespace Umbraco.Core.Persistence.Querying
         public PocoToSqlExpressionVisitor(ISqlSyntaxProvider syntaxProvider)
             : base(syntaxProvider)
         {
-            
+            _pd = new Database.PocoData(typeof(T));
         }
 
         [Obsolete("Use the overload the specifies a SqlSyntaxProvider")]
         public PocoToSqlExpressionVisitor()
-            : base(SqlSyntaxContext.SqlSyntaxProvider)
-        {
-            _pd = new Database.PocoData(typeof(T));
+            : this(SqlSyntaxContext.SqlSyntaxProvider)
+        {            
         }
 
         protected override string VisitMemberAccess(MemberExpression m)

src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs

@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Persistence.Repositories
+{
+    internal enum BaseQueryType
+    {
+        Full,
+        Ids,
+        Count
+    }
+}

src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs

@@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = GetBaseQuery(false)
                 .Where(GetBaseWhereClause(), new { Id = id })
-                .Where<DocumentDto>(x => x.Newest)
+                .Where<DocumentDto>(x => x.Newest, SqlSyntax)
                 .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(SqlSyntax.SelectTop(sql, 1)).FirstOrDefault();
@@ -70,63 +70,90 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected override IEnumerable<IContent> PerformGetAll(params int[] ids)
         {
-            var sql = GetBaseQuery(false);
-            if (ids.Any())
+            Func<Sql, Sql> translate = s =>
             {
-                sql.Where("umbracoNode.id in (@ids)", new { ids });
-            }
+                if (ids.Any())
+                {
+                    s.Where("umbracoNode.id in (@ids)", new { ids });
+                }
+                //we only want the newest ones with this method
+                s.Where<DocumentDto>(x => x.Newest, SqlSyntax);
+                return s;
+            };
+            
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
 
-            //we only want the newest ones with this method
-            sql.Where<DocumentDto>(x => x.Newest);
-
-            return ProcessQuery(sql);
+            return ProcessQuery(translate(sqlBaseFull), translate(sqlBaseIds));
         }
 
         protected override IEnumerable<IContent> PerformGetByQuery(IQuery<IContent> query)
         {
-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Newest)
-                                .OrderByDescending<ContentVersionDto>(x => x.VersionDate)
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
 
-            return ProcessQuery(sql);
+            Func<SqlTranslator<IContent>, Sql> translate = (translator) =>
+            {
+                return translator.Translate()
+                    .Where<DocumentDto>(x => x.Newest, SqlSyntax)
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
+            var translatorFull = new SqlTranslator<IContent>(sqlBaseFull, query);
+            var translatorIds = new SqlTranslator<IContent>(sqlBaseIds, query);
+
+            return ProcessQuery(translate(translatorFull), translate(translatorIds));
         }
 
         #endregion        
 
         #region Overrides of PetaPocoRepositoryBase<IContent>
 
-
-        protected override Sql GetBaseQuery(bool isCount)
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
         {
-            var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsDocument.nodeId" : "*"))
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<DocumentDto, ContentVersionDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            if (queryType == BaseQueryType.Full)
+            {
+                //The only reason we apply this left outer join is to be able to pull back the DocumentPublishedReadOnlyDto
+                //information with the entire data set, so basically this will get both the latest document and also it's published
+                //version if it has one. When performing a count or when just retrieving Ids like in paging, this is unecessary 
+                //and causes huge performance overhead for the SQL server, especially when sorting the result. 
+                //To fix this perf overhead we'd need another index on :
+                // CREATE NON CLUSTERED INDEX ON cmsDocument.node + cmsDocument.published
+
+                var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
                 SqlSyntax.GetQuotedTableName("cmsDocument"),
                 SqlSyntax.GetQuotedTableName("cmsDocument2"),
                 SqlSyntax.GetQuotedColumnName("nodeId"),
                 SqlSyntax.GetQuotedColumnName("published"));
 
-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<DocumentDto, ContentVersionDto>(left => left.VersionId, right => right.VersionId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-
                 // cannot do this because PetaPoco does not know how to alias the table
                 //.LeftOuterJoin<DocumentPublishedReadOnlyDto>()
                 //.On<DocumentDto, DocumentPublishedReadOnlyDto>(left => left.NodeId, right => right.NodeId)
                 // so have to rely on writing our own SQL
-                .Append(sqlx/*, new { @published = true }*/)
+                sql.Append(sqlx /*, new { @published = true }*/);
+            }
+
+            sql.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
 
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
             return sql;
         }
 
+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
+        }
+
         protected override string GetBaseWhereClause()
         {
             return "umbracoNode.id = @Id";
@@ -173,20 +200,32 @@ namespace Umbraco.Core.Persistence.Repositories
             // not bring much safety - so this reverts to updating each record individually,
             // and it may be slower in the end, but should be more resilient.
 
-            var baseId = 0;
             var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
+
+            Func<int, Sql, Sql> translate = (bId, sql) =>
+            {
+                if (contentTypeIdsA.Length > 0)
+                {
+                    sql.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
+
+                sql
+                    .Where<NodeDto>(x => x.NodeId > bId && x.Trashed == false, SqlSyntax)
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
+
+                return sql;
+            };
+
+            var baseId = 0;
+            
             while (true)
             {
                 // get the next group of nodes
-                var query = GetBaseQuery(false);
-                if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
-                query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId && x.Trashed == false)
-                    .Where<DocumentDto>(x => x.Published)
-                    .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
-                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
+                var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.Full));
+                var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids));
+
+                var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), SqlSyntax.SelectTop(sqlIds, groupSize))
                     .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
                     .ToList();
 
@@ -212,17 +251,23 @@ namespace Umbraco.Core.Persistence.Repositories
 
         public override IEnumerable<IContent> GetAllVersions(int id)
         {
-            var sql = GetBaseQuery(false)
-                .Where(GetBaseWhereClause(), new { Id = id })
-                .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
-            return ProcessQuery(sql, true);
+            Func<Sql, Sql> translate = s =>
+            {
+                return s.Where(GetBaseWhereClause(), new {Id = id})
+                    .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
+            };
+
+            var sqlFull = translate(GetBaseQuery(BaseQueryType.Full));
+            var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
+            
+            return ProcessQuery(sqlFull, sqlIds, true);
         }
 
         public override IContent GetByVersion(Guid versionId)
         {
             var sql = GetBaseQuery(false);
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql).FirstOrDefault();
 
@@ -238,10 +283,10 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = new Sql()
                 .Select("*")
-                .From<DocumentDto>()
-                .InnerJoin<ContentVersionDto>().On<ContentVersionDto, DocumentDto>(left => left.VersionId, right => right.VersionId)
-                .Where<ContentVersionDto>(x => x.VersionId == versionId)
-                .Where<DocumentDto>(x => x.Newest != true);
+                .From<DocumentDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax).On<ContentVersionDto, DocumentDto>(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .Where<ContentVersionDto>(x => x.VersionId == versionId, SqlSyntax)
+                .Where<DocumentDto>(x => x.Newest != true, SqlSyntax);
             var dto = Database.Fetch<DocumentDto, ContentVersionDto>(sql).FirstOrDefault();
 
             if (dto == null) return;
@@ -616,21 +661,27 @@ namespace Umbraco.Core.Persistence.Repositories
 
         public IEnumerable<IContent> GetByPublishedVersion(IQuery<IContent> query)
         {
+            Func<SqlTranslator<IContent>, Sql> translate = t =>
+            {
+                return t.Translate()
+                    .Where<DocumentDto>(x => x.Published, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
+                    .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            };
+
             // we WANT to return contents in top-down order, ie parents should come before children
             // ideal would be pure xml "document order" which can be achieved with:
             // ORDER BY substring(path, 1, len(path) - charindex(',', reverse(path))), sortOrder
             // but that's probably an overkill - sorting by level,sortOrder should be enough
 
-            var sqlClause = GetBaseQuery(false);
-            var translator = new SqlTranslator<IContent>(sqlClause, query);
-            var sql = translator.Translate()
-                                .Where<DocumentDto>(x => x.Published)
-                                .OrderBy<NodeDto>(x => x.Level, SqlSyntax)
-                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
+            var sqlFull = GetBaseQuery(BaseQueryType.Full);
+            var translatorFull = new SqlTranslator<IContent>(sqlFull, query);
+            var sqlIds = GetBaseQuery(BaseQueryType.Ids);
+            var translatorIds = new SqlTranslator<IContent>(sqlIds, query);
 
-            return ProcessQuery(sql, true);
+            return ProcessQuery(translate(translatorFull), translate(translatorIds), true);
         }
-
+        
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -662,10 +713,14 @@ namespace Umbraco.Core.Persistence.Repositories
             parent.Attributes.Append(pIdAtt);
             xmlDoc.AppendChild(parent);
 
-            const string sql = @"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.xml, umbracoNode.level from umbracoNode
+            //Ensure that only nodes that have published versions are selected
+            var sql = string.Format(@"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.{0}, umbracoNode.{1} from umbracoNode
 inner join cmsContentXml on cmsContentXml.nodeId = umbracoNode.id and umbracoNode.nodeObjectType = @type
 where umbracoNode.id in (select cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1)
-order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
+order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
+                SqlSyntax.GetQuotedColumnName("xml"),
+                SqlSyntax.GetQuotedColumnName("level"),
+                SqlSyntax.GetQuotedColumnName("level"));
 
             XmlElement last = null;
 
@@ -802,9 +857,9 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
             
             Func<Tuple<string, object[]>> filterCallback = () => new Tuple<string, object[]>(filterSql.SQL, filterSql.Arguments);
 
-            return GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<DocumentDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsDocument", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
 
         }
@@ -835,16 +890,32 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
             return base.GetDatabaseFieldNameForOrderBy(orderBy);
         }
 
-        private IEnumerable<IContent> ProcessQuery(Sql sql, bool withCache = false)
+        /// <summary>
+        /// This is the underlying method that processes most queries for this repository
+        /// </summary>
+        /// <param name="sqlFull">
+        /// The full SQL with the outer join to return all data required to create an IContent
+        /// </param>
+        /// <param name="sqlIds">
+        /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
+        /// </param>
+        /// <param name="withCache"></param>
+        /// <returns></returns>
+        private IEnumerable<IContent> ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false)
         {
             // fetch returns a list so it's ok to iterate it in this method
-            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sql);
+            var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sqlFull);
             if (dtos.Count == 0) return Enumerable.Empty<IContent>();
 
             var content = new IContent[dtos.Count];
             var defs = new List<DocumentDefinition>();
             var templateIds = new List<int>();
-
+            
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IContentType>();
+            
             for (var i = 0; i < dtos.Count; i++)
             {
                 var dto = dtos[i];
@@ -863,9 +934,19 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
 
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
-                var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
-                var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
+
+                IContentType contentType;
+                if (contentTypes.ContainsKey(dto.ContentVersionDto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
+                }
+                
+                content[i] = ContentFactory.BuildEntity(dto, contentType);
 
                 // need template
                 if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
@@ -886,7 +967,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
                 .ToDictionary(x => x.Id, x => x);
 
             // load all properties for all documents from database in 1 query
-            var propertyData = GetPropertyCollection(sql, defs);
+            var propertyData = GetPropertyCollection(sqlIds, defs);
 
             // assign
             var dtoIndex = 0;
@@ -906,7 +987,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
             return content;
@@ -923,8 +1004,7 @@ order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder";
         {
             var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
 
-            var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var content = factory.BuildEntity(dto);
+            var content = ContentFactory.BuildEntity(dto, contentType);
 
             //Check if template id is set on DocumentDto, and get ITemplate if it is.
             if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)

src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs

@@ -62,17 +62,27 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected override IEnumerable<EntityContainer> PerformGetAll(params int[] ids)
         {
-            //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
-            return ids.InGroupsOf(2000).SelectMany(@group =>
+            if (ids.Any())
+            {
+                //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
+                return ids.InGroupsOf(2000).SelectMany(@group =>
+                {
+                    var sql = GetBaseQuery(false)
+                        .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId})
+                        .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new {ids = @group});
+
+                    sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
+
+                    return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
+                });
+            }
+            else
             {
                 var sql = GetBaseQuery(false)
-                    .Where("nodeObjectType=@umbracoObjectTypeId", new { umbracoObjectTypeId = NodeObjectTypeId })
-                    .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new { ids = @group });
-
+                    .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId});
                 sql.OrderBy<NodeDto>(x => x.Level, SqlSyntax);
-
                 return Database.Fetch<NodeDto>(sql).Select(CreateEntity);
-            });
+            }
         }
 
         protected override IEnumerable<EntityContainer> PerformGetByQuery(IQuery<EntityContainer> query)

src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs

@@ -96,5 +96,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IContent> GetPagedResultsByQuery(IQuery<IContent> query, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter = null);
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs

@@ -38,15 +38,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <returns>An Enumerable list of <see cref="IMedia"/> objects</returns>
         IEnumerable<IMedia> GetPagedResultsByQuery(IQuery<IMedia> query, long pageIndex, int pageSize, out long totalRecords,
             string orderBy, Direction orderDirection, bool orderBySystemField, string filter = "");
-
-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords);
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs

@@ -73,6 +73,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="content"></param>
         /// <param name="xml"></param>
         void AddOrUpdatePreviewXml(IMember content, Func<IMember, XElement> xml);
-
+        
     }
 }

src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs

@@ -67,5 +67,16 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="id">Id of the <see cref="TEntity"/> object to delete versions from</param>
         /// <param name="versionDate">Latest version date</param>
         void DeleteVersions(int id, DateTime versionDate);
+
+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords);
     }
 }

src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs

@@ -76,7 +76,7 @@ namespace Umbraco.Core.Persistence.Repositories
             var sqlClause = GetBaseQuery(false);
             var translator = new SqlTranslator<IMedia>(sqlClause, query);
             var sql = translator.Translate()
-                                .OrderBy<NodeDto>(x => x.SortOrder);
+                                .OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
 
             return ProcessQuery(sql);
         }
@@ -84,18 +84,23 @@ namespace Umbraco.Core.Persistence.Repositories
         #endregion
 
         #region Overrides of PetaPocoRepositoryBase<int,IMedia>
+        
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
+        {
+            var sql = new Sql();
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsContentVersion.contentId" : "*"))
+                .From<ContentVersionDto>(SqlSyntax)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax)
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
+            return sql;
+        }
 
         protected override Sql GetBaseQuery(bool isCount)
         {
-            var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<ContentVersionDto>()
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
-            return sql;
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
         }
 
         protected override string GetBaseWhereClause()
@@ -148,6 +153,11 @@ namespace Umbraco.Core.Persistence.Repositories
             var content = new IMedia[dtos.Count];
             var defs = new List<DocumentDefinition>();
 
+            //track the looked up content types, even though the content types are cached
+            // they still need to be deep cloned out of the cache and we don't want to add
+            // the overhead of deep cloning them on every item in this loop
+            var contentTypes = new Dictionary<int, IMediaType>();
+
             for (var i = 0; i < dtos.Count; i++)
             {
                 var dto = dtos[i];
@@ -165,9 +175,19 @@ namespace Umbraco.Core.Persistence.Repositories
 
                 // else, need to fetch from the database
                 // content type repository is full-cache so OK to get each one independently
-                var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
-                var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-                content[i] = factory.BuildEntity(dto);
+
+                IMediaType contentType;
+                if (contentTypes.ContainsKey(dto.ContentDto.ContentTypeId))
+                {
+                    contentType = contentTypes[dto.ContentDto.ContentTypeId];
+                }
+                else
+                {
+                    contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+                    contentTypes[dto.ContentDto.ContentTypeId] = contentType;
+                }
+                
+                content[i] = MediaFactory.BuildEntity(dto, contentType);
 
                 // need properties
                 defs.Add(new DocumentDefinition(
@@ -195,7 +215,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
-                ((Entity) cc).ResetDirtyProperties(false);
+                cc.ResetDirtyProperties(false);
             }
 
             return content;
@@ -205,26 +225,16 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = GetBaseQuery(false);
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
-            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate);
+            sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
 
             var dto = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sql).FirstOrDefault();
 
             if (dto == null)
                 return null;
 
-            var mediaType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
+            var content = CreateMediaFromDto(dto, versionId, sql);
 
-            var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
-
-            var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) });
-
-            media.Properties = properties[dto.NodeId];
-
-            //on initial construction we don't want to have dirty properties tracked
-            // http://issues.umbraco.org/issue/U4-1946
-            ((Entity)media).ResetDirtyProperties(false);
-            return media;
+            return content;
         }
 
         public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
@@ -245,7 +255,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     query = query
                         .WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
                 query = query
-                    .Where<NodeDto>(x => x.NodeId > baseId)
+                    .Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
                     .OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
                 var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
                     .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
@@ -495,37 +505,13 @@ namespace Umbraco.Core.Persistence.Repositories
                 filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
             }
 
-            return GetPagedResultsByQuery<ContentVersionDto, Models.Media>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<ContentVersionDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsContentVersion", "contentId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
 
         }
 
-        /// <summary>
-        /// Gets paged media descendants as XML by path
-        /// </summary>
-        /// <param name="path">Path starts with</param>
-        /// <param name="pageIndex">Page number</param>
-        /// <param name="pageSize">Page size</param>
-        /// <param name="totalRecords">Total records the query would return without paging</param>
-        /// <returns>A paged enumerable of XML entries of media items</returns>
-        public IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords)
-        {
-            Sql query;
-            if (path == "-1")
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE nodeObjectType = @0)", Guid.Parse(Constants.ObjectTypes.Media)).OrderBy("nodeId");
-            }
-            else
-            {
-                query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE path LIKE @0)", path.EnsureEndsWith(",%")).OrderBy("nodeId");
-            }
-            var pagedResult = Database.Page<ContentXmlDto>(pageIndex+1, pageSize, query);
-            totalRecords = pagedResult.TotalItems;
-            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
-        }
-
         /// <summary>
         /// Private method to create a media object from a ContentDto
         /// </summary>
@@ -536,9 +522,8 @@ namespace Umbraco.Core.Persistence.Repositories
         private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
         {
             var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
-
-            var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
-            var media = factory.BuildEntity(dto);
+            
+            var media = MediaFactory.BuildEntity(dto, contentType);
 
             var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
 

src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs

@@ -107,24 +107,29 @@ namespace Umbraco.Core.Persistence.Repositories
 
         #region Overrides of PetaPocoRepositoryBase<int,IMembershipUser>
 
-        protected override Sql GetBaseQuery(bool isCount)
+        protected override Sql GetBaseQuery(BaseQueryType queryType)
         {
             var sql = new Sql();
-            sql.Select(isCount ? "COUNT(*)" : "*")
-                .From<MemberDto>()
-                .InnerJoin<ContentVersionDto>()
-                .On<ContentVersionDto, MemberDto>(left => left.NodeId, right => right.NodeId)
-                .InnerJoin<ContentDto>()
-                .On<ContentVersionDto, ContentDto>(left => left.NodeId, right => right.NodeId)
+            sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsMember.nodeId" : "*"))
+                .From<MemberDto>(SqlSyntax)
+                .InnerJoin<ContentVersionDto>(SqlSyntax)
+                .On<ContentVersionDto, MemberDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .InnerJoin<ContentDto>(SqlSyntax)
+                .On<ContentVersionDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 //We're joining the type so we can do a query against the member type - not sure if this adds much overhead or not?
                 // the execution plan says it doesn't so we'll go with that and in that case, it might be worth joining the content
                 // types by default on the document and media repo's so we can query by content type there too.
-                .InnerJoin<ContentTypeDto>().On<ContentTypeDto, ContentDto>(left => left.NodeId, right => right.ContentTypeId)
-                .InnerJoin<NodeDto>()
-                .On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
-                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
+                .InnerJoin<ContentTypeDto>(SqlSyntax)
+                .On<ContentTypeDto, ContentDto>(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                .InnerJoin<NodeDto>(SqlSyntax)
+                .On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax);
             return sql;
+        }
 
+        protected override Sql GetBaseQuery(bool isCount)
+        {
+            return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
         }
 
         protected override string GetBaseWhereClause()
@@ -381,7 +386,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Where(GetBaseWhereClause(), new { Id = id })
                 .OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
             return ProcessQuery(sql, true);
-        }
+        }        
 
         public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
         {
@@ -617,9 +622,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString().Trim(), args.ToArray());
             }
 
-            return GetPagedResultsByQuery<MemberDto, Member>(query, pageIndex, pageSize, out totalRecords,
+            return GetPagedResultsByQuery<MemberDto>(query, pageIndex, pageSize, out totalRecords,
                 new Tuple<string, string>("cmsMember", "nodeId"),
-                sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField,
+                (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
                 filterCallback);
         }
 

src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Configuration.UmbracoSettings;
 using Umbraco.Core.Logging;
@@ -25,8 +27,6 @@ using Umbraco.Core.IO;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
-    using SqlSyntax;
-
     internal abstract class VersionableRepositoryBase<TId, TEntity> : PetaPocoRepositoryBase<TId, TEntity>
         where TEntity : class, IAggregateRoot
     {
@@ -137,6 +137,40 @@ namespace Umbraco.Core.Persistence.Repositories
 
         #endregion
 
+        /// <summary>
+        /// Gets paged document descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="orderBy"></param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public virtual IEnumerable<XElement> GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords)
+        {
+            var query = new Sql().Select(string.Format("umbracoNode.id, cmsContentXml.{0}", SqlSyntax.GetQuotedColumnName("xml")))
+                .From("umbracoNode")
+                .InnerJoin("cmsContentXml").On("cmsContentXml.nodeId = umbracoNode.id");
+
+            if (path == "-1")
+            {
+                query.Where("umbracoNode.nodeObjectType = @type", new { type = NodeObjectTypeId });
+            }
+            else
+            {
+                query.Where(string.Format("umbracoNode.{0} LIKE (@0)", SqlSyntax.GetQuotedColumnName("path")), path.EnsureEndsWith(",%"));
+            }
+
+            //each order by param needs to be in a bracket! see: https://github.com/toptensoftware/PetaPoco/issues/177
+            query.OrderBy(orderBy == null
+                ? "(umbracoNode.id)"                
+                : string.Join(",", orderBy.Select(x => string.Format("({0})", SqlSyntax.GetQuotedColumnName(x)))));
+
+            var pagedResult = Database.Page<ContentXmlDto>(pageIndex + 1, pageSize, query);
+            totalRecords = pagedResult.TotalItems;
+            return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml));
+        }
+
         public int CountDescendants(int parentId, string contentTypeAlias = null)
         {
             var pathMatch = parentId == -1
@@ -348,12 +382,8 @@ namespace Umbraco.Core.Persistence.Repositories
                                 ON CustomPropData.CustomPropValContentId = umbracoNode.id
                 ", sortedInt, sortedDecimal, sortedDate, sortedString, nodeIdSelect.Item2, nodeIdSelect.Item1, versionQuery, sortedSql.Arguments.Length, newestQuery);
 
-                //insert this just above the first LEFT OUTER JOIN (for cmsDocument) or the last WHERE (everything else)
-                string newSql;
-                if (nodeIdSelect.Item1 == "cmsDocument")
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.IndexOf("LEFT OUTER JOIN"), outerJoinTempTable);
-                else
-                    newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);
+                //insert this just above the last WHERE
+                string newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable);
 
                 var newArgs = sortedSql.Arguments.ToList();
                 newArgs.Add(orderBy);
@@ -370,11 +400,14 @@ namespace Umbraco.Core.Persistence.Repositories
                 }
             }
 
-            //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
-            // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
-            // is empty for many nodes)
-            // see: http://issues.umbraco.org/issue/U4-8831
-            sortedSql.OrderBy("umbracoNode.id");
+            if (orderBySystemField && orderBy != "umbracoNode.id")
+            {
+                //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique.
+                // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column
+                // is empty for many nodes)
+                // see: http://issues.umbraco.org/issue/U4-8831
+                sortedSql.OrderBy("umbracoNode.id");
+            }
 
             return sortedSql;
 
@@ -384,7 +417,6 @@ namespace Umbraco.Core.Persistence.Repositories
         /// A helper method for inheritors to get the paged results by query in a way that minimizes queries
         /// </summary>
         /// <typeparam name="TDto">The type of the d.</typeparam>
-        /// <typeparam name="TContentBase">The 'true' entity type (i.e. Content, Member, etc...)</typeparam>
         /// <param name="query">The query.</param>
         /// <param name="pageIndex">Index of the page.</param>
         /// <param name="pageSize">Size of the page.</param>
@@ -397,34 +429,30 @@ namespace Umbraco.Core.Persistence.Repositories
         /// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
         /// <returns></returns>
         /// <exception cref="System.ArgumentNullException">orderBy</exception>
-        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto, TContentBase>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
+        protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
             Tuple<string, string> nodeIdSelect,
-            Func<Sql, IEnumerable<TEntity>> processQuery,
+            Func<Sql, Sql, IEnumerable<TEntity>> processQuery,
             string orderBy,
             Direction orderDirection,
             bool orderBySystemField,
             Func<Tuple<string, object[]>> defaultFilter = null)
-            where TContentBase : class, IAggregateRoot, TEntity
         {
             if (orderBy == null) throw new ArgumentNullException("orderBy");
 
-            // Get base query
-            var sqlBase = GetBaseQuery(false);
+            // Get base query for returning IDs
+            var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
+            // Get base query for returning all data
+            var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
 
             if (query == null) query = new Query<TEntity>();
-            var translator = new SqlTranslator<TEntity>(sqlBase, query);
-            var sqlQuery = translator.Translate();
-
-            // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, 
-            // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
-            // So we'll modify the SQL.
-            var sqlNodeIds = new Sql(
-                sqlQuery.SQL.Replace("SELECT *", string.Format("SELECT {0}.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2)),
-                sqlQuery.Arguments);
+            var translatorIds = new SqlTranslator<TEntity>(sqlBaseIds, query);
+            var sqlQueryIds = translatorIds.Translate();
+            var translatorFull = new SqlTranslator<TEntity>(sqlBaseFull, query);
+            var sqlQueryFull = translatorFull.Translate();
 
             //get sorted and filtered sql
             var sqlNodeIdsWithSort = GetSortedSqlForPagedResults(
-                GetFilteredSqlForPagedResults(sqlNodeIds, defaultFilter),
+                GetFilteredSqlForPagedResults(sqlQueryIds, defaultFilter),
                 orderDirection, orderBy, orderBySystemField, nodeIdSelect);
 
             // Get page of results and total count
@@ -440,31 +468,25 @@ namespace Umbraco.Core.Persistence.Repositories
                 //Crete the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
                 var args = sqlNodeIdsWithSort.Arguments;
                 string sqlStringCount, sqlStringPage;
-                Database.BuildPageQueries<TDto>(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage);
+                Database.BuildPageQueries<TDto>(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage);               
 
-                //if this is for sql server, the sqlPage will start with a SELECT * but we don't want that, we only want to return the nodeId                
-                sqlStringPage = sqlStringPage
-                    .Replace("SELECT *",
-                        //This ensures we only take the field name of the node id select and not the table name - since the resulting select
-                        // will ony work with the field name.
-                        "SELECT " + nodeIdSelect.Item2);
-
-                //We need to make this an inner join on the paged query
-                var splitQuery = sqlQuery.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
-                var withInnerJoinSql = new Sql(splitQuery[0])
+                //We need to make this FULL query an inner join on the paged ID query
+                var splitQuery = sqlQueryFull.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None);
+                var fullQueryWithPagedInnerJoin = new Sql(splitQuery[0])
                     .Append("INNER JOIN (")
                     //join the paged query with the paged query arguments
                     .Append(sqlStringPage, args)
                     .Append(") temp ")
                     .Append(string.Format("ON {0}.{1} = temp.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2))
                     //add the original where clause back with the original arguments
-                    .Where(splitQuery[1], sqlQuery.Arguments);
+                    .Where(splitQuery[1], sqlQueryIds.Arguments);
 
                 //get sorted and filtered sql
                 var fullQuery = GetSortedSqlForPagedResults(
-                    GetFilteredSqlForPagedResults(withInnerJoinSql, defaultFilter),
+                    GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter),
                     orderDirection, orderBy, orderBySystemField, nodeIdSelect);
-                return processQuery(fullQuery);
+
+                return processQuery(fullQuery, sqlNodeIdsWithSort);
             }
             else
             {
@@ -476,9 +498,9 @@ namespace Umbraco.Core.Persistence.Repositories
 
         protected IDictionary<int, PropertyCollection> GetPropertyCollection(
             Sql docSql,
-            IEnumerable<DocumentDefinition> documentDefs)
+            IReadOnlyCollection<DocumentDefinition> documentDefs)
         {
-            if (documentDefs.Any() == false) return new Dictionary<int, PropertyCollection>();
+            if (documentDefs.Count == 0) return new Dictionary<int, PropertyCollection>();
 
             //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use 
             // the statement to go get the property data for all of the items by using an inner join
@@ -489,23 +511,12 @@ namespace Umbraco.Core.Persistence.Repositories
                 parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
             }
 
-            var propSql = new Sql(@"SELECT cmsPropertyData.*
-FROM cmsPropertyData
-INNER JOIN cmsPropertyType
-ON cmsPropertyData.propertytypeid = cmsPropertyType.id
-INNER JOIN 
-	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
-ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-LEFT OUTER JOIN cmsDataTypePreValues
-ON cmsPropertyType.dataTypeId = cmsDataTypePreValues.datatypeNodeId", docSql.Arguments);
-
-            var allPropertyData = Database.Fetch<PropertyDataDto>(propSql);
-
-            //This is a lazy access call to get all prevalue data for the data types that make up all of these properties which we use
-            // below if any property requires tag support
-            var allPreValues = new Lazy<IEnumerable<DataTypePreValueDto>>(() =>
-            {
-                var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
+            //This retrieves all pre-values for all data types that are referenced for all property types
+            // that exist in the data set.
+            //Benchmarks show that eagerly loading these so that we can lazily read the property data 
+            // below (with the use of Query intead of Fetch) go about 30% faster, so we'll eagerly load
+            // this now since we cannot execute another reader inside of reading the property data.
+            var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId
 FROM cmsDataTypePreValues a
 WHERE EXISTS(
     SELECT DISTINCT b.id as preValIdInner
@@ -517,25 +528,85 @@ WHERE EXISTS(
     ON cmsPropertyType.contentTypeId = docData.contentType
     WHERE a.id = b.id)", docSql.Arguments);
 
-                return Database.Fetch<DataTypePreValueDto>(preValsSql);
-            });
+            var allPreValues = Database.Fetch<DataTypePreValueDto>(preValsSql);
+
+            //It's Important with the sort order here! We require this to be sorted by node id,
+            // this is required because this data set can be huge depending on the page size. Due
+            // to it's size we need to be smart about iterating over the property values to build 
+            // the document. Before we used to use Linq to get the property data for a given content node 
+            // and perform a Distinct() call. This kills performance because that would mean if we had 7000 nodes
+            // and on each iteration we will perform a lookup on potentially 100,000 property rows against the node
+            // id which turns out to be a crazy amount of iterations. Instead we know it's sorted by this value we'll 
+            // keep an index stored of the rows being read so we never have to re-iterate the entire data set
+            // on each document iteration.
+            var propSql = new Sql(@"SELECT cmsPropertyData.*
+FROM cmsPropertyData
+INNER JOIN cmsPropertyType
+ON cmsPropertyData.propertytypeid = cmsPropertyType.id
+INNER JOIN 
+	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
+ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
+ORDER BY contentNodeId, propertytypeid
+", docSql.Arguments);
+
+            //This does NOT fetch all data into memory in a list, this will read
+            // over the records as a data reader, this is much better for performance and memory,
+            // but it means that during the reading of this data set, nothing else can be read
+            // from SQL server otherwise we'll get an exception.
+            var allPropertyData = Database.Query<PropertyDataDto>(propSql);
 
             var result = new Dictionary<int, PropertyCollection>();
-
             var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
+            //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
+            var resolvedCompositionProperties = new Dictionary<int, PropertyType[]>();
 
-            //iterate each definition grouped by it's content type - this will mean less property type iterations while building 
-            // up the property collections
-            foreach (var compositionGroup in documentDefs.GroupBy(x => x.Composition))
+            //keep track of the current property data item being enumerated
+            var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
+
+            try
             {
-                var compositionProperties = compositionGroup.Key.CompositionPropertyTypes.ToArray();
-
-                foreach (var def in compositionGroup)
+                //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
+                // which allows us to more efficiently iterate over the large data set of property values
+                foreach (var def in documentDefs.OrderBy(x => x.Id))
                 {
-                    var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct();
+                    //get the resolved proeprties from our local cache, or resolve them and put them in cache
+                    PropertyType[] compositionProperties;
+                    if (resolvedCompositionProperties.ContainsKey(def.Composition.Id))
+                    {
+                        compositionProperties = resolvedCompositionProperties[def.Composition.Id];
+                    }
+                    else
+                    {
+                        compositionProperties = def.Composition.CompositionPropertyTypes.ToArray();
+                        resolvedCompositionProperties[def.Composition.Id] = compositionProperties;
+                    }
 
-                    var propertyFactory = new PropertyFactory(compositionProperties, def.Version, def.Id, def.CreateDate, def.VersionDate);
-                    var properties = propertyFactory.BuildEntity(propertyDataDtos.ToArray()).ToArray();
+                    var propertyDataDtos = new List<PropertyDataDto>();
+
+                    //Check if there is a current enumerated item and check if we match and add it
+                    if (propertyDataSetEnumerator.Current != null)
+                    {
+                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        {
+                            propertyDataDtos.Add(propertyDataSetEnumerator.Current);
+                        }
+                    }
+                    //Move to the next position, see if we match and add it, if not exit
+                    while (propertyDataSetEnumerator.MoveNext())
+                    {
+                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        {
+                            propertyDataDtos.Add(propertyDataSetEnumerator.Current);
+                        }
+                        else
+                        {
+                            //the node id has changed so we need to exit the loop,
+                            //the enumerator position will be maintained
+                            break;
+                        }
+                    }
+
+                    var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray();
 
                     foreach (var property in properties)
                     {
@@ -552,7 +623,7 @@ WHERE EXISTS(
                             propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport;
 
                             //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up
-                            var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
+                            var preValData = allPreValues.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
                                 .Distinct()
                                 .ToArray();
 
@@ -575,8 +646,13 @@ WHERE EXISTS(
                     result[def.Id] = new PropertyCollection(properties);
                 }
             }
+            finally
+            {
+                propertyDataSetEnumerator.Dispose();
+            }
 
             return result;
+
         }
 
         public class DocumentDefinition
@@ -688,5 +764,12 @@ WHERE EXISTS(
 
             return allsuccess;
         }
+
+        /// <summary>
+        /// For Paging, repositories must support returning different query for the query type specified
+        /// </summary>
+        /// <param name="queryType"></param>
+        /// <returns></returns>
+        protected abstract Sql GetBaseQuery(BaseQueryType queryType);
     }
 }

src/Umbraco.Core/Services/ContentService.cs

@@ -585,7 +585,7 @@ namespace Umbraco.Core.Services
 
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
-        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             long total;
             var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter);
@@ -604,7 +604,7 @@ namespace Umbraco.Core.Services
         /// <param name="orderDirection">Direction to order by</param>
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns> 
-        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter);
         }
@@ -1764,6 +1764,31 @@ namespace Umbraco.Core.Services
             return true;
         }
 
+        /// <summary>
+        /// Gets paged content descendants as XML by path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        public IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords)
+        {
+            Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
+            Mandate.ParameterCondition(pageSize > 0, "pageSize");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateContentRepository(uow))
+            {
+                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize,
+                    //This order by is VERY important! This allows us to figure out what is implicitly not published, see ContentRepository.BuildXmlCache and
+                    // UmbracoContentIndexer.PerformIndexAll which uses the logic based on this sort order
+                    new[] {"level", "parentID", "sortOrder"},
+                    out totalRecords);
+                return contents;
+            }
+        }
+
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -2192,6 +2217,11 @@ namespace Umbraco.Core.Services
                 }
             }
 
+            if (string.IsNullOrWhiteSpace(content.Name))
+            {
+                throw new ArgumentException("Cannot save content with empty name.");
+            }
+
             using (new WriteLock(Locker))
             {
                 var uow = UowProvider.GetUnitOfWork();

src/Umbraco.Core/Services/ContentTypeService.cs

@@ -719,8 +719,13 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Optional id of the user saving the ContentType</param>
         public void Save(IContentType contentType, int userId = 0)
         {
-	        if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
-				return;
+            if (SavingContentType.IsRaisedEventCancelled(new SaveEventArgs<IContentType>(contentType), this))
+                return;
+
+            if (string.IsNullOrWhiteSpace(contentType.Name))
+            {
+                throw new ArgumentException("Cannot save content type with empty name.");
+            }
 
             using (new WriteLock(Locker))
             {

src/Umbraco.Core/Services/DataTypeService.cs

@@ -361,8 +361,13 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Id of the user issueing the save</param>
         public void Save(IDataTypeDefinition dataTypeDefinition, int userId = 0)
         {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
-				return;
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IDataTypeDefinition>(dataTypeDefinition), this)) 
+                return;
+
+            if (string.IsNullOrWhiteSpace(dataTypeDefinition.Name))
+            {
+                throw new ArgumentException("Cannot save datatype with empty name.");
+            }
 
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateDataTypeDefinitionRepository(uow))

src/Umbraco.Core/Services/IContentService.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Xml;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -95,6 +96,19 @@ namespace Umbraco.Core.Services
     /// </summary>
     public interface IContentService : IService
     {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table based on the given path
+        /// </summary>
+        /// <param name="path">Path starts with</param>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>
+        /// <remarks>
+        /// If -1 is passed, then this will return all content xml entries, otherwise will return all descendents from the path
+        /// </remarks>
+        IEnumerable<XElement> GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords);
+
         /// <summary>
         /// This builds the Xml document used for the XML cache
         /// </summary>
@@ -245,7 +259,7 @@ namespace Umbraco.Core.Services
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IContent> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IContent"/> objects by Parent Id
@@ -259,7 +273,7 @@ namespace Umbraco.Core.Services
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IContent"/> objects by Parent Id

src/Umbraco.Core/Services/IMediaService.cs

@@ -169,7 +169,7 @@ namespace Umbraco.Core.Services
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
         IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IMedia"/> objects by Parent Id
@@ -183,7 +183,7 @@ namespace Umbraco.Core.Services
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
         IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
-            string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "");
+            string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "");
 
         /// <summary>
         /// Gets a collection of <see cref="IMedia"/> objects by Parent Id

src/Umbraco.Core/Services/IMemberService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Xml.Linq;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
@@ -13,6 +14,15 @@ namespace Umbraco.Core.Services
     /// </summary>
     public interface IMemberService : IMembershipMemberService
     {
+        /// <summary>
+        /// Gets all XML entries found in the cmsContentXml table
+        /// </summary>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of content items</returns>        
+        IEnumerable<XElement> GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords);
+
         /// <summary>
         /// Rebuilds all xml content in the cmsContentXml table for all documents
         /// </summary>

src/Umbraco.Core/Services/MacroService.cs

@@ -142,19 +142,24 @@ namespace Umbraco.Core.Services
         /// <param name="userId">Optional Id of the user deleting the macro</param>
         public void Save(IMacro macro, int userId = 0)
         {
-	        if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this)) 
-				return;
-	        
-			var uow = UowProvider.GetUnitOfWork();
-	        using (var repository = RepositoryFactory.CreateMacroRepository(uow))
-	        {
-		        repository.AddOrUpdate(macro);
-		        uow.Commit();
+            if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMacro>(macro), this))
+                return;
 
-		        Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
-	        }
+            if (string.IsNullOrWhiteSpace(macro.Name))
+            {
+                throw new ArgumentException("Cannot save macro with empty name.");
+            }
 
-	        Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMacroRepository(uow))
+            {
+                repository.AddOrUpdate(macro);
+                uow.Commit();
+
+                Saved.RaiseEvent(new SaveEventArgs<IMacro>(macro, false), this);
+            }
+
+            Audit(AuditType.Save, "Save Macro performed by user", userId, -1);
         }
 
         ///// <summary>

src/Umbraco.Core/Services/MediaService.cs

@@ -451,7 +451,7 @@ namespace Umbraco.Core.Services
 
         [Obsolete("Use the overload with 'long' parameter types instead")]
         [EditorBrowsable(EditorBrowsableState.Never)]
-        public IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IMedia> GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             long total;
             var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter);
@@ -470,7 +470,7 @@ namespace Umbraco.Core.Services
         /// <param name="orderDirection">Direction to order by</param>
         /// <param name="filter">Search text filter</param>
         /// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
-        public IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "")
+        public IEnumerable<IMedia> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "")
         {
             return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter);
         }
@@ -837,7 +837,11 @@ namespace Umbraco.Core.Services
                 {
                     return OperationStatus.Cancelled(evtMsgs);
                 }
+            }
 
+            if (string.IsNullOrWhiteSpace(media.Name))
+            {
+                throw new ArgumentException("Cannot save media with empty name.");
             }
 
             var uow = UowProvider.GetUnitOfWork();
@@ -1224,7 +1228,7 @@ namespace Umbraco.Core.Services
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateMediaRepository(uow))
             {
-                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, out totalRecords);
+                var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, null, out totalRecords);
                 return contents;
             }
         }

src/Umbraco.Core/Services/MemberService.cs

@@ -612,6 +612,26 @@ namespace Umbraco.Core.Services
             Audit(AuditType.Publish, "MemberService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
         }
 
+        /// <summary>
+        /// Gets paged member descendants as XML by path
+        /// </summary>
+        /// <param name="pageIndex">Page number</param>
+        /// <param name="pageSize">Page size</param>
+        /// <param name="totalRecords">Total records the query would return without paging</param>
+        /// <returns>A paged enumerable of XML entries of member items</returns>
+        public IEnumerable<XElement> GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords)
+        {
+            Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
+            Mandate.ParameterCondition(pageSize > 0, "pageSize");
+
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMemberRepository(uow))
+            {
+                var contents = repository.GetPagedXmlEntriesByPath("-1", pageIndex, pageSize, null, out totalRecords);
+                return contents;
+            }
+        }
+
         #endregion
 
         #region IMembershipMemberService Implementation
@@ -978,6 +998,11 @@ namespace Umbraco.Core.Services
                 }
             }
 
+            if (string.IsNullOrWhiteSpace(entity.Name))
+            {
+                throw new ArgumentException("Cannot save member with empty name.");
+            }
+
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateMemberRepository(uow))
             {

src/Umbraco.Core/Services/MemberTypeService.cs

@@ -78,6 +78,11 @@ namespace Umbraco.Core.Services
             if (Saving.IsRaisedEventCancelled(new SaveEventArgs<IMemberType>(memberType), this))
                 return;
 
+            if (string.IsNullOrWhiteSpace(memberType.Name))
+            {
+                throw new ArgumentException("Cannot save MemberType with empty name.");
+            }
+
             using (new WriteLock(Locker))
             {
                 var uow = UowProvider.GetUnitOfWork();

src/Umbraco.Core/Services/UserService.cs

@@ -125,6 +125,11 @@ namespace Umbraco.Core.Services
         {
             if (userType == null) throw new ArgumentNullException("userType");
 
+            if (string.IsNullOrWhiteSpace(username))
+            {
+                throw new ArgumentException("Cannot create user with empty username.");
+            }
+
             //TODO: PUT lock here!!
 
             var uow = UowProvider.GetUnitOfWork();
@@ -312,6 +317,15 @@ namespace Umbraco.Core.Services
                     return;
             }
 
+            if (string.IsNullOrWhiteSpace(entity.Username))
+            {
+                throw new ArgumentException("Cannot save user with empty username.");
+            }
+            if (string.IsNullOrWhiteSpace(entity.Name))
+            {
+                throw new ArgumentException("Cannot save user with empty name.");
+            }
+
             var uow = UowProvider.GetUnitOfWork();
             using (var repository = RepositoryFactory.CreateUserRepository(uow))
             {
@@ -353,6 +367,14 @@ namespace Umbraco.Core.Services
             {
                 foreach (var member in entities)
                 {
+                    if (string.IsNullOrWhiteSpace(member.Username))
+                    {
+                        throw new ArgumentException("Cannot save user with empty username.");
+                    }
+                    if (string.IsNullOrWhiteSpace(member.Name))
+                    {
+                        throw new ArgumentException("Cannot save user with empty name.");
+                    }
                     repository.AddOrUpdate(member);
                 }
                 //commit the whole lot in one go

src/Umbraco.Core/Sync/ColdBootType.cs

@@ -1,24 +0,0 @@
-namespace Umbraco.Core.Sync
-{
-    /// <summary>
-    /// The type of Cold Boot taking place
-    /// </summary>
-    public enum ColdBootType
-    {
-        /// <summary>
-        /// Indicates there was no cold boot
-        /// </summary>
-        NoColdBoot,
-
-        /// <summary>
-        /// This indicates that there is no lasysynced file for the current machine/appid
-        /// </summary>
-        NeverSynced,
-
-        /// <summary>
-        /// This indicates that the number of instructions needing to be processed exceeds the value stored 
-        /// for the <see cref="DatabaseServerMessengerOptions.MaxProcessingInstructionCount"/>
-        /// </summary>
-        ExceedsMaxProcessingInstructionCount
-    }
-}

src/Umbraco.Core/Sync/DatabaseServerMessenger.cs

@@ -43,11 +43,6 @@ namespace Umbraco.Core.Sync
         protected DatabaseServerMessengerOptions Options { get; private set; }
         protected ApplicationContext ApplicationContext { get { return _appContext; } }
 
-        /// <summary>
-        /// Returns the cold boot type detected for the current app domain
-        /// </summary>
-        public ColdBootType ColdBootType { get; private set; }
-
         public DatabaseServerMessenger(ApplicationContext appContext, bool distributedEnabled, DatabaseServerMessengerOptions options)
             : base(distributedEnabled)
         {
@@ -151,12 +146,11 @@ namespace Umbraco.Core.Sync
         /// </remarks>
         private void Initialize()
         {
-            ColdBootType = ColdBootType.NoColdBoot;
-
             lock (_locko)
             {
                 if (_released) return;
-                
+
+                var coldboot = false;
                 if (_lastId < 0) // never synced before
                 {
                     // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
@@ -165,7 +159,7 @@ namespace Umbraco.Core.Sync
                         + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
                         + " the database and maintain cache updates based on that Id.");
 
-                    ColdBootType = ColdBootType.NeverSynced;
+                    coldboot = true;
                 }
                 else
                 {
@@ -181,11 +175,11 @@ namespace Umbraco.Core.Sync
                             + " to the latest found in the database and maintain cache updates based on that Id.",
                             () => count, () => Options.MaxProcessingInstructionCount);
 
-                        ColdBootType = ColdBootType.ExceedsMaxProcessingInstructionCount;
+                        coldboot = true;
                     }
                 }
 
-                if (ColdBootType != ColdBootType.NoColdBoot)
+                if (coldboot)
                 {
                     // go get the last id in the db and store it
                     // note: do it BEFORE initializing otherwise some instructions might get lost

src/Umbraco.Core/Umbraco.Core.csproj

@@ -466,6 +466,7 @@
     <Compile Include="Persistence\RecordPersistenceType.cs" />
     <Compile Include="Persistence\Relators\AccessRulesRelator.cs" />
     <Compile Include="Persistence\Repositories\AuditRepository.cs" />
+    <Compile Include="Persistence\Repositories\BaseQueryType.cs" />
     <Compile Include="Persistence\Repositories\EntityContainerRepository.cs" />
     <Compile Include="Persistence\Repositories\DomainRepository.cs" />
     <Compile Include="Persistence\Repositories\ExternalLoginRepository.cs" />
@@ -1311,7 +1312,6 @@
     <Compile Include="Strings\ContentBaseExtensions.cs" />
     <Compile Include="Strings\Diff.cs" />
     <Compile Include="Sync\BatchedWebServiceServerMessenger.cs" />
-    <Compile Include="Sync\ColdBootType.cs" />
     <Compile Include="Sync\IServerRegistrar2.cs" />
     <Compile Include="Sync\ServerRole.cs" />
     <Compile Include="Sync\DatabaseServerMessenger.cs" />

src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs

@@ -172,6 +172,38 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Can_Get_All_Containers()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            EntityContainer container1, container2, container3;
+            using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid))
+            {
+                container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container1" };
+                containerRepository.AddOrUpdate(container1);
+                container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container2" };
+                containerRepository.AddOrUpdate(container2);
+                container3 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container3" };
+                containerRepository.AddOrUpdate(container3);
+                unitOfWork.Commit();
+                Assert.That(container1.Id, Is.GreaterThan(0));
+                Assert.That(container2.Id, Is.GreaterThan(0));
+                Assert.That(container3.Id, Is.GreaterThan(0));
+            }
+            using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid))
+            {
+                var found1 = containerRepository.Get(container1.Id);
+                Assert.IsNotNull(found1);
+                var found2 = containerRepository.Get(container2.Id);
+                Assert.IsNotNull(found2);
+                var found3 = containerRepository.Get(container3.Id);
+                Assert.IsNotNull(found3);
+                var allContainers = containerRepository.GetAll();
+                Assert.AreEqual(3, allContainers.Count());
+            }
+        }
+
         [Test]
         public void Can_Delete_Container()
         {

src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
+using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Documents;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using Moq;
 using NUnit.Framework;
@@ -31,12 +33,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Children_Are_Sorted()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
 
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
                 Assert.IsNotNull(result);
                 Assert.AreEqual(1, result.TotalItemCount);
@@ -60,12 +65,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Result_Has_All_Values()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
 
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
                 Assert.IsNotNull(result);
                 Assert.AreEqual(1, result.TotalItemCount);

src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs

@@ -27,6 +27,7 @@ using UmbracoExamine;
 using UmbracoExamine.DataServices;
 using umbraco.BusinessLogic;
 using System.Linq;
+using Lucene.Net.Index;
 
 namespace Umbraco.Tests.PublishedContent
 {
@@ -106,11 +107,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Children_Sorted_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -135,11 +139,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Do_Not_Find_In_Recycle_Bin()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -175,11 +182,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Children_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -197,11 +207,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Descendants_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -219,11 +232,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void DescendantsOrSelf_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var ctx = GetUmbracoContext("/test", 1234);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
@@ -241,12 +257,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ancestors_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
                 var ctx = GetUmbracoContext("/test", 1234);
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
                 //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace
@@ -260,12 +279,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void AncestorsOrSelf_With_Examine()
         {
-            using (var luceneDir = new RAMDirectory())
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
             {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
+                //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
                 indexer.RebuildIndex();
                 var ctx = GetUmbracoContext("/test", 1234);
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
                 var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
 
                 //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace

src/Umbraco.Tests/Services/ContentServiceTests.cs

@@ -668,6 +668,17 @@ namespace Umbraco.Tests.Services
             Assert.Throws<Exception>(() => contentService.CreateContent("Test", -1, "umbAliasDoesntExist"));
         }
 
+        [Test]
+        public void Cannot_Save_Content_With_Empty_Name()
+        {
+            // Arrange
+            var contentService = ServiceContext.ContentService;
+            var content = new Content(string.Empty, -1, ServiceContext.ContentTypeService.GetContentType("umbTextpage"));
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => contentService.Save(content));
+        }
+
         [Test]
         public void Can_Get_Content_By_Id()
         {

src/Umbraco.Tests/Services/ContentTypeServiceTests.cs

@@ -304,6 +304,42 @@ namespace Umbraco.Tests.Services
             Assert.IsNull(deletedContentType);
         }
 
+        [Test]
+        public void Can_Create_Container()
+        {
+            // Arrange
+            var cts = ServiceContext.ContentTypeService;
+
+            // Act
+            var container = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container.Name = "container1";
+            cts.SaveContentTypeContainer(container);
+
+            // Assert
+            var createdContainer = cts.GetContentTypeContainer(container.Id);
+            Assert.IsNotNull(createdContainer);
+        }
+
+        [Test]
+        public void Can_Get_All_Containers()
+        {
+            // Arrange
+            var cts = ServiceContext.ContentTypeService;
+
+            // Act
+            var container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container1.Name = "container1";
+            cts.SaveContentTypeContainer(container1);
+
+            var container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid);
+            container2.Name = "container2";
+            cts.SaveContentTypeContainer(container2);
+
+            // Assert
+            var containers = cts.GetContentTypeContainers(new int[0]);
+            Assert.AreEqual(2, containers.Count());
+        }
+
         [Test]
         public void Deleting_ContentType_Sends_Correct_Number_Of_DeletedEntities_In_Events()
         {
@@ -717,6 +753,16 @@ namespace Umbraco.Tests.Services
             Assert.DoesNotThrow(() => service.GetContentType("advancedPage"));
         }
 
+        [Test]
+        public void Cannot_Save_ContentType_With_Empty_Name()
+        {
+            // Arrange
+            var contentType = MockedContentTypes.CreateSimpleContentType("contentType", string.Empty);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.ContentTypeService.Save(contentType));
+        }
+
         [Test]
         public void Cannot_Rename_PropertyType_Alias_On_Composition_Which_Would_Cause_Conflict_In_Other_Composition()
         {

src/Umbraco.Tests/Services/DataTypeServiceTests.cs

@@ -221,5 +221,18 @@ namespace Umbraco.Tests.Services
             Assert.AreEqual("preVal1", preVals.PreValuesAsArray.First().Value);
             Assert.AreEqual("preVal2", preVals.PreValuesAsArray.Last().Value);            
         }
+
+        [Test]
+        public void Cannot_Save_DataType_With_Empty_Name()
+        {
+            // Arrange
+            var dataTypeService = ServiceContext.DataTypeService;
+
+            // Act
+            var dataTypeDefinition = new DataTypeDefinition(-1, "Test.TestEditor") { Name = string.Empty, DatabaseType = DataTypeDatabaseType.Ntext };
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => dataTypeService.Save(dataTypeDefinition));
+        }
     }
 }

src/Umbraco.Tests/Services/MacroServiceTests.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System;
+using System.Linq;
 using Moq;
 using NUnit.Framework;
 using Umbraco.Core;
@@ -217,6 +218,17 @@ namespace Umbraco.Tests.Services
 
         }
 
+        [Test]
+        public void Cannot_Save_Macro_With_Empty_Name()
+        {
+            // Arrange
+            var macroService = ServiceContext.MacroService;
+            var macro = new Macro("test", string.Empty, scriptPath: "~/Views/MacroPartials/Test.cshtml", cacheDuration: 1234);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => macroService.Save(macro));
+        }
+
         //[Test]
         //public void Can_Get_Many_By_Alias()
         //{

src/Umbraco.Tests/Services/MediaServiceTests.cs

@@ -81,6 +81,19 @@ namespace Umbraco.Tests.Services
             Assert.That(mediaChild.Trashed, Is.False);
         }
 
+        [Test]
+        public void Cannot_Save_Media_With_Empty_Name()
+        {
+            // Arrange
+            var mediaService = ServiceContext.MediaService;
+            var mediaType = MockedContentTypes.CreateVideoMediaType();
+            ServiceContext.ContentTypeService.Save(mediaType);
+            var media = mediaService.CreateMedia(string.Empty, -1, "video");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => mediaService.Save(media));
+        }
+
         [Test]
         public void Ensure_Content_Xml_Created()
         {

src/Umbraco.Tests/Services/MemberServiceTests.cs

@@ -185,6 +185,18 @@ namespace Umbraco.Tests.Services
             Assert.AreEqual(2, membersInRole.Count());
         }
 
+        [Test]
+        public void Cannot_Save_Member_With_Empty_Name()
+        {
+            IMemberType memberType = MockedContentTypes.CreateSimpleMemberType();
+            ServiceContext.MemberTypeService.Save(memberType);
+            IMember member = MockedMember.CreateSimpleMember(memberType, string.Empty, "test@test.com", "pass", "test");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.MemberService.Save(member));
+            
+        }
+
         [TestCase("MyTestRole1", "test1", StringPropertyMatchType.StartsWith, 1)]
         [TestCase("MyTestRole1", "test", StringPropertyMatchType.StartsWith, 3)]
         [TestCase("MyTestRole1", "test1", StringPropertyMatchType.Exact, 1)]

src/Umbraco.Tests/Services/MemberTypeServiceTests.cs

@@ -173,6 +173,16 @@ namespace Umbraco.Tests.Services
             }
         }
 
+        [Test]
+        public void Cannot_Save_MemberType_With_Empty_Name()
+        {
+            // Arrange
+            IMemberType memberType = MockedContentTypes.CreateSimpleMemberType("memberTypeAlias", string.Empty);
+            
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => ServiceContext.MemberTypeService.Save(memberType));
+        }
+
         //[Test]
         //public void Can_Save_MemberType_Structure_And_Create_A_Member_Based_On_It()
         //{

src/Umbraco.Tests/Services/UserServiceTests.cs

@@ -490,6 +490,43 @@ namespace Umbraco.Tests.Services
             Assert.IsTrue(result4.AllowedSections.Contains("test"));
         }
 
+        [Test]
+        public void Cannot_Create_User_With_Empty_Username()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.CreateUserWithIdentity(string.Empty, "john@umbraco.io", userType));
+        }
+
+        [Test]
+        public void Cannot_Save_User_With_Empty_Username()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+            var user = userService.CreateUserWithIdentity("John Doe", "john@umbraco.io", userType);
+            user.Username = string.Empty;
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.Save(user));
+        }
+
+        [Test]
+        public void Cannot_Save_User_With_Empty_Name()
+        {
+            // Arrange
+            var userService = ServiceContext.UserService;
+            var userType = userService.GetUserTypeByAlias("admin");
+            var user = userService.CreateUserWithIdentity("John Doe", "john@umbraco.io", userType);
+            user.Name = string.Empty;
+
+            // Act & Assert
+            Assert.Throws<ArgumentException>(() => userService.Save(user));
+        }
+
         [Test]
         public void Get_By_Profile_Username()
         {

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -57,8 +57,9 @@
     <Reference Include="AutoMapper.Net4, Version=3.3.1.0, Culture=neutral, PublicKeyToken=be96cd2c38ef1005, processorArchitecture=MSIL">
       <HintPath>..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.80.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.80\lib\net45\Examine.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
@@ -163,6 +164,8 @@
     <Compile Include="Routing\NiceUrlRoutesTests.cs" />
     <Compile Include="TestHelpers\Entities\MockedPropertyTypes.cs" />
     <Compile Include="TryConvertToTests.cs" />
+    <Compile Include="UmbracoExamine\RandomIdRAMDirectory.cs" />
+    <Compile Include="UmbracoExamine\UmbracoContentIndexerTests.cs" />
     <Compile Include="Web\AngularIntegration\AngularAntiForgeryTests.cs" />
     <Compile Include="Web\AngularIntegration\ContentModelSerializationTests.cs" />
     <Compile Include="Web\AngularIntegration\JsInitializationTests.cs" />

src/Umbraco.Tests/UmbracoExamine/EventsTest.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
 using Examine;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using NUnit.Framework;
 using Umbraco.Tests.TestHelpers;
@@ -15,53 +17,39 @@ namespace Umbraco.Tests.UmbracoExamine
 		[Test]
 		public void Events_Ignoring_Node()
 		{
-			//change the parent id so that they are all ignored
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				999); //change to 999
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+		    using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+		    {
+                //change the parent id so that they are all ignored
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    999); //change to 999
 
-			var isIgnored = false;
+                var isIgnored = false;
 
-			EventHandler<IndexingNodeDataEventArgs> ignoringNode = (s, e) =>
-				{
-					isIgnored = true;
-				};
+                EventHandler<IndexingNodeDataEventArgs> ignoringNode = (s, e) =>
+                {
+                    isIgnored = true;
+                };
 
-			_indexer.IgnoringNode += ignoringNode;
+                indexer.IgnoringNode += ignoringNode;
 
-			//get a node from the data repo
-			var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
-			                          .Root
-			                          .Elements()
-			                          .First();
+                //get a node from the data repo
+                var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
+                                          .Root
+                                          .Elements()
+                                          .First();
 
-			_indexer.ReIndexNode(node, IndexTypes.Content);
+                indexer.ReIndexNode(node, IndexTypes.Content);
 
 
-			Assert.IsTrue(isIgnored);
-
+                Assert.IsTrue(isIgnored);
+            }
 		}
 
 		private readonly TestContentService _contentService = new TestContentService();
-		private static UmbracoExamineSearcher _searcher;
-		private static UmbracoContentIndexer _indexer;
-		private Lucene.Net.Store.Directory _luceneDir;
-
-	    public override void Initialize()
-	    {
-	        base.Initialize();
-
-            _luceneDir = new RAMDirectory();
-            _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
-            _indexer.RebuildIndex();
-            _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
-        }
-
-	    public override void TearDown()
-	    {
-	        base.TearDown();
-            _luceneDir.Dispose();
-        }
         
 	}
 }

src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs

@@ -7,6 +7,7 @@ using Examine.LuceneEngine.Config;
 using Examine.LuceneEngine.Providers;
 using Lucene.Net.Analysis;
 using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using Moq;
 using Umbraco.Core;
@@ -33,7 +34,7 @@ namespace Umbraco.Tests.UmbracoExamine
     internal static class IndexInitializer
     {
         public static UmbracoContentIndexer GetUmbracoIndexer(
-                Directory luceneDir,
+                IndexWriter writer,
                 Analyzer analyzer = null,
                 IDataService dataService = null,
                 IContentService contentService = null,
@@ -41,7 +42,8 @@ namespace Umbraco.Tests.UmbracoExamine
                 IDataTypeService dataTypeService = null,
                 IMemberService memberService = null,
                 IUserService userService = null,
-                IContentTypeService contentTypeService = null)
+                IContentTypeService contentTypeService = null, 
+                bool supportUnpublishedContent = false)
         {
             if (dataService == null)
             {
@@ -177,15 +179,17 @@ namespace Umbraco.Tests.UmbracoExamine
             var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies);
 
             var i = new UmbracoContentIndexer(indexCriteria,
-                                              luceneDir, //custom lucene directory
-                                                  dataService,
-                                                  contentService,
-                                                  mediaService,
-                                                  dataTypeService,
-                                                  userService,
-                                                  contentTypeService,
-                                              analyzer,
-                                              false);
+                writer, 
+                dataService,
+                contentService,
+                mediaService,
+                dataTypeService,
+                userService,
+                contentTypeService,
+                false)
+            {
+                SupportUnpublishedContent = supportUnpublishedContent
+            };
 
             //i.IndexSecondsInterval = 1;
 
@@ -193,13 +197,14 @@ namespace Umbraco.Tests.UmbracoExamine
 
             return i;
         }
-        public static UmbracoExamineSearcher GetUmbracoSearcher(Directory luceneDir, Analyzer analyzer = null)
+
+        public static UmbracoExamineSearcher GetUmbracoSearcher(IndexWriter writer, Analyzer analyzer = null)
         {
             if (analyzer == null)
             {
                 analyzer = new StandardAnalyzer(Version.LUCENE_29);
             }
-            return new UmbracoExamineSearcher(luceneDir, analyzer);
+            return new UmbracoExamineSearcher(writer, analyzer);
         }
 
         public static LuceneSearcher GetLuceneSearcher(Directory luceneDir)

src/Umbraco.Tests/UmbracoExamine/IndexTest.cs

@@ -1,11 +1,11 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using Examine;
 using Examine.LuceneEngine;
 using Examine.LuceneEngine.Providers;
 using Examine.LuceneEngine.SearchCriteria;
 using Examine.SearchCriteria;
+using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Index;
 using Lucene.Net.Search;
 using Lucene.Net.Store;
@@ -15,218 +15,230 @@ using UmbracoExamine;
 
 namespace Umbraco.Tests.UmbracoExamine
 {
-
     /// <summary>
     /// Tests the standard indexing capabilities
     /// </summary>
     [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
     [TestFixture, RequiresSTA]
     public class IndexTest : ExamineBaseTest
-	{
+    {
+        /// <summary>
+        /// Check that the node signalled as protected in the content service is not present in the index.
+        /// </summary>
+        [Test]
+        public void Index_Protected_Content_Not_Indexed()
+        {
 
-		///// <summary>
-		/// <summary>
-		/// Check that the node signalled as protected in the content service is not present in the index.
-		/// </summary>
-		[Test]
-		public void Index_Protected_Content_Not_Indexed()
-		{
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			var protectedQuery = new BooleanQuery();
-			protectedQuery.Add(
-				new BooleanClause(
-					new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)),
-					BooleanClause.Occur.MUST));
+                var protectedQuery = new BooleanQuery();
+                protectedQuery.Add(
+                    new BooleanClause(
+                        new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)),
+                        BooleanClause.Occur.MUST));
 
-			protectedQuery.Add(
-				new BooleanClause(
-					new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, TestContentService.ProtectedNode.ToString())),
-					BooleanClause.Occur.MUST));
+                protectedQuery.Add(
+                    new BooleanClause(
+                        new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, TestContentService.ProtectedNode.ToString())),
+                        BooleanClause.Occur.MUST));
 
-			var collector = new AllHitsCollector(false, true);
-			var s = _searcher.GetSearcher();
-			s.Search(protectedQuery, collector);
+                var collector = new AllHitsCollector(false, true);
+                var s = searcher.GetSearcher();
+                s.Search(protectedQuery, collector);
 
-			Assert.AreEqual(0, collector.Count, "Protected node should not be indexed");
+                Assert.AreEqual(0, collector.Count, "Protected node should not be indexed");
+            }
 
-		}
+        }
 
-		[Test]
-		public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
-		{
-			//change parent id to 1116
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				1116);
-			
-			//rebuild so it excludes children unless they are under 1116
-			_indexer.RebuildIndex();
+        [Test]
+        public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//ensure that node 2112 doesn't exist
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(0, results.Count());
+                var mediaService = new TestMediaService();
 
-			//get a node from the data repo (this one exists underneath 2222)
-			var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
-			                        .Root
-			                        .Elements()
-			                        .Where(x => (int)x.Attribute("id") == 2112)
-			                        .First();
+                //change parent id to 1116
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    1116);
 
-            var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
-            Assert.AreEqual("-1,1111,2222,2112", currPath);
+                //rebuild so it excludes children unless they are under 1116
+                indexer.RebuildIndex();
 
-			//now mimic moving 2112 to 1116
-			//node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
-            node.SetAttributeValue("path", "-1,1116,2112");
-			node.SetAttributeValue("parentID", "1116");
+                //ensure that node 2112 doesn't exist
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
 
-			//now reindex the node, this should first delete it and then WILL add it because of the parent id constraint
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                //get a node from the data repo (this one exists underneath 2222)
+                var node = mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
+                                        .Root
+                                        .Elements()
+                                        .First(x => (int)x.Attribute("id") == 2112);
 
-			//RESET the parent id
-			existingCriteria = ((IndexCriteria)_indexer.IndexerData);
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				null);
+                var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
+                Assert.AreEqual("-1,1111,2222,2112", currPath);
 
-			//now ensure it's deleted
-			var newResults = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(1, newResults.Count());
-		}
+                //now mimic moving 2112 to 1116
+                //node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
+                node.SetAttributeValue("path", "-1,1116,2112");
+                node.SetAttributeValue("parentID", "1116");
 
-		[Test]
-        [Ignore]
-		public void Index_Move_Media_To_Non_Indexable_ParentID()
-		{
-			//get a node from the data repo (this one exists underneath 2222)
-			var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
-			                        .Root
-			                        .Elements()
-			                        .Where(x => (int)x.Attribute("id") == 2112)
-			                        .First();
+                //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint
+                indexer.ReIndexNode(node, IndexTypes.Media);
 
-            var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
-            Assert.AreEqual("-1,1111,2222,2112", currPath);
+                //RESET the parent id
+                existingCriteria = ((IndexCriteria)indexer.IndexerData);
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    null);
 
-			//ensure it's indexed
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                //now ensure it's deleted
+                var newResults = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(1, newResults.TotalItemCount);
+            }
 
-			//change the parent node id to be the one it used to exist under
-			var existingCriteria = _indexer.IndexerData;
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				2222);
+            
+        }
 
-			//now mimic moving the node underneath 1116 instead of 2222
-			node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
-			node.SetAttributeValue("parentID", "1116");
+        [Test]
+        public void Index_Move_Media_To_Non_Indexable_ParentID()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//now reindex the node, this should first delete it and then NOT add it because of the parent id constraint
-			_indexer.ReIndexNode(node, IndexTypes.Media);
+                var mediaService = new TestMediaService();
 
-			//RESET the parent id
-			existingCriteria = ((IndexCriteria)_indexer.IndexerData);
-			_indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
-				null);
+                //get a node from the data repo (this one exists underneath 2222)
+                var node = mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]")
+                                    .Root
+                                    .Elements()
+                                    .First(x => (int)x.Attribute("id") == 2112);
 
-			//now ensure it's deleted
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile());
-			Assert.AreEqual(0, results.Count());
+                var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112
+                Assert.AreEqual("-1,1111,2222,2112", currPath);
 
-		}
+                //ensure it's indexed
+                indexer.ReIndexNode(node, IndexTypes.Media);
+
+                //change the parent node id to be the one it used to exist under
+                var existingCriteria = indexer.IndexerData;
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    2222);
+
+                //now mimic moving the node underneath 1116 instead of 2222
+                node.SetAttributeValue("path", currPath.Replace("2222", "1116"));
+                node.SetAttributeValue("parentID", "1116");
+
+                //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint
+                indexer.ReIndexNode(node, IndexTypes.Media);
+
+                //RESET the parent id
+                existingCriteria = ((IndexCriteria)indexer.IndexerData);
+                indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes,
+                    null);
+
+                //now ensure it's deleted
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
+            }
+        }
 
 
-		/// <summary>
-		/// This will ensure that all 'Content' (not media) is cleared from the index using the Lucene API directly.
-		/// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly.
-		/// </summary>
-		[Test]
-		public void Index_Reindex_Content()
-		{
-			var s = (IndexSearcher)_searcher.GetSearcher();
+        /// <summary>
+        /// This will ensure that all 'Content' (not media) is cleared from the index using the Lucene API directly.
+        /// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly.
+        /// </summary>
+        [Test]
+        public void Index_Reindex_Content()
+        {
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, supportUnpublishedContent: true))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			//first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine!
-			
-			var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content);
-		    var writer = _indexer.GetIndexWriter();
-            writer.DeleteDocuments(contentTerm);
-            writer.Commit();
-			
+                var s = (IndexSearcher)searcher.GetSearcher();
 
-			//make sure the content is gone. This is done with lucene APIs, not examine!
-			var collector = new AllHitsCollector(false, true);
-			var query = new TermQuery(contentTerm);
-			s = (IndexSearcher)_searcher.GetSearcher(); //make sure the searcher is up do date.
-			s.Search(query, collector);
-			Assert.AreEqual(0, collector.Count);
+                //first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine!
 
-			//call our indexing methods
-			_indexer.IndexAll(IndexTypes.Content);
+                var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content);
+                writer.DeleteDocuments(contentTerm);
+                writer.Commit();
 
-			collector = new AllHitsCollector(false, true);
-			s = (IndexSearcher)_searcher.GetSearcher(); //make sure the searcher is up do date.
-			s.Search(query, collector);
-            //var ids = new List<string>();
-            //for (var i = 0; i < collector.Count;i++)
-            //{
-            //    ids.Add(s.Doc(collector.GetDocId(i)).GetValues("__NodeId")[0]);
-            //}
-			Assert.AreEqual(20, collector.Count);
-		}
+                //make sure the content is gone. This is done with lucene APIs, not examine!
+                var collector = new AllHitsCollector(false, true);
+                var query = new TermQuery(contentTerm);
+                s = (IndexSearcher)searcher.GetSearcher(); //make sure the searcher is up do date.
+                s.Search(query, collector);
+                Assert.AreEqual(0, collector.Count);
 
-		/// <summary>
-		/// This will delete an item from the index and ensure that all children of the node are deleted too!
-		/// </summary>
-		[Test]
-        [Ignore]
-		public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
-		{
+                //call our indexing methods
+                indexer.IndexAll(IndexTypes.Content);
 
-			//now delete a node that has children
+                collector = new AllHitsCollector(false, true);
+                s = (IndexSearcher)searcher.GetSearcher(); //make sure the searcher is up do date.
+                s.Search(query, collector);
+                //var ids = new List<string>();
+                //for (var i = 0; i < collector.Count;i++)
+                //{
+                //    ids.Add(s.Doc(collector.GetDocId(i)).GetValues("__NodeId")[0]);
+                //}
+                Assert.AreEqual(21, collector.Count);
+            }
+        }
 
-			_indexer.DeleteFromIndex(1140.ToString());
-			//this node had children: 1141 & 1142, let's ensure they are also removed
+        /// <summary>
+        /// This will delete an item from the index and ensure that all children of the node are deleted too!
+        /// </summary>
+        [Test]
+        public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
+        {
 
-			var results = _searcher.Search(_searcher.CreateSearchCriteria().Id(1141).Compile());
-			Assert.AreEqual(0, results.Count());
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {
+                indexer.RebuildIndex();
 
-			results = _searcher.Search(_searcher.CreateSearchCriteria().Id(1142).Compile());
-			Assert.AreEqual(0, results.Count());
+                //now delete a node that has children
 
-		}
+                indexer.DeleteFromIndex(1140.ToString());
+                //this node had children: 1141 & 1142, let's ensure they are also removed
 
-		#region Private methods and properties
+                var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
 
-		private readonly TestContentService _contentService = new TestContentService();
-		private readonly TestMediaService _mediaService = new TestMediaService();
-
-		private static UmbracoExamineSearcher _searcher;
-		private static UmbracoContentIndexer _indexer;
-
-		#endregion
-
-		#region Initialize and Cleanup
-
-		private Lucene.Net.Store.Directory _luceneDir;
-
-	    public override void TearDown()
-	    {
-	        base.TearDown();
-            _luceneDir.Dispose();
+                results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile());
+                Assert.AreEqual(0, results.TotalItemCount);
+            }
+        }
+        
+        #region Initialize and Cleanup
+            
+        public override void TearDown()
+        {
+            base.TearDown();
+        
             UmbracoExamineSearcher.DisableInitializationCheck = null;
             BaseUmbracoIndexer.DisableInitializationCheck = null;
         }
         
-
-	    public override void Initialize()
-	    {
-	        base.Initialize();
-            _luceneDir = new RAMDirectory();
-            _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
-            _indexer.RebuildIndex();
-            _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
-        }
-        
-
-		#endregion
-	}
+        #endregion
+    }
 }

src/Umbraco.Tests/UmbracoExamine/RandomIdRAMDirectory.cs

@@ -0,0 +1,14 @@
+using System;
+using Lucene.Net.Store;
+
+namespace Umbraco.Tests.UmbracoExamine
+{
+    public class RandomIdRAMDirectory : RAMDirectory
+    {
+        private readonly string _lockId = Guid.NewGuid().ToString();
+        public override string GetLockID()
+        {
+            return _lockId;
+        }
+    }
+}

src/Umbraco.Tests/UmbracoExamine/SearchTests.cs

@@ -8,6 +8,8 @@ using Examine.LuceneEngine.Providers;
 using Lucene.Net.Store;
 using NUnit.Framework;
 using Examine.LuceneEngine.SearchCriteria;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Index;
 using Umbraco.Tests.TestHelpers;
 
 namespace Umbraco.Tests.UmbracoExamine
@@ -20,19 +22,17 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Test_Sort_Order_Sorting()
         {
-            //var newIndexFolder = new DirectoryInfo(Path.Combine("App_Data\\SearchTests", Guid.NewGuid().ToString()));
-            //System.IO.Directory.CreateDirectory(newIndexFolder.FullName);
-
-            using (var luceneDir = new RAMDirectory())
-            //using (var luceneDir = new SimpleFSDirectory(newIndexFolder))
-            {
-                var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, null,
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, null,
                     new TestDataService()
-                        {
-                            ContentService = new TestContentService(TestFiles.umbraco_sort)
-                        });
-                indexer.RebuildIndex();
-                var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
+                    {
+                        ContentService = new TestContentService(TestFiles.umbraco_sort)
+                    },
+                    supportUnpublishedContent: true))
+            using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
+            {                
+                indexer.RebuildIndex();                
 
                 var s = (LuceneSearcher)searcher;
                 var luceneSearcher = s.GetSearcher();
@@ -69,25 +69,7 @@ namespace Umbraco.Tests.UmbracoExamine
                 currentSort = sort;
             }
             return true;
-        }
-
-        //[Test]
-        //public void Test_Index_Type_With_German_Analyzer()
-        //{
-        //    using (var luceneDir = new RAMDirectory())
-        //    {
-        //        var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir,
-        //            new GermanAnalyzer());
-        //        indexer.RebuildIndex();
-        //        var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);    
-        //    }
-        //}
-
-        //private readonly TestContentService _contentService = new TestContentService();
-        //private readonly TestMediaService _mediaService = new TestMediaService();
-        //private static UmbracoExamineSearcher _searcher;
-        //private static UmbracoContentIndexer _indexer;
-        //private Lucene.Net.Store.Directory _luceneDir;
+        }        
 
     }
 }

src/Umbraco.Tests/UmbracoExamine/UmbracoContentIndexerTests.cs

@@ -0,0 +1,107 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Xml.Linq;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using UmbracoExamine;
+
+namespace Umbraco.Tests.UmbracoExamine
+{
+    [TestFixture]
+    public class UmbracoContentIndexerTests : ExamineBaseTest
+    {
+        [Test]
+        public void Get_Serialized_Content_No_Published_Content()
+        {
+            var contentSet = new List<IContent>
+            {
+                Mock.Of<IContent>(c => c.Id == 1 && c.Path == "-1,1" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 2 && c.Path == "-1,2" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 3 && c.Path == "-1,3" && c.Published == false && c.Level == 1),      // no
+                Mock.Of<IContent>(c => c.Id == 4 && c.Path == "-1,4" && c.Published == false && c.Level == 1),      // no
+
+                Mock.Of<IContent>(c => c.Id == 5 && c.Path == "-1,1,5" && c.Published && c.Level == 2),
+                Mock.Of<IContent>(c => c.Id == 6 && c.Path == "-1,2,6" && c.Published == false && c.Level == 2),    // no
+                Mock.Of<IContent>(c => c.Id == 7 && c.Path == "-1,3,7" && c.Published && c.Level == 2),             // no
+                Mock.Of<IContent>(c => c.Id == 8 && c.Path == "-1,4,8" && c.Published && c.Level == 2),             // no
+                Mock.Of<IContent>(c => c.Id == 9 && c.Path == "-1,4,9" && c.Published && c.Level == 2),             // no
+
+                Mock.Of<IContent>(c => c.Id == 10 && c.Path == "-1,1,5,10" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 15 && c.Path == "-1,1,5,15" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 11 && c.Path == "-1,2,6,11" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 16 && c.Path == "-1,2,6,16" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 12 && c.Path == "-1,3,7,12" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 17 && c.Path == "-1,3,7,17" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 13 && c.Path == "-1,4,8,13" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 18 && c.Path == "-1,4,8,18" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 14 && c.Path == "-1,4,9,14" && c.Published && c.Level == 3),         // no
+                Mock.Of<IContent>(c => c.Id == 19 && c.Path == "-1,4,9,19" && c.Published && c.Level == 3),         // no
+            };
+
+            //ensure the rest of the required values are populted
+            foreach (var content in contentSet)
+            {
+                var mock = Mock.Get(content);
+                mock.Setup(x => x.ContentType).Returns(Mock.Of<IContentType>(type => type.Icon == "hello"));
+            }
+
+            contentSet.Sort((a, b) => Comparer<int>.Default.Compare(a.Level, b.Level));
+
+            var published = new HashSet<string>();
+
+            var result = UmbracoContentIndexer.GetSerializedContent(false, content => new XElement("test"), contentSet, published)
+                .WhereNotNull()
+                .ToArray();
+
+            Assert.AreEqual(5, result.Length);
+        }
+
+        [Test]
+        public void Get_Serialized_Content_With_Published_Content()
+        {
+            var contentSet = new List<IContent>
+            {
+                Mock.Of<IContent>(c => c.Id == 1 && c.Path == "-1,1" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 2 && c.Path == "-1,2" && c.Published && c.Level == 1),
+                Mock.Of<IContent>(c => c.Id == 3 && c.Path == "-1,3" && c.Published == false && c.Level == 1),      
+                Mock.Of<IContent>(c => c.Id == 4 && c.Path == "-1,4" && c.Published == false && c.Level == 1),      
+
+                Mock.Of<IContent>(c => c.Id == 5 && c.Path == "-1,1,5" && c.Published && c.Level == 2),
+                Mock.Of<IContent>(c => c.Id == 6 && c.Path == "-1,2,6" && c.Published == false && c.Level == 2),    
+                Mock.Of<IContent>(c => c.Id == 7 && c.Path == "-1,3,7" && c.Published && c.Level == 2),             
+                Mock.Of<IContent>(c => c.Id == 8 && c.Path == "-1,4,8" && c.Published && c.Level == 2),             
+                Mock.Of<IContent>(c => c.Id == 9 && c.Path == "-1,4,9" && c.Published && c.Level == 2),             
+
+                Mock.Of<IContent>(c => c.Id == 10 && c.Path == "-1,1,5,10" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 15 && c.Path == "-1,1,5,15" && c.Published && c.Level == 3),
+                Mock.Of<IContent>(c => c.Id == 11 && c.Path == "-1,2,6,11" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 16 && c.Path == "-1,2,6,16" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 12 && c.Path == "-1,3,7,12" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 17 && c.Path == "-1,3,7,17" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 13 && c.Path == "-1,4,8,13" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 18 && c.Path == "-1,4,8,18" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 14 && c.Path == "-1,4,9,14" && c.Published && c.Level == 3),         
+                Mock.Of<IContent>(c => c.Id == 19 && c.Path == "-1,4,9,19" && c.Published && c.Level == 3),          
+            };
+
+            //ensure the rest of the required values are populted
+            foreach (var content in contentSet)
+            {
+                var mock = Mock.Get(content);
+                mock.Setup(x => x.ContentType).Returns(Mock.Of<IContentType>(type => type.Icon == "hello"));
+            }
+
+            contentSet.Sort((a, b) => Comparer<int>.Default.Compare(a.Level, b.Level));
+
+            var published = new HashSet<string>();
+
+            var result = UmbracoContentIndexer.GetSerializedContent(true, content => new XElement("test"), contentSet, published)
+                .WhereNotNull()
+                .ToArray();
+
+            Assert.AreEqual(19, result.Length);
+        }
+    }
+}

src/Umbraco.Tests/packages.config

@@ -2,7 +2,7 @@
 <packages>
   <package id="AspNetWebApi.SelfHost" version="4.0.20710.0" targetFramework="net45" />
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.80" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.Mvc" version="5.2.3" targetFramework="net45" />

src/Umbraco.Web.UI.Client/src/init.js

@@ -1,6 +1,6 @@
 /** Executed when the application starts, binds to events and set global state */
-app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache',
-    function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache) {
+app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache', 'localStorageService',
+    function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache, localStorageService) {
 
         //This sets the default jquery ajax headers to include our csrf token, we
         // need to user the beforeSend method because our token changes per user/login so
@@ -13,6 +13,11 @@ app.run(['userService', '$log', '$rootScope', '$location', 'navigationService',
 
         /** Listens for authentication and checks if our required assets are loaded, if/once they are we'll broadcast a ready event */
         eventsService.on("app.authenticated", function(evt, data) {
+
+            //Removes all stored LocalStorage browser items - that may contain sensitive data
+            //So if a machine or computer is shared and a new user logs in, we clear out the previous persons localStorage items
+            localStorageService.clearAll();
+
             assetsService._loadInitAssets().then(function() {
                 appState.setGlobalState("isReady", true);
 

src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js

@@ -1,7 +1,7 @@
 //used for the media picker dialog
 angular.module("umbraco")
     .controller("Umbraco.Overlays.MediaPickerController",
-        function($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, $cookieStore, localizationService) {
+        function ($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, localStorageService, localizationService) {
 
             if (!$scope.model.title) {
                 $scope.model.title = localizationService.localize("defaultdialogs_selectMedia");
@@ -15,7 +15,7 @@ angular.module("umbraco")
             $scope.multiPicker = (dialogOptions.multiPicker && dialogOptions.multiPicker !== "0") ? true : false;
             $scope.startNodeId = dialogOptions.startNodeId ? dialogOptions.startNodeId : -1;
             $scope.cropSize = dialogOptions.cropSize;
-            $scope.lastOpenedNode = $cookieStore.get("umbLastOpenedMediaNodeId");
+            $scope.lastOpenedNode = localStorageService.get("umbLastOpenedMediaNodeId");
             if ($scope.onlyImages) {
                 $scope.acceptedFileTypes = mediaHelper
                     .formatFileTypes(Umbraco.Sys.ServerVariables.umbracoSettings.imageFileTypes);
@@ -133,8 +133,7 @@ angular.module("umbraco")
                     });
                 $scope.currentFolder = folder;
 
-                // for some reason i cannot set cookies with cookieStore
-                document.cookie = "umbLastOpenedMediaNodeId=" + folder.id;
+                localStorageService.set("umbLastOpenedMediaNodeId", folder.id);
 
             };
 

src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js

@@ -34,18 +34,24 @@
                 fields: {},
                 file: file
             }).progress(function (evt) {
+                
+                // hack: in some browsers the progress event is called after success
+                // this prevents the UI from going back to a uploading state
+                if(vm.zipFile.uploadStatus !== "done" && vm.zipFile.uploadStatus !== "error") {
 
-                // set view state to uploading
-                vm.state = 'uploading';
+                    // set view state to uploading
+                    vm.state = 'uploading';
 
-                // calculate progress in percentage
-                var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10);
+                    // calculate progress in percentage
+                    var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10);
 
-                // set percentage property on file
-                vm.zipFile.uploadProgress = progressPercentage;
+                    // set percentage property on file
+                    vm.zipFile.uploadProgress = progressPercentage;
 
-                // set uploading status on file
-                vm.zipFile.uploadStatus = "uploading";
+                    // set uploading status on file
+                    vm.zipFile.uploadStatus = "uploading";
+
+                }
 
             }).success(function (data, status, headers, config) {
 

src/Umbraco.Web.UI.Client/src/views/packager/views/installed.html

@@ -12,8 +12,8 @@
                 <div class="umb-package-list__item" ng-repeat="installedPackage in vm.installedPackages">
 
                     <div class="umb-package-list__item-icon">
-                        <i ng-if="!installedPackage.icon" class="icon-box"></i>
-                        <img ng-if="installedPackage.icon" ng-src="{{installedPackage.icon}}" />
+                        <i ng-if="!installedPackage.iconUrl" class="icon-box"></i>
+                        <img ng-if="installedPackage.iconUrl" ng-src="{{installedPackage.iconUrl}}" />
                     </div>
 
                     <div class="umb-package-list__item-content">

src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js

@@ -2,7 +2,7 @@ angular.module("umbraco")
 .controller("Umbraco.PropertyEditors.GoogleMapsController",
     function ($element, $rootScope, $scope, notificationsService, dialogService, assetsService, $log, $timeout) {
 
-        assetsService.loadJs('http://www.google.com/jsapi')
+        assetsService.loadJs('https://www.google.com/jsapi')
             .then(function () {
                 google.load("maps", "3",
                             {

src/Umbraco.Web.UI/Umbraco.Web.UI.csproj

@@ -127,8 +127,8 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\packages\dotless.1.4.1.0\lib\dotless.Core.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.80.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.80\lib\net45\Examine.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
@@ -138,8 +138,9 @@
     <Reference Include="ImageProcessor, Version=2.5.1.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\ImageProcessor.2.5.1\lib\net45\ImageProcessor.dll</HintPath>
     </Reference>
-    <Reference Include="ImageProcessor.Web, Version=4.7.2.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\ImageProcessor.Web.4.7.2\lib\net45\ImageProcessor.Web.dll</HintPath>
+    <Reference Include="ImageProcessor.Web, Version=4.8.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\ImageProcessor.Web.4.8.0\lib\net45\ImageProcessor.Web.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="log4net, Version=1.2.11.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -2421,9 +2422,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\"
         <WebProjectProperties>
           <UseIIS>True</UseIIS>
           <AutoAssignPort>True</AutoAssignPort>
-          <DevelopmentServerPort>7570</DevelopmentServerPort>
+          <DevelopmentServerPort>7580</DevelopmentServerPort>
           <DevelopmentServerVPath>/</DevelopmentServerVPath>
-          <IISUrl>http://localhost:7570</IISUrl>
+          <IISUrl>http://localhost:7580</IISUrl>
           <NTLMAuthentication>False</NTLMAuthentication>
           <UseCustomServer>False</UseCustomServer>
           <CustomServerUrl>

src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx

@@ -13,6 +13,7 @@
     
    <umb:JsInclude ID="JsInclude1" runat="server" FilePath="Application/NamespaceManager.js" PathNameAlias="UmbracoClient" Priority="0" />
    <umb:JsInclude ID="JsInclude3" runat="server" FilePath="ui/jquery.js" PathNameAlias="UmbracoClient" Priority="1" />
+   <umb:JsInclude ID="JsInclude4" runat="server" FilePath="lib/jquery-migrate/jquery-migrate.min.js" PathNameAlias="UmbracoRoot" Priority="1" />
    <umb:JsInclude ID="JsInclude6" runat="server" FilePath="ui/base2.js" PathNameAlias="UmbracoClient" Priority="1" />
    <umb:JsInclude ID="JsInclude11" runat="server" FilePath="UI/knockout.js" PathNameAlias="UmbracoClient" Priority="3" />
    <umb:JsInclude ID="JsInclude12" runat="server" FilePath="UI/knockout.mapping.js" PathNameAlias="UmbracoClient" Priority="4" />

src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs

@@ -48,6 +48,15 @@ namespace Umbraco.Web.UI.Umbraco.Dashboard {
         /// </remarks>
         protected global::ClientDependency.Core.Controls.JsInclude JsInclude3;
         
+        /// <summary>
+        /// JsInclude4 control.
+        /// </summary>
+        /// <remarks>
+        /// Auto-generated field.
+        /// To modify move field declaration from designer file to code-behind file.
+        /// </remarks>
+        protected global::ClientDependency.Core.Controls.JsInclude JsInclude4;
+        
         /// <summary>
         /// JsInclude6 control.
         /// </summary>

src/Umbraco.Web.UI/packages.config

@@ -4,9 +4,9 @@
   <package id="ClientDependency" version="1.9.2" targetFramework="net45" />
   <package id="ClientDependency-Mvc5" version="1.8.0.0" targetFramework="net45" />
   <package id="dotless" version="1.4.1.0" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.80" targetFramework="net45" />
   <package id="ImageProcessor" version="2.5.1" targetFramework="net45" />
-  <package id="ImageProcessor.Web" version="4.7.2" targetFramework="net45" />
+  <package id="ImageProcessor.Web" version="4.8.0" targetFramework="net45" />
   <package id="ImageProcessor.Web.Config" version="2.3.0" targetFramework="net45" />
   <package id="log4net-mediumtrust" version="2.0.0" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />

src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml

@@ -1142,7 +1142,7 @@ To manage your website, simply open the Umbraco back office and start adding con
 
   <area alias="modelsBuilder">
       <key alias="buildingModels">Building models</key>
-      <key alias="waitingMessage">this can take abit of time, don't worry</key>
+      <key alias="waitingMessage">this can take a bit of time, don't worry</key>
       <key alias="modelsGenerated">Models generated</key>
       <key alias="modelsGeneratedError">Models could not be generated</key>
       <key alias="modelsExceptionInUlog">Models generation has failed, see exception in U log</key>

src/Umbraco.Web.UI/umbraco/config/lang/ru.xml

@@ -27,6 +27,9 @@
     <key alias="publish">Опубликовать</key>
     <key alias="refreshNode">Обновить узлы</key>
     <key alias="republish">Опубликовать весь сайт</key>
+    <key alias="SetPermissionsForThePage">Установить разрешения для страницы '%0%'</key>
+    <key alias="chooseWhereToMove">Выберите, куда переместить</key>
+    <key alias="toInTheTreeStructureBelow">В структуре документов ниже</key>
     <key alias="restore" version="7.3.0">Восстановить</key>
     <key alias="rights">Разрешения</key>
     <key alias="rollback">Откатить</key>
@@ -189,7 +192,7 @@
     <key alias="memberrole">Роль участника</key>
     <key alias="membertype">Тип участника</key>
     <key alias="noDate">Дата не указана</key>
-    <key alias="nodeName">Заголовок страницы</key>
+    <key alias="nodeName">Заголовок ссылки</key>
     <key alias="notmemberof">Не является членом групп(ы)</key>
     <key alias="otherElements">Свойства</key>
     <key alias="parentNotPublished">Этот документ опубликован, но скрыт, потому что его родительский документ '%0%' не опубликован</key>
@@ -341,9 +344,10 @@
     <key alias="viewCacheItem">Просмотр элемента кэша</key>
     <key alias="createFolder">Создать папку...</key>
     <key alias="relateToOriginalLabel">Связать с оригиналом</key>
+    <key alias="includeDescendants">Включая все дочерние</key>
     <key alias="theFriendliestCommunity">Самое дружелюбное сообщество</key>
     <key alias="linkToPage">Ссылка на страницу</key>
-    <key alias="openInNewWindow">Открывает документ по ссылке в новом окне или вкладке браузера</key>
+    <key alias="openInNewWindow">Открывать ссылку в новом окне или вкладке браузера</key>
     <key alias="linkToMedia">Ссылка на медиа-файл</key>
     <key alias="selectMedia">Выбрать медиа</key>
     <key alias="selectIcon">Выбрать значок</key>

src/Umbraco.Web/Cache/ContentTypeCacheRefresher.cs

@@ -131,9 +131,7 @@ namespace Umbraco.Web.Cache
             ClearAllIsolatedCacheByEntityType<IMediaType>();
             ClearAllIsolatedCacheByEntityType<IMember>();
             ClearAllIsolatedCacheByEntityType<IMemberType>();
-
-            //all property type cache
-            ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheByKeySearch(CacheKeys.PropertyTypeCacheKey);
+            
             //all content type property cache
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheByKeySearch(CacheKeys.ContentTypePropertiesCacheKey);     
             //all content type cache
@@ -266,12 +264,6 @@ namespace Umbraco.Web.Cache
         /// </returns>
         private static void ClearContentTypeCache(JsonPayload payload)
         {
-            //clears the cache for each property type associated with the content type
-            foreach (var pid in payload.PropertyTypeIds)
-            {
-                ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(CacheKeys.PropertyTypeCacheKey + pid);
-            }
-
             //clears the cache associated with the Content type itself
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(string.Format("{0}{1}", CacheKeys.ContentTypeCacheKey, payload.Id));
             //clears the cache associated with the content type properties collection

src/Umbraco.Web/Editors/BackOfficeController.cs

@@ -262,7 +262,7 @@ namespace Umbraco.Web.Editors
                             },
                             {
                                 "mediaTypeApiBaseUrl", Url.GetUmbracoApiServiceBaseUrl<MediaTypeController>(
-                                    controller => controller.GetAllowedChildren(0))
+                                    controller => controller.GetAllowedChildren("0"))
                             },
                             {
                                 "macroApiBaseUrl", Url.GetUmbracoApiServiceBaseUrl<MacroController>(

src/Umbraco.Web/Editors/EntityController.cs

@@ -614,7 +614,7 @@ namespace Umbraco.Web.Editors
                     .Select(Mapper.Map<EntityBasic>);
 
                 // entities are in "some" order, put them back in order
-                var xref = entities.ToDictionary(x => x.Id);
+                var xref = entities.ToDictionary(x => x.Key);
                 var result = keysArray.Select(x => xref.ContainsKey(x) ? xref[x] : null).Where(x => x != null);
 
                 return result;

src/Umbraco.Web/Editors/MediaController.cs

@@ -36,6 +36,7 @@ using Umbraco.Core.Configuration;
 using Umbraco.Core.Persistence.FaultHandling;
 using Umbraco.Web.UI;
 using Notification = Umbraco.Web.Models.ContentEditing.Notification;
+using Umbraco.Core.Persistence;
 
 namespace Umbraco.Web.Editors
 {
@@ -175,7 +176,37 @@ namespace Umbraco.Web.Editors
         /// Returns the child media objects
         /// </summary>
         [FilterAllowedOutgoingMedia(typeof(IEnumerable<ContentItemBasic<ContentPropertyBasic, IMedia>>), "Items")]
-        public PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(int id,
+        public PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(string id,
+            int pageNumber = 0,
+            int pageSize = 0,
+            string orderBy = "SortOrder",
+            Direction orderDirection = Direction.Ascending,
+            bool orderBySystemField = true,
+            string filter = "")
+        {
+            int idInt; Guid idGuid;
+
+            if (Guid.TryParse(id, out idGuid))
+            {
+                var entity = Services.EntityService.GetByKey(idGuid);
+                if (entity != null)
+                {
+                    return GetChildren(entity.Id, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter);
+                }
+                else
+                {
+                    throw new HttpResponseException(HttpStatusCode.NotFound);
+                }
+            }
+            else if (int.TryParse(id, out idInt))
+            {
+                return GetChildren(idInt, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter);
+            }
+
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+
+        private PagedResult<ContentItemBasic<ContentPropertyBasic, IMedia>> GetChildren(int id,
             int pageNumber = 0,
             int pageSize = 0,
             string orderBy = "SortOrder",
@@ -448,12 +479,37 @@ namespace Umbraco.Web.Editors
             }
 
             //get the string json from the request
-            int parentId;
-            if (int.TryParse(result.FormData["currentFolder"], out parentId) == false)
+            int parentId; bool entityFound;
+            string currentFolderId = result.FormData["currentFolder"];
+            if (int.TryParse(currentFolderId, out parentId) == false)
             {
-                return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer");
+                // if a guid then try to look up the entity
+                Guid idGuid;
+                if (Guid.TryParse(currentFolderId, out idGuid))
+                {
+                    var entity = Services.EntityService.GetByKey(idGuid);
+                    if (entity != null)
+                    {
+                        entityFound = true;
+                        parentId = entity.Id;
+                    }
+                    else
+                    {
+                        throw new EntityNotFoundException(currentFolderId, "The passed id doesn't exist");
+                    }
+                }
+                else
+                {
+                    return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid");
+                }
+
+                if (entityFound == false)
+                {
+                    return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid");
+                }
             }
 
+
             //ensure the user has access to this folder by parent id!
             if (CheckPermissions(
                new Dictionary<string, object>(),

src/Umbraco.Web/Editors/MediaTypeController.cs

@@ -11,6 +11,8 @@ using System.Net;
 using System.Net.Http;
 using Umbraco.Web.WebApi;
 using Umbraco.Core.Services;
+using Umbraco.Core.Models.EntityBase;
+using System;
 
 namespace Umbraco.Web.Editors
 {
@@ -174,7 +176,22 @@ namespace Umbraco.Web.Editors
         /// </summary>
         /// <param name="contentId"></param>
         [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)]
-        public IEnumerable<ContentTypeBasic> GetAllowedChildren(int contentId)
+        public IEnumerable<ContentTypeBasic> GetAllowedChildren(string contentId)
+        {
+            Guid idGuid = Guid.Empty;
+            int idInt;
+            if (Guid.TryParse(contentId, out idGuid)) { 
+                var entity = ApplicationContext.Services.EntityService.GetByKey(idGuid);
+                return GetAllowedChildrenInternal(entity.Id);
+            } else if (int.TryParse(contentId, out idInt))
+            {
+                return GetAllowedChildrenInternal(idInt);
+            }
+
+            throw new HttpResponseException(HttpStatusCode.NotFound);
+        }
+
+        private IEnumerable<ContentTypeBasic> GetAllowedChildrenInternal(int contentId)
         {
             if (contentId == Constants.System.RecycleBinContent)
                 return Enumerable.Empty<ContentTypeBasic>();

src/Umbraco.Web/ImageCropperTemplateExtensions.cs

@@ -81,7 +81,7 @@ namespace Umbraco.Web
         /// Use focal point, to generate an output image using the focal point instead of the predefined crop
         /// </param>
         /// <param name="useCropDimensions">
-        /// Use crop dimensions to have the output image sized according to the predefined crop sizes, this will override the width and height parameters>.
+        /// Use crop dimensions to have the output image sized according to the predefined crop sizes, this will override the width and height parameters.
         /// </param>
         /// <param name="cacheBuster">
         /// Add a serialised date of the last edit of the item to ensure client cache refresh when updated

src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs

@@ -92,7 +92,12 @@ namespace Umbraco.Web.PropertyEditors
                             //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect 
                             // the website. 
                         }
+                        catch (ArgumentException)
+                        {
+                            //swallow on purpose to prevent this error:
+                            // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
 
+                        }
                     }
                 }
             }

src/Umbraco.Web/Trees/ContentTreeController.cs

@@ -202,11 +202,13 @@ namespace Umbraco.Web.Trees
         /// <returns></returns>
         protected override bool HasPathAccess(string id, FormDataCollection queryStrings)
         {
-            var content = Services.ContentService.GetById(int.Parse(id));
-            if (content == null)
+            var entity = GetEntityFromId(id);
+            if (entity == null)
             {
                 return false;
             }
+
+            IContent content = Services.ContentService.GetById(entity.Id);
             return Security.CurrentUser.HasPathAccess(content);
         }
 

src/Umbraco.Web/Trees/ContentTreeControllerBase.cs

@@ -96,7 +96,7 @@ namespace Umbraco.Web.Trees
                     LogHelper.Warn<ContentTreeControllerBase>("The user " + Security.CurrentUser.Username + " does not have access to the tree node " + id);
                     return new TreeNodeCollection();
                 }
-                
+
                 // So there's an alt id specified, it's not the root node and the user has access to it, great! But there's one thing we
                 // need to consider: 
                 // If the tree is being rendered in a dialog view we want to render only the children of the specified id, but
@@ -110,7 +110,7 @@ namespace Umbraco.Web.Trees
                     id = Constants.System.Root.ToString(CultureInfo.InvariantCulture);
                 }
             }
-            
+
             var entities = GetChildEntities(id);
             nodes.AddRange(entities.Select(entity => GetSingleTreeNode(entity, id, queryStrings)).Where(node => node != null));
             return nodes;
@@ -122,11 +122,24 @@ namespace Umbraco.Web.Trees
 
         protected IEnumerable<IUmbracoEntity> GetChildEntities(string id)
         {
+            // use helper method to ensure we support both integer and guid lookups
             int iid;
-            if (int.TryParse(id, out iid) == false)
+
+            // if it's the root node, we won't use the look up
+            if (id != "-1")
             {
-                throw new InvalidCastException("The id for the media tree must be an integer");
+                var idEntity = GetEntityFromId(id);
+                if (idEntity == null)
+                {
+                    throw new HttpResponseException(HttpStatusCode.NotFound);
+                }
+                iid = idEntity.Id;
             }
+            else
+            {
+                iid = int.Parse(id);
+            }
+
 
             //if a request is made for the root node data but the user's start node is not the default, then
             // we need to return their start node data
@@ -134,11 +147,12 @@ namespace Umbraco.Web.Trees
             {
                 //just return their single start node, it will show up under the 'Content' label
                 var startNode = Services.EntityService.Get(UserStartNode, UmbracoObjectType);
-                if (startNode == null)
+                if (startNode != null)
+                    return new[] { startNode };
+                else
                 {
                     throw new EntityNotFoundException(UserStartNode, "User's start content node could not be found");
                 }
-                return new[] { startNode };
             }
 
             return Services.EntityService.GetChildren(iid, UmbracoObjectType).ToArray();
@@ -176,9 +190,9 @@ namespace Umbraco.Web.Trees
                 {
                     id = altStartId;
                 }
-                
+
                 var nodes = GetTreeNodesInternal(id, queryStrings);
-                
+
                 //only render the recycle bin if we are not in dialog and the start id id still the root
                 if (IsDialog(queryStrings) == false && id == Constants.System.Root.ToInvariantString())
                 {
@@ -210,8 +224,9 @@ namespace Umbraco.Web.Trees
         /// </remarks>
         private TreeNodeCollection GetTreeNodesInternal(string id, FormDataCollection queryStrings)
         {
+            IUmbracoEntity current = GetEntityFromId(id);
+
             //before we get the children we need to see if this is a container node
-            var current = Services.EntityService.Get(int.Parse(id), UmbracoObjectType);
 
             //test if the parent is a listview / container
             if (current != null && current.IsContainer())
@@ -286,5 +301,32 @@ namespace Umbraco.Web.Trees
         {
             return allowedUserOptions.Select(x => x.Action).OfType<ActionBrowse>().Any();
         }
+
+        /// <summary>
+        /// Get an entity via an id that can be either an integer or a Guid
+        /// </summary>
+        /// <param name="id"></param>
+        /// <returns></returns>
+        internal IUmbracoEntity GetEntityFromId(string id)
+        {
+            IUmbracoEntity entity;
+            Guid idGuid = Guid.Empty;
+            int idInt;
+            if (Guid.TryParse(id, out idGuid))
+            {
+                entity = Services.EntityService.GetByKey(idGuid, UmbracoObjectType);
+
+            }
+            else if (int.TryParse(id, out idInt))
+            {
+                entity = Services.EntityService.Get(idInt, UmbracoObjectType);
+            }
+            else
+            {
+                return null;
+            }
+
+            return entity;
+        }
     }
 }

src/Umbraco.Web/Umbraco.Web.csproj

@@ -112,8 +112,9 @@
     <Reference Include="dotless.Core, Version=1.4.1.0, Culture=neutral, PublicKeyToken=96b446c9e63eae34, processorArchitecture=MSIL">
       <HintPath>..\packages\dotless.1.4.1.0\lib\dotless.Core.dll</HintPath>
     </Reference>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.80.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.80\lib\net45\Examine.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
       <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>

src/Umbraco.Web/WebServices/ExamineManagementApiController.cs

@@ -166,6 +166,8 @@ namespace Umbraco.Web.WebServices
             var msg = ValidateLuceneIndexer(indexerName, out indexer);
             if (msg.IsSuccessStatusCode)
             {
+                LogHelper.Info<ExamineManagementApiController>(string.Format("Rebuilding index '{0}'", indexerName));
+
                 //remove it in case there's a handler there alraedy
                 indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
                 //now add a single handler
@@ -201,6 +203,8 @@ namespace Umbraco.Web.WebServices
             //ensure it's not listening anymore
             indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
 
+            LogHelper.Info<ExamineManagementApiController>(string.Format("Rebuilding index '{0}' done, {1} items committed (can differ from the number of items in the index)", indexer.Name, indexer.CommitCount));
+
             var cacheKey = "temp_indexing_op_" + indexer.Name;
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey);
         }
@@ -266,7 +270,10 @@ namespace Umbraco.Web.WebServices
                 var val = p.GetValue(indexer, null);
                 if (val == null)
                 {
-                    LogHelper.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name);
+                    // Do not warn for new new attribute that is optional
+                    if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false)
+                        LogHelper.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name);
+
                     val = string.Empty;
                 }
                 indexerModel.ProviderProperties.Add(p.Name, val.ToString());

src/Umbraco.Web/packages.config

@@ -3,7 +3,7 @@
   <package id="AutoMapper" version="3.3.1" targetFramework="net45" />
   <package id="ClientDependency" version="1.9.2" targetFramework="net45" />
   <package id="dotless" version="1.4.1.0" targetFramework="net45" />
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.80" targetFramework="net45" />
   <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Markdown" version="1.14.7" targetFramework="net45" />

src/Umbraco.Web/umbraco.presentation/umbraco/controls/ContentTypeControl.cs

@@ -2,6 +2,7 @@ using System;
 using System.Web.UI.HtmlControls;
 using System.Web.UI.WebControls;
 using System.Collections;
+using System.ComponentModel;
 using System.IO;
 using Umbraco.Core.IO;
 using System.Linq;
@@ -11,6 +12,8 @@ namespace umbraco.controls
 	/// <summary>
 	/// Summary description for ContentTypeControl.
 	/// </summary>
+	[Obsolete("No longer used, will be removed in v8")]
+    [EditorBrowsable(EditorBrowsableState.Never)]
 	public class ContentTypeControl : uicontrols.TabView
 	{
 		public event System.EventHandler OnSave;

src/Umbraco.Web/umbraco.presentation/umbraco/users/EditUser.aspx.cs

@@ -3,6 +3,7 @@ using System.Collections;
 using System.Configuration.Provider;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Web;
 using System.Web.Security;
 using System.Web.UI;
@@ -320,13 +321,14 @@ namespace umbraco.cms.presentation.user
             }
 
             // Populate dropdowns
-            foreach (DocumentType dt in DocumentType.GetAllAsList())
-                cDocumentType.Items.Add(
-                    new ListItem(dt.Text, dt.Alias)
-                    );
+            var allContentTypes = Services.ContentTypeService.GetAllContentTypes().ToList();
+            foreach (var dt in allContentTypes)
+            {
+                cDocumentType.Items.Add(new ListItem(dt.Name, dt.Alias));
+            }
 
             // populate fields
-            ArrayList fields = new ArrayList();
+            var fields = new ArrayList();
             cDescription.ID = "cDescription";
             cCategories.ID = "cCategories";
             cExcerpt.ID = "cExcerpt";
@@ -334,9 +336,9 @@ namespace umbraco.cms.presentation.user
             cCategories.Items.Add(new ListItem(ui.Text("choose"), ""));
             cExcerpt.Items.Add(new ListItem(ui.Text("choose"), ""));
 
-            foreach (PropertyType pt in PropertyType.GetAll())
+            foreach (var pt in allContentTypes.SelectMany(x => x.PropertyTypes).OrderBy(x => x.Name))
             {
-                if (!fields.Contains(pt.Alias))
+                if (fields.Contains(pt.Alias) == false)
                 {
                     cDescription.Items.Add(new ListItem(string.Format("{0} ({1})", pt.Name, pt.Alias), pt.Alias));
                     cCategories.Items.Add(new ListItem(string.Format("{0} ({1})", pt.Name, pt.Alias), pt.Alias));

src/UmbracoExamine/BaseUmbracoIndexer.cs

@@ -49,6 +49,7 @@ namespace UmbracoExamine
         /// <param name="indexPath"></param>
         /// <param name="dataService"></param>
         /// <param name="analyzer"></param>
+        /// <param name="async"></param>
         protected BaseUmbracoIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async)
             : base(indexerData, indexPath, analyzer, async)
         {
@@ -61,6 +62,19 @@ namespace UmbracoExamine
 			DataService = dataService;
 		}
 
+        /// <summary>
+        /// Creates an NRT indexer
+        /// </summary>
+        /// <param name="indexerData"></param>
+        /// <param name="writer"></param>
+        /// <param name="async"></param>
+        /// <param name="dataService"></param>
+        protected BaseUmbracoIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService, bool async) 
+            : base(indexerData, writer, async)
+        {
+            DataService = dataService;
+        }
+
         #endregion
 
         /// <summary>
@@ -95,7 +109,7 @@ namespace UmbracoExamine
         /// Determines if the manager will call the indexing methods when content is saved or deleted as
         /// opposed to cache being updated.
         /// </summary>
-        public bool SupportUnpublishedContent { get; protected set; }
+        public bool SupportUnpublishedContent { get; protected internal set; }
 
         /// <summary>
         /// The data service used for retreiving and submitting data to the cms

src/UmbracoExamine/UmbracoContentIndexer.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Xml;
 using System.Xml.Linq;
 using Examine;
 using Lucene.Net.Documents;
@@ -15,6 +17,7 @@ using Examine.LuceneEngine;
 using Examine.LuceneEngine.Config;
 using UmbracoExamine.Config;
 using Lucene.Net.Analysis;
+using Lucene.Net.Index;
 using Umbraco.Core.Persistence.Querying;
 using IContentService = Umbraco.Core.Services.IContentService;
 using IMediaService = Umbraco.Core.Services.IMediaService;
@@ -144,6 +147,34 @@ namespace UmbracoExamine
             _contentTypeService = contentTypeService;
         }
 
+        /// <summary>
+        /// Creates an NRT indexer
+        /// </summary>
+        /// <param name="indexerData"></param>
+        /// <param name="writer"></param>
+        /// <param name="dataService"></param>
+        /// <param name="contentTypeService"></param>
+        /// <param name="async"></param>
+        /// <param name="contentService"></param>
+        /// <param name="mediaService"></param>
+        /// <param name="dataTypeService"></param>
+        /// <param name="userService"></param>
+        public UmbracoContentIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService,
+            IContentService contentService,
+            IMediaService mediaService,
+            IDataTypeService dataTypeService,
+            IUserService userService,
+            IContentTypeService contentTypeService,
+            bool async) 
+            : base(indexerData, writer, dataService, async)
+        {
+            _contentService = contentService;
+            _mediaService = mediaService;
+            _dataTypeService = dataTypeService;
+            _userService = userService;
+            _contentTypeService = contentTypeService;
+        }
+
         #endregion
 
         #region Constants & Fields        
@@ -229,7 +260,13 @@ namespace UmbracoExamine
                 SupportProtectedContent = supportProtected;
             else
                 SupportProtectedContent = false;
-            
+
+            bool disableXmlDocLookup;
+            if (config["disableXmlDocLookup"] != null && bool.TryParse(config["disableXmlDocLookup"], out disableXmlDocLookup))
+                DisableXmlDocumentLookup = disableXmlDocLookup;
+            else
+                DisableXmlDocumentLookup = false;
+
             base.Initialize(name, config);
         }
 
@@ -237,6 +274,11 @@ namespace UmbracoExamine
 
         #region Properties
 
+        /// <summary>
+        /// Whether to use the cmsContentXml data to re-index when possible (i.e. for published content, media and members)
+        /// </summary>
+        public bool DisableXmlDocumentLookup { get; private set; }
+
         /// <summary>
         /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected.
         /// This property is ignored if SupportUnpublishedContent is set to true.
@@ -359,131 +401,250 @@ namespace UmbracoExamine
         }
         #endregion
 
-        #region Protected
-
-        /// <summary>
-        /// This is a static query, it's parameters don't change so store statically
-        /// </summary>
-        private IQuery<IContent> _publishedQuery;
+        #region Protected        
 
         protected override void PerformIndexAll(string type)
         {
+            if (SupportedTypes.Contains(type) == false)
+                return;
+
             const int pageSize = 10000;
             var pageIndex = 0;
 
-            switch (type)
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - Start data queries - {0}", type));
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            try
             {
-                case IndexTypes.Content:
-                    var contentParentId = -1;
-                    if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
-                    {
-                        contentParentId = IndexerData.ParentNodeId.Value;
-                    }
-                    IContent[] content;
-
-                    //used to track non-published entities so we can determine what items are implicitly not published
-                    var notPublished = new HashSet<string>();
-
-                    do
-                    {
-                        long total;
-
-                        IEnumerable<IContent> descendants;
-                        if (SupportUnpublishedContent)
+                switch (type)
+                {
+                    case IndexTypes.Content:
+                        var contentParentId = -1;
+                        if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
                         {
-                            descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
-                        }
-                        else
-                        {
-                            if (_publishedQuery == null)
-                            {
-                                _publishedQuery = Query<IContent>.Builder.Where(x => x.Published == true);
-                            }
-
-                            //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine
-                            // which descendent nodes are implicitly not published
-                            descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null);
-                        }                        
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (IndexerData.IncludeNodeTypes.Any())
-                        {
-                            content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
-                        }
-                        else
-                        {
-                            content = descendants.ToArray();
-                        }
-                        AddNodesToIndex(GetSerializedContent(content, notPublished).WhereNotNull(), type);
-                        pageIndex++;
-                    } while (content.Length == pageSize);
-
-                    notPublished.Clear();
-
-                    break;
-                case IndexTypes.Media:
-                    var mediaParentId = -1;
-
-                    if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
-                    {
-                        mediaParentId = IndexerData.ParentNodeId.Value;
-                    }
-                    
-                    XElement[] mediaXElements;
-                    
-                    var mediaTypes = _contentTypeService.GetAllMediaTypes().ToArray();
-                    var icons = mediaTypes.ToDictionary(x => x.Id, y => y.Icon);
-
-                    do
-                    {
-                        long total;
-                        if (mediaParentId == -1)
-                        {
-                            mediaXElements = _mediaService.GetPagedXmlEntries("-1", pageIndex, pageSize, out total).ToArray();
-                        }
-                        else
-                        {
-                            //Get the parent
-                            var parent = _mediaService.GetById(mediaParentId);
-                            if (parent == null)
-                                mediaXElements = new XElement[0];
-                            else
-                                mediaXElements = _mediaService.GetPagedXmlEntries(parent.Path, pageIndex, pageSize, out total).ToArray();
-                        }
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (IndexerData.IncludeNodeTypes.Any())
-                        {
-                            var includeNodeTypeIds = mediaTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id);
-                            mediaXElements = mediaXElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue<int>("nodeType"))).ToArray();
+                            contentParentId = IndexerData.ParentNodeId.Value;
                         }
                         
-                        foreach (var element in mediaXElements)
+                        if (SupportUnpublishedContent == false && DisableXmlDocumentLookup == false)
                         {
-                            element.Add(new XAttribute("icon", icons[element.AttributeValue<int>("nodeType")]));
+                            //get all node Ids that have a published version - this is a fail safe check, in theory
+                            // only document nodes that have a published version would exist in the cmsContentXml table
+                            var allNodesWithPublishedVersions = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                                "select DISTINCT cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1");
+
+                            XElement last = null;
+                            var trackedIds = new HashSet<string>();
+
+                            ReindexWithXmlEntries(type, contentParentId,
+                                () => _contentTypeService.GetAllContentTypes().ToArray(),
+                                (path, pIndex, pSize) =>
+                                {
+                                    long totalContent;
+
+                                    //sorted by: umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder
+                                    var result = _contentService.GetPagedXmlEntries(path, pIndex, pSize, out totalContent).ToArray();
+
+                                    //then like we do in the ContentRepository.BuildXmlCache we need to track what Parents have been processed
+                                    // already so that we can then exclude implicitly unpublished content items
+                                    var filtered = new List<XElement>();
+
+                                    foreach (var xml in result)
+                                    {
+                                        var id = xml.AttributeValue<int>("id");
+                                        
+                                        //don't include this if it doesn't have a published version
+                                        if (allNodesWithPublishedVersions.Contains(id) == false)
+                                            continue;
+
+                                        var parentId = xml.AttributeValue<string>("parentID");
+
+                                        if (parentId == null) continue; //this shouldn't happen
+
+                                        //if the parentid is changing
+                                        if (last != null && last.AttributeValue<string>("parentID") != parentId)
+                                        {
+                                            var found = trackedIds.Contains(parentId);
+                                            if (found == false)
+                                            {
+                                                //Need to short circuit here, if the parent is not there it means that the parent is unpublished
+                                                // and therefore the child is not published either so cannot be included in the xml cache
+                                                continue;
+                                            }                                            
+                                        }
+                                        
+                                        last = xml;
+                                        trackedIds.Add(xml.AttributeValue<string>("id"));
+
+                                        filtered.Add(xml);
+                                    }
+
+                                    return new Tuple<long, XElement[]>(totalContent, filtered.ToArray());
+                                },
+                                i => _contentService.GetById(i));
+                        }
+                        else
+                        {
+                            //used to track non-published entities so we can determine what items are implicitly not published
+                            //currently this is not in use apart form in tests
+                            var notPublished = new HashSet<string>();
+
+                            int currentPageSize;
+                            do
+                            {
+                                long total;
+
+                                IContent[] descendants;
+                                if (SupportUnpublishedContent)
+                                {
+                                    descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "umbracoNode.id").ToArray();
+                                }
+                                else
+                                {
+                                    //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine
+                                    // which descendent nodes are implicitly not published
+                                    descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null).ToArray();
+                                }
+
+                                // need to store decendants count before filtering, in order for loop to work correctly
+                                currentPageSize = descendants.Length;
+
+                                //if specific types are declared we need to post filter them
+                                //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
+                                IEnumerable<IContent> content;
+                                if (IndexerData.IncludeNodeTypes.Any())
+                                {
+                                    content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias));
+                                }
+                                else
+                                {
+                                    content = descendants;
+                                }
+
+                                AddNodesToIndex(GetSerializedContent(
+                                    SupportUnpublishedContent,
+                                    c => _serializer.Serialize(_contentService, _dataTypeService, _userService, c),
+                                    content, notPublished).WhereNotNull(), type);
+
+                                pageIndex++;
+                            } while (currentPageSize == pageSize);
                         }
 
-                        AddNodesToIndex(mediaXElements, type);
-                        pageIndex++;
-                    } while (mediaXElements.Length == pageSize);
+                        break;
+                    case IndexTypes.Media:
+                        var mediaParentId = -1;
 
-                    break;
+                        if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
+                        {
+                            mediaParentId = IndexerData.ParentNodeId.Value;
+                        }
+
+                        ReindexWithXmlEntries(type, mediaParentId,
+                            () => _contentTypeService.GetAllMediaTypes().ToArray(),
+                            (path, pIndex, pSize) =>
+                            {
+                                long totalMedia;
+                                var result = _mediaService.GetPagedXmlEntries(path, pIndex, pSize, out totalMedia).ToArray();
+                                return new Tuple<long, XElement[]>(totalMedia, result);
+                            },
+                            i => _mediaService.GetById(i));
+
+                        break;
+                }
             }
+            finally
+            {
+                stopwatch.Stop();
+            }
+            
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - End data queries - {0}, took {1}ms", type, stopwatch.ElapsedMilliseconds));
         }
 
-        private IEnumerable<XElement> GetSerializedContent(IEnumerable<IContent> content, ISet<string> notPublished)
+        /// <summary>
+        /// Performs a reindex of a type based on looking up entries from the cmsContentXml table - but using callbacks to get this data since
+        /// we don't have a common underlying service interface for the media/content stuff
+        /// </summary>
+        /// <param name="type"></param>
+        /// <param name="parentId"></param>
+        /// <param name="getContentTypes"></param>
+        /// <param name="getPagedXmlEntries"></param>
+        /// <param name="getContent"></param>
+        internal void ReindexWithXmlEntries<TContentType>(
+            string type, 
+            int parentId,
+            Func<TContentType[]> getContentTypes, 
+            Func<string, int, int, Tuple<long, XElement[]>> getPagedXmlEntries,
+            Func<int, IContentBase> getContent)
+            where TContentType: IContentTypeComposition
+        {
+            const int pageSize = 10000;
+            var pageIndex = 0;            
+
+            XElement[] xElements;
+
+            var contentTypes = getContentTypes();
+            var icons = contentTypes.ToDictionary(x => x.Id, y => y.Icon);
+
+            do
+            {
+                long total;
+                if (parentId == -1)
+                {
+                    var pagedElements = getPagedXmlEntries("-1", pageIndex, pageSize);
+                    total = pagedElements.Item1;
+                    xElements = pagedElements.Item2;
+                }
+                else
+                {
+                    //Get the parent
+                    var parent = getContent(parentId);
+                    if (parent == null)
+                        xElements = new XElement[0];
+                    else
+                    {
+                        var pagedElements = getPagedXmlEntries(parent.Path, pageIndex, pageSize);
+                        total = pagedElements.Item1;
+                        xElements = pagedElements.Item2;
+                    }
+                }
+
+                //if specific types are declared we need to post filter them
+                //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
+                if (IndexerData.IncludeNodeTypes.Any())
+                {
+                    var includeNodeTypeIds = contentTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id);
+                    xElements = xElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue<int>("nodeType"))).ToArray();
+                }
+
+                foreach (var element in xElements)
+                {
+                    if (element.Attribute("icon") == null)
+                    {
+                        element.Add(new XAttribute("icon", icons[element.AttributeValue<int>("nodeType")]));
+                    }
+                }
+
+                AddNodesToIndex(xElements, type);
+                pageIndex++;
+            } while (xElements.Length == pageSize);
+        }
+
+        internal static IEnumerable<XElement> GetSerializedContent(
+            bool supportUnpublishdContent, 
+            Func<IContent, XElement> serializer, 
+            IEnumerable<IContent> content, 
+            ISet<string> notPublished)
         {            
             foreach (var c in content)
             {
-                if (SupportUnpublishedContent == false)
+                if (supportUnpublishdContent == false)
                 {
                     //if we don't support published content and this is not published then track it and return null
                     if (c.Published == false)
                     {
                         notPublished.Add(c.Path);
                         yield return null;
+                        continue;
                     }
 
                     //if we don't support published content, check if this content item exists underneath any already tracked
@@ -491,14 +652,11 @@ namespace UmbracoExamine
                     if (notPublished.Any(path => c.Path.StartsWith(string.Format("{0},", path))))
                     {
                         yield return null;
+                        continue;
                     }
-                }
+                }                
 
-                var xml = _serializer.Serialize(
-                    _contentService,
-                    _dataTypeService,
-                    _userService,
-                    c);
+                var xml = serializer(c);                
 
                 //add a custom 'icon' attribute
                 xml.Add(new XAttribute("icon", c.ContentType.Icon));
@@ -520,7 +678,7 @@ namespace UmbracoExamine
 
         public override void RebuildIndex()
         {
-            DataService.LogService.AddVerboseLog(-1, "Rebuilding index");
+            DataService.LogService.AddInfoLog(-1, "Rebuilding index");
             base.RebuildIndex();
         }
 

src/UmbracoExamine/UmbracoExamine.csproj

@@ -82,8 +82,9 @@
     <AssemblyOriginatorKeyFile>..\Solution Items\TheFARM-Public.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.80.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.80\lib\net45\Examine.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>

src/UmbracoExamine/UmbracoExamineSearcher.cs

@@ -51,12 +51,41 @@ namespace UmbracoExamine
         /// </summary>
         public override string Name
         {
-            get
-            {
-                return _name;
-            }
+            get { return _name; }
+        }
+        
+        /// <summary>
+        /// Constructor to allow for creating an indexer at runtime
+        /// </summary>
+        /// <param name="indexPath"></param>
+        /// <param name="analyzer"></param>
+
+        public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer)
+            : base(indexPath, analyzer)
+        {
         }
 
+        /// <summary>
+        /// Constructor to allow for creating an indexer at runtime
+        /// </summary>
+        /// <param name="luceneDirectory"></param>
+        /// <param name="analyzer"></param>
+        public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer)
+            : base(luceneDirectory, analyzer)
+        {
+        }
+
+        /// <summary>
+        /// Creates an NRT searcher
+        /// </summary>
+        /// <param name="writer"></param>
+        /// <param name="analyzer"></param>
+        public UmbracoExamineSearcher(IndexWriter writer, Analyzer analyzer) 
+            : base(writer, analyzer)
+        {
+        }
+
+        #endregion
 
         public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
         {
@@ -115,30 +144,6 @@ namespace UmbracoExamine
             }
         }
 
-        /// <summary>
-        /// Constructor to allow for creating an indexer at runtime
-        /// </summary>
-        /// <param name="indexPath"></param>
-        /// <param name="analyzer"></param>
-
-        public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer)
-            : base(indexPath, analyzer)
-        {
-        }
-
-        /// <summary>
-        /// Constructor to allow for creating an indexer at runtime
-        /// </summary>
-        /// <param name="luceneDirectory"></param>
-        /// <param name="analyzer"></param>
-
-        public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer)
-            : base(luceneDirectory, analyzer)
-        {
-        }
-
-        #endregion
-
         /// <summary>
         /// Used for unit tests
         /// </summary>

src/UmbracoExamine/UmbracoMemberIndexer.cs

@@ -9,6 +9,8 @@ using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Services;
 using UmbracoExamine.Config;
 using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
 using Examine;
 using System.IO;
 using UmbracoExamine.DataServices;
@@ -24,6 +26,7 @@ namespace UmbracoExamine
     {
 
         private readonly IMemberService _memberService;
+        private readonly IMemberTypeService _memberTypeService;
         private readonly IDataTypeService _dataTypeService;
 
         /// <summary>
@@ -33,6 +36,7 @@ namespace UmbracoExamine
         {
             _dataTypeService = ApplicationContext.Current.Services.DataTypeService;
             _memberService = ApplicationContext.Current.Services.MemberService;
+            _memberTypeService = ApplicationContext.Current.Services.MemberTypeService;
         }
 
         /// <summary>
@@ -48,6 +52,7 @@ namespace UmbracoExamine
         {
             _dataTypeService = ApplicationContext.Current.Services.DataTypeService;
             _memberService = ApplicationContext.Current.Services.MemberService;
+            _memberTypeService = ApplicationContext.Current.Services.MemberTypeService;
         }
 
         /// <summary>
@@ -60,6 +65,8 @@ namespace UmbracoExamine
         /// <param name="memberService"></param>
         /// <param name="analyzer"></param>
         /// <param name="async"></param>
+        [Obsolete("Use the ctor specifying all dependencies instead")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService,
               IDataTypeService dataTypeService,
               IMemberService memberService,
@@ -68,9 +75,31 @@ namespace UmbracoExamine
         {
             _dataTypeService = dataTypeService;
             _memberService = memberService;
+            _memberTypeService = ApplicationContext.Current.Services.MemberTypeService;
         }
 
-
+        /// <summary>
+        /// Constructor to allow for creating an indexer at runtime
+        /// </summary>
+        /// <param name="indexerData"></param>
+        /// <param name="indexPath"></param>
+        /// <param name="dataService"></param>
+        /// <param name="dataTypeService"></param>
+        /// <param name="memberService"></param>
+        /// <param name="memberTypeService"></param>
+        /// <param name="analyzer"></param>
+        /// <param name="async"></param>
+        public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService,
+              IDataTypeService dataTypeService,
+              IMemberService memberService,
+              IMemberTypeService memberTypeService,
+              Analyzer analyzer, bool async)
+            : base(indexerData, indexPath, dataService, analyzer, async)
+        {
+            _dataTypeService = dataTypeService;
+            _memberService = memberService;
+            _memberTypeService = memberTypeService;
+        }
 
         /// <summary>
         /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config
@@ -129,40 +158,68 @@ namespace UmbracoExamine
             if (SupportedTypes.Contains(type) == false)
                 return;
 
-            const int pageSize = 1000;
-            var pageIndex = 0;
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - Start data queries - {0}", type));
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
 
-            IMember[] members;
-
-            if (IndexerData.IncludeNodeTypes.Any())
+            try
             {
-                //if there are specific node types then just index those
-                foreach (var nodeType in IndexerData.IncludeNodeTypes)
+                if (DisableXmlDocumentLookup == false)
                 {
-                    do
+                    ReindexWithXmlEntries(type, -1,
+                        () => _memberTypeService.GetAll().ToArray(),
+                        (path, pIndex, pSize) =>
+                        {
+                            long totalContent;
+                            var result = _memberService.GetPagedXmlEntries(pIndex, pSize, out totalContent).ToArray();
+                            return new Tuple<long, XElement[]>(totalContent, result);
+                        },
+                        i => _memberService.GetById(i));
+                }
+                else
+                {
+                    const int pageSize = 1000;
+                    var pageIndex = 0;
+
+                    IMember[] members;
+
+                    if (IndexerData.IncludeNodeTypes.Any())
                     {
-                        long total;
-                        members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
+                        //if there are specific node types then just index those
+                        foreach (var nodeType in IndexerData.IncludeNodeTypes)
+                        {
+                            do
+                            {
+                                long total;
+                                members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
 
-                        AddNodesToIndex(GetSerializedMembers(members), type);
+                                AddNodesToIndex(GetSerializedMembers(members), type);
 
-                        pageIndex++;
-                    } while (members.Length == pageSize);
+                                pageIndex++;
+                            } while (members.Length == pageSize);
+                        }
+                    }
+                    else
+                    {
+                        //no node types specified, do all members
+                        do
+                        {
+                            int total;
+                            members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray();
+
+                            AddNodesToIndex(GetSerializedMembers(members), type);
+
+                            pageIndex++;
+                        } while (members.Length == pageSize);
+                    }
                 }
             }
-            else
+            finally
             {
-                //no node types specified, do all members
-                do
-                {
-                    int total;
-                    members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray();
-
-                    AddNodesToIndex(GetSerializedMembers(members), type);
-
-                    pageIndex++;
-                } while (members.Length == pageSize);
+                stopwatch.Stop();
             }
+            
+            DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - End data queries - {0}, took {1}ms", type, stopwatch.ElapsedMilliseconds));
         }
 
         private IEnumerable<XElement> GetSerializedMembers(IEnumerable<IMember> members)
@@ -181,7 +238,9 @@ namespace UmbracoExamine
             var fields = base.GetSpecialFieldsToIndex(allValuesForIndexing);
 
             //adds the special path property to the index
-            fields.Add("__key", allValuesForIndexing["__key"]);
+            string valuesForIndexing;
+            if (allValuesForIndexing.TryGetValue("__key", out valuesForIndexing))
+                fields.Add("__key", valuesForIndexing);
 
             return fields;
 

src/UmbracoExamine/packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.80" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="SharpZipLib" version="0.86.0" targetFramework="net45" />
 </packages>

src/umbraco.MacroEngines/packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Examine" version="0.1.70.0" targetFramework="net45" />
+  <package id="Examine" version="0.1.80" targetFramework="net45" />
   <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net45" />
   <package id="Lucene.Net" version="2.9.4.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.Mvc" version="5.2.3" targetFramework="net45" />

src/umbraco.MacroEngines/umbraco.MacroEngines.csproj

@@ -45,8 +45,9 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Examine, Version=0.1.70.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Examine.0.1.70.0\lib\Examine.dll</HintPath>
+    <Reference Include="Examine, Version=0.1.80.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Examine.0.1.80\lib\net45\Examine.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
       <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>

src/umbraco.cms/businesslogic/Tags/Tag.cs

@@ -351,7 +351,7 @@ namespace umbraco.cms.businesslogic.Tags
                 {
                     Document cnode = new Document(rr.GetInt("nodeid"));
 
-                    if (cnode != null && cnode.Published)
+                    if (cnode.Published)
                         docs.Add(cnode);
                 }
             }

src/umbraco.cms/businesslogic/propertytype/propertytype.cs

@@ -54,32 +54,31 @@ namespace umbraco.cms.businesslogic.propertytype
 
         public PropertyType(int id)
         {
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader dr = sqlHelper.ExecuteReader(
-                "Select mandatory, DataTypeId, propertyTypeGroupId, ContentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=@id",
-                sqlHelper.CreateParameter("@id", id)))
+            var found = ApplicationContext.Current.DatabaseContext.Database
+                .SingleOrDefault<dynamic>(
+                    "Select mandatory, DataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=@id",
+                    new {id = id});
+
+            if (found == null)
+                throw new ArgumentException("Propertytype with id: " + id + " doesnt exist!");
+
+            _mandatory = found.mandatory;
+            _id = id;
+
+            if (found.propertyTypeGroupId != null)
             {
-                if (!dr.Read())
-                    throw new ArgumentException("Propertytype with id: " + id + " doesnt exist!");
-
-                _mandatory = dr.GetBoolean("mandatory");
-                _id = id;
-
-                if (!dr.IsNull("propertyTypeGroupId"))
-                {
-                    _propertyTypeGroup = dr.GetInt("propertyTypeGroupId");
-                    //TODO: Remove after refactoring!
-                    _tabId = _propertyTypeGroup;
-                }
-
-                _sortOrder = dr.GetInt("sortOrder");
-                _alias = dr.GetString("alias");
-                _name = dr.GetString("Name");
-                _validationRegExp = dr.GetString("validationRegExp");
-                _DataTypeId = dr.GetInt("DataTypeId");
-                _contenttypeid = dr.GetInt("contentTypeId");
-                _description = dr.GetString("description");
+                _propertyTypeGroup = found.propertyTypeGroupId;
+                //TODO: Remove after refactoring!
+                _tabId = _propertyTypeGroup;
             }
+
+            _sortOrder = found.sortOrder;
+            _alias = found.alias;
+            _name = found.name;
+            _validationRegExp = found.validationRegExp;
+            _DataTypeId = found.DataTypeId;
+            _contenttypeid = found.contentTypeId;
+            _description = found.description;
         }
 
         #endregion
@@ -92,7 +91,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _DataTypeId = value.Id;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery(
                         "Update cmsPropertyType set DataTypeId = " + value.Id + " where id=" + Id);
@@ -119,7 +117,6 @@ namespace umbraco.cms.businesslogic.propertytype
             {
                 _tabId = value;
                 PropertyTypeGroup = value;
-                InvalidateCache();
             }
         }
 
@@ -148,7 +145,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _mandatory = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery("Update cmsPropertyType set mandatory = @mandatory where id = @id",
                         sqlHelper.CreateParameter("@mandatory", value),
@@ -162,7 +158,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _validationRegExp = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery("Update cmsPropertyType set validationRegExp = @validationRegExp where id = @id",
                         sqlHelper.CreateParameter("@validationRegExp", value), sqlHelper.CreateParameter("@id", Id));
@@ -199,7 +194,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _description = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery("Update cmsPropertyType set description = @description where id = @id",
                         sqlHelper.CreateParameter("@description", value),
@@ -213,7 +207,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _sortOrder = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery("Update cmsPropertyType set sortOrder = @sortOrder where id = @id",
                         sqlHelper.CreateParameter("@sortOrder", value),
@@ -227,7 +220,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _alias = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery("Update cmsPropertyType set alias = @alias where id= @id",
                         sqlHelper.CreateParameter("@alias", Casing.SafeAliasWithForcingCheck(_alias)),
@@ -264,7 +256,6 @@ namespace umbraco.cms.businesslogic.propertytype
             set
             {
                 _name = value;
-                InvalidateCache();
                 using (var sqlHelper = Application.SqlHelper)
                     sqlHelper.ExecuteNonQuery(
                         "UPDATE cmsPropertyType SET name=@name WHERE id=@id",
@@ -331,17 +322,17 @@ namespace umbraco.cms.businesslogic.propertytype
         public static IEnumerable<PropertyType> GetPropertyTypes()
         {
             var result = new List<PropertyType>();
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader dr = 
-                sqlHelper.ExecuteReader("select id from cmsPropertyType order by Name"))
+
+            var propertyTypeIds = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                "select id from cmsPropertyType order by Name");
+
+            foreach (var propertyTypeId in propertyTypeIds)
             {
-                while (dr.Read())
-                {
-                    PropertyType pt = GetPropertyType(dr.GetInt("id"));
-                    if (pt != null)
-                        result.Add(pt);
-                }
+                PropertyType pt = GetPropertyType(propertyTypeId);
+                if (pt != null)
+                    result.Add(pt);
             }
+            
             return result;
         }
 
@@ -353,18 +344,17 @@ namespace umbraco.cms.businesslogic.propertytype
 		public static IEnumerable<PropertyType> GetPropertyTypesByGroup(int groupId)
         {
             var result = new List<PropertyType>();
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader dr = 
-                sqlHelper.ExecuteReader("SELECT id FROM cmsPropertyType WHERE propertyTypeGroupId = @groupId order by SortOrder",
-                    sqlHelper.CreateParameter("@groupId", groupId)))
+
+            var propertyTypeIds = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                "SELECT id FROM cmsPropertyType WHERE propertyTypeGroupId = @groupId order by SortOrder", new {groupId = groupId});
+
+            foreach (var propertyTypeId in propertyTypeIds)
             {
-                while (dr.Read())
-                {
-                    PropertyType pt = GetPropertyType(dr.GetInt("id"));
-                    if (pt != null)
-                        result.Add(pt);
-                }
+                PropertyType pt = GetPropertyType(propertyTypeId);
+                if (pt != null)
+                    result.Add(pt);
             }
+            
             return result;
         }
 
@@ -376,20 +366,18 @@ namespace umbraco.cms.businesslogic.propertytype
         public static IEnumerable<PropertyType> GetByDataTypeDefinition(int dataTypeDefId)
         {
             var result = new List<PropertyType>();
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader dr = 
-                sqlHelper.ExecuteReader(
-                    "select id, Name from cmsPropertyType where dataTypeId=@dataTypeId order by Name",
-                    sqlHelper.CreateParameter("@dataTypeId", dataTypeDefId)))
+
+            var propertyTypeIds = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                "select id from cmsPropertyType where dataTypeId=@dataTypeId order by Name", new {dataTypeId = dataTypeDefId});
+
+            foreach (var propertyTypeId in propertyTypeIds)
             {
-                while (dr.Read())
-                {
-                    PropertyType pt = GetPropertyType(dr.GetInt("id"));
-                    if (pt != null)
-                        result.Add(pt);
-                }
+                PropertyType pt = GetPropertyType(propertyTypeId);
+                if (pt != null)
+                    result.Add(pt);
             }
-            return result.ToList();
+            
+            return result;
         }
 
         public void delete()
@@ -411,7 +399,6 @@ namespace umbraco.cms.businesslogic.propertytype
 
             // delete cache from either master (via tabid) or current contentype
             FlushCacheBasedOnTab();
-            InvalidateCache();
         }
 
         public void FlushCacheBasedOnTab()
@@ -478,8 +465,6 @@ namespace umbraco.cms.businesslogic.propertytype
 
         protected virtual void FlushCache()
         {
-            // clear local cache
-            ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(GetCacheKey(Id));
 
             // clear cache in contentype
             ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(CacheKeys.ContentTypePropertiesCacheKey + _contenttypeid);
@@ -496,31 +481,9 @@ namespace umbraco.cms.businesslogic.propertytype
 
         public static PropertyType GetPropertyType(int id)
         {
-            return ApplicationContext.Current.ApplicationCache.RuntimeCache.GetCacheItem<PropertyType>(
-                GetCacheKey(id),
-                timeout:        TimeSpan.FromMinutes(30),
-                getCacheItem: () =>
-                {
-                    try
-                    {
-                        return new PropertyType(id);
-                    }
-                    catch
-                    {
-                        return null;
-                    }
-                });
-        }
-
-        private void InvalidateCache()
-        {
-            ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(GetCacheKey(Id));
-        }
-
-        private static string GetCacheKey(int id)
-        {
-            return CacheKeys.PropertyTypeCacheKey + id;
+            return new PropertyType(id);
         }
+        
 
         #endregion
     }

src/umbraco.cms/businesslogic/web/Document.cs

@@ -433,21 +433,20 @@ namespace umbraco.cms.businesslogic.web
         {
             XmlDocument xd = new XmlDocument();
 
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader dr = sqlHelper.ExecuteReader("select nodeId from cmsDocument"))
+            var nodeIds = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+                "select nodeId from cmsDocument");
+
+            foreach (var nodeId in nodeIds)
             {
-                while (dr.Read())
+                try
                 {
-                    try
-                    {
-                        new Document(dr.GetInt("nodeId")).SaveXmlPreview(xd);
-                    }
-                    catch (Exception ee)
-                    {
-                        LogHelper.Error<Document>("Error generating preview xml", ee);
-                    }
+                    new Document(nodeId).SaveXmlPreview(xd);
                 }
-            }
+                catch (Exception ee)
+                {
+                    LogHelper.Error<Document>("Error generating preview xml", ee);
+                }
+            }            
         }
 
         /// <summary>

src/umbraco.editorControls/PickerRelations/PickerRelationsEventHandler.cs

@@ -2,8 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
-
-using umbraco.BusinessLogic; // ApplicationBase
+// ApplicationBase
 using umbraco.businesslogic;
 using umbraco.cms.businesslogic; // SaveEventArgs
 using umbraco.cms.businesslogic.media; // Media
@@ -12,6 +11,8 @@ using umbraco.cms.businesslogic.web; // Documentusing umbraco.cms.businesslogic.
 using umbraco.cms.businesslogic.property;
 using umbraco.cms.businesslogic.relation;
 using umbraco.DataLayer;
+using Umbraco.Core;
+using Application = umbraco.BusinessLogic.Application;
 
 namespace umbraco.editorControls.PickerRelations
 {
@@ -212,7 +213,7 @@ namespace umbraco.editorControls.PickerRelations
 		private static void DeleteRelations(RelationType relationType, int contentNodeId, bool reverseIndexing, string instanceIdentifier)
 		{
 			//if relationType is bi-directional or a reverse index then we can't get at the relations via the API, so using SQL
-			string getRelationsSql = "SELECT id FROM umbracoRelation WHERE relType = " + relationType.Id.ToString() + " AND ";
+			string getRelationsSql = "SELECT id FROM umbracoRelation WHERE relType = " + relationType.Id + " AND ";
 
 			if (reverseIndexing || relationType.Dual)
 			{
@@ -229,19 +230,16 @@ namespace umbraco.editorControls.PickerRelations
 
 			getRelationsSql += " AND comment = '" + instanceIdentifier + "'";
 
-            using (var sqlHelper = Application.SqlHelper)
-            using (IRecordsReader relations = sqlHelper.ExecuteReader(getRelationsSql))
-			{
-				//clear data
-				Relation relation;
-			    while (relations.Read())
-				{
-				    relation = new Relation(relations.GetInt("id"));
+		    var relationIds = ApplicationContext.Current.DatabaseContext.Database.Fetch<int>(
+		        getRelationsSql);
+		    foreach (var relationId in relationIds)
+		    {
+                var relation = new Relation(relationId);
 
-				    // TODO: [HR] check to see if an instance identifier is used
-					relation.Delete();
-				}
-			}
+                // TODO: [HR] check to see if an instance identifier is used
+                relation.Delete();
+            }
+            
 		}
 
 		/// <summary>