Merge temp8 into temp8-3675-variant-tags

src/Umbraco.Core/Collections/ConcurrentHashSet.cs

@@ -70,10 +70,7 @@ namespace Umbraco.Core.Collections
         /// The number of elements contained in the <see cref="T:System.Collections.ICollection"/>.
         /// </returns>
         /// <filterpriority>2</filterpriority>
-        public int Count
-        {
-            get { return GetThreadSafeClone().Count; }
-        }
+        public int Count => GetThreadSafeClone().Count;
 
         /// <summary>
         /// Gets a value indicating whether the <see cref="T:System.Collections.Generic.ICollection`1"/> is read-only.
@@ -81,10 +78,7 @@ namespace Umbraco.Core.Collections
         /// <returns>
         /// true if the <see cref="T:System.Collections.Generic.ICollection`1"/> is read-only; otherwise, false.
         /// </returns>
-        public bool IsReadOnly
-        {
-            get { return false; }
-        }
+        public bool IsReadOnly => false;
 
         /// <summary>
         /// Adds an item to the <see cref="T:System.Collections.Generic.ICollection`1"/>.

src/Umbraco.Core/Constants-Indexes.cs

@@ -0,0 +1,19 @@
+using System;
+using System.ComponentModel;
+
+namespace Umbraco.Core
+{
+    public static partial class Constants
+    {
+        public static class UmbracoIndexes
+        {
+            public const string InternalIndexName = InternalIndexPath + "Index";
+            public const string ExternalIndexName = ExternalIndexPath + "Index";
+            public const string MembersIndexName = MembersIndexPath + "Index";
+
+            public const string InternalIndexPath = "Internal";
+            public const string ExternalIndexPath = "External";
+            public const string MembersIndexPath = "Members";
+        }
+    }
+}

src/Umbraco.Core/EnumerableExtensions.cs

@@ -1,11 +1,6 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.ComponentModel;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
-using System.Text;
-using Umbraco.Core.Logging;
 
 namespace Umbraco.Core
 {
@@ -14,6 +9,18 @@ namespace Umbraco.Core
     ///</summary>
     public static class EnumerableExtensions
     {
+        /// <summary>
+        /// Wraps this object instance into an IEnumerable{T} consisting of a single item.
+        /// </summary>
+        /// <typeparam name="T"> Type of the object. </typeparam>
+        /// <param name="item"> The instance that will be wrapped. </param>
+        /// <returns> An IEnumerable{T} consisting of a single item. </returns>
+        public static IEnumerable<T> Yield<T>(this T item)
+        {
+            // see EnumeratorBenchmarks - this is faster, and allocates less, than returning an array
+            yield return item;
+        }
+
         public static IEnumerable<IEnumerable<T>> InGroupsOf<T>(this IEnumerable<T> source, int groupSize)
         {
             if (source == null)

src/Umbraco.Core/Models/GridValue.cs

@@ -5,6 +5,8 @@ using Newtonsoft.Json.Linq;
 
 namespace Umbraco.Core.Models
 {
+    //TODO: Make a property value converter for this!
+
     /// <summary>
     /// A model representing the value saved for the grid
     /// </summary>
@@ -19,7 +21,7 @@ namespace Umbraco.Core.Models
         public class GridSection
         {
             [JsonProperty("grid")]
-            public string Grid { get; set; }
+            public string Grid { get; set; } //fixme: what is this?
 
             [JsonProperty("rows")]
             public IEnumerable<GridRow> Rows { get; set; }
@@ -46,7 +48,7 @@ namespace Umbraco.Core.Models
         public class GridArea
         {
             [JsonProperty("grid")]
-            public string Grid { get; set; }
+            public string Grid { get; set; } //fixme: what is this?
 
             [JsonProperty("controls")]
             public IEnumerable<GridControl> Controls { get; set; }

src/Umbraco.Core/Models/PagedResult.cs

@@ -19,7 +19,7 @@ namespace Umbraco.Core.Models
 
             if (pageSize > 0)
             {
-                TotalPages = (long)Math.Ceiling(totalItems / (Decimal)pageSize);
+                TotalPages = (long)Math.Ceiling(totalItems / (decimal)pageSize);
             }
             else
             {

src/Umbraco.Core/Models/PublicAccessEntry.cs

@@ -21,6 +21,10 @@ namespace Umbraco.Core.Models
 
         public PublicAccessEntry(IContent protectedNode, IContent loginNode, IContent noAccessNode, IEnumerable<PublicAccessRule> ruleCollection)
         {
+            if (protectedNode == null) throw new ArgumentNullException(nameof(protectedNode));
+            if (loginNode == null) throw new ArgumentNullException(nameof(loginNode));
+            if (noAccessNode == null) throw new ArgumentNullException(nameof(noAccessNode));
+
             LoginNodeId = loginNode.Id;
             NoAccessNodeId = noAccessNode.Id;
             _protectedNodeId = protectedNode.Id;

src/Umbraco.Core/ObjectExtensions.cs

@@ -723,8 +723,8 @@ namespace Umbraco.Core
             {
                 return typeConverter;
             }
-
-            TypeConverter converter = TypeDescriptor.GetConverter(target);
+            
+            var converter = TypeDescriptor.GetConverter(target);
             if (converter.CanConvertFrom(source))
             {
                 return DestinationTypeConverterCache[key] = converter;

src/Umbraco.Core/PropertyEditors/DataEditor.cs

@@ -153,6 +153,9 @@ namespace Umbraco.Core.PropertyEditors
             set => _defaultConfiguration = value;
         }
 
+        /// <inheritdoc />
+        public virtual IPropertyIndexValueFactory PropertyIndexValueFactory => new DefaultPropertyIndexValueFactory();
+
         /// <summary>
         /// Creates a value editor instance.
         /// </summary>

src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValueFactory.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using Umbraco.Core.Models;
+
+namespace Umbraco.Core.PropertyEditors
+{
+    /// <summary>
+    /// Provides a default implementation for <see ref="IPropertyIndexValueFactory">, returning a single field to index containing the property value.
+    /// </summary>
+    public class DefaultPropertyIndexValueFactory : IPropertyIndexValueFactory
+    {
+        /// <inheritdoc />
+        public IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published)
+        {
+            yield return new KeyValuePair<string, IEnumerable<object>>(
+                property.Alias,
+                property.GetValue(culture, segment, published).Yield());
+        }
+    }
+}

src/Umbraco.Core/PropertyEditors/IDataEditor.cs

@@ -65,5 +65,10 @@ namespace Umbraco.Core.PropertyEditors
         /// <para>Is expected to throw if the editor does not support being configured, e.g. for most parameter editors.</para>
         /// </remarks>
         IConfigurationEditor GetConfigurationEditor();
+
+        /// <summary>
+        /// Gets the index value factory for the editor.
+        /// </summary>
+        IPropertyIndexValueFactory PropertyIndexValueFactory { get; }
     }
-}
+}

src/Umbraco.Core/PropertyEditors/IPropertyIndexValueFactory.cs

@@ -0,0 +1,24 @@
+using System.Collections.Generic;
+using Umbraco.Core.Models;
+
+namespace Umbraco.Core.PropertyEditors
+{
+    /// <summary>
+    /// Represents a property index value factory.
+    /// </summary>
+    public interface IPropertyIndexValueFactory
+    {
+        /// <summary>
+        /// Gets the index values for a property.
+        /// </summary>
+        /// <remarks>
+        /// <para>Returns key-value pairs, where keys are indexed field names. By default, that would be the property alias,
+        /// and there would be only one pair, but some implementations (see for instance the grid one) may return more than
+        /// one pair, with different indexed field names.</para>
+        /// <para>And then, values are an enumerable of objects, because each indexed field can in turn have multiple
+        /// values. By default, there would be only one object: the property value. But some implementations may return
+        /// more than one value for a given field.</para>
+        /// </remarks>
+        IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published);
+    }
+}

src/Umbraco.Core/Services/IContentService.cs

@@ -176,10 +176,8 @@ namespace Umbraco.Core.Services
         /// <param name="pageIndex">The page number.</param>
         /// <param name="pageSize">The page size.</param>
         /// <param name="totalRecords">Total number of documents.</param>
-        /// <param name="orderBy">A field to order by.</param>
-        /// <param name="orderDirection">The ordering direction.</param>
-        /// <param name="orderBySystemField">A flag indicating whether the ordering field is a system field.</param>
         /// <param name="filter">Query filter.</param>
+        /// <param name="ordering">Ordering infos.</param>
         IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
             IQuery<IContent> filter = null, Ordering ordering = null);
 

src/Umbraco.Core/StringExtensions.cs

@@ -540,7 +540,7 @@ namespace Umbraco.Core
         public static string StripHtml(this string text)
         {
             const string pattern = @"<(.|\n)*?>";
-            return Regex.Replace(text, pattern, string.Empty);
+            return Regex.Replace(text, pattern, string.Empty, RegexOptions.Compiled);
         }
 
         /// <summary>

src/Umbraco.Core/Umbraco.Core.csproj

@@ -313,6 +313,7 @@
     <Compile Include="Constants-PropertyTypeGroups.cs" />
     <Compile Include="Constants-Security.cs" />
     <Compile Include="Constants-System.cs" />
+    <Compile Include="Constants-Indexes.cs" />
     <Compile Include="Constants-Web.cs" />
     <Compile Include="Constants.cs" />
     <Compile Include="ContentVariationExtensions.cs" />
@@ -442,6 +443,7 @@
     <Compile Include="PropertyEditors\ConfigurationEditorOfTConfiguration.cs" />
     <Compile Include="PropertyEditors\DataEditorAttribute.cs" />
     <Compile Include="PropertyEditors\ConfigurationEditor.cs" />
+    <Compile Include="PropertyEditors\DefaultPropertyIndexValueFactory.cs" />
     <Compile Include="PropertyEditors\DropDownFlexibleConfiguration.cs" />
     <Compile Include="PropertyEditors\EditorType.cs" />
     <Compile Include="PropertyEditors\IConfigurationEditor.cs" />
@@ -450,6 +452,7 @@
     <Compile Include="PropertyEditors\ImageCropperConfiguration.cs" />
     <Compile Include="PropertyEditors\IManifestValueValidator.cs" />
     <Compile Include="PropertyEditors\IValueFormatValidator.cs" />
+    <Compile Include="PropertyEditors\IPropertyIndexValueFactory.cs" />
     <Compile Include="PropertyEditors\IValueRequiredValidator.cs" />
     <Compile Include="PropertyEditors\LabelConfiguration.cs" />
     <Compile Include="PropertyEditors\LabelConfigurationEditor.cs" />

src/Umbraco.Examine/BaseValueSetBuilder.cs

@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.PropertyEditors;
+
+namespace Umbraco.Examine
+{
+
+    /// <inheritdoc />
+    public abstract class BaseValueSetBuilder<TContent> : IValueSetBuilder<TContent>
+        where TContent : IContentBase
+    {
+        protected bool PublishedValuesOnly { get; }
+        private readonly PropertyEditorCollection _propertyEditors;
+
+        protected BaseValueSetBuilder(PropertyEditorCollection propertyEditors, bool publishedValuesOnly)
+        {
+            PublishedValuesOnly = publishedValuesOnly;
+            _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors));
+        }
+
+        /// <inheritdoc />
+        public abstract IEnumerable<ValueSet> GetValueSets(params TContent[] content);
+
+        protected void AddPropertyValue(Property property, string culture, string segment, IDictionary<string, IEnumerable<object>> values)
+        {
+            var editor = _propertyEditors[property.PropertyType.PropertyEditorAlias];
+            if (editor == null) return;
+
+            var indexVals = editor.PropertyIndexValueFactory.GetIndexValues(property, culture, segment, PublishedValuesOnly);
+            foreach (var keyVal in indexVals)
+            {
+                if (keyVal.Key.IsNullOrWhiteSpace()) continue;
+
+                var cultureSuffix = culture == null ? string.Empty : "_" + culture;
+
+                foreach (var val in keyVal.Value)
+                {
+                    switch (val)
+                    {
+                        //only add the value if its not null or empty (we'll check for string explicitly here too)
+                        case null:
+                            continue;
+                        case string strVal:
+                            {
+                                if (strVal.IsNullOrWhiteSpace()) return;
+                                var key = $"{keyVal.Key}{cultureSuffix}";
+                                if (values.TryGetValue(key, out var v))
+                                    values[key] = new List<object>(v) { val }.ToArray();
+                                else
+                                    values.Add($"{keyVal.Key}{cultureSuffix}", val.Yield());
+                            }
+                            break;
+                        default:
+                            {
+                                var key = $"{keyVal.Key}{cultureSuffix}";
+                                if (values.TryGetValue(key, out var v))
+                                    values[key] = new List<object>(v) { val }.ToArray();
+                                else
+                                    values.Add($"{keyVal.Key}{cultureSuffix}", val.Yield());
+                            }
+                            
+                            break;
+                    }
+                }
+            }
+        }
+    }
+
+}

src/Umbraco.Examine/Config/IndexFieldCollection.cs

@@ -0,0 +1,47 @@
+using System.Configuration;
+
+namespace Umbraco.Examine.Config
+{
+    public sealed class IndexFieldCollection : ConfigurationElementCollection
+    {
+        #region Overridden methods to define collection
+        protected override ConfigurationElement CreateNewElement()
+        {
+            return new ConfigIndexField();
+        }
+        protected override object GetElementKey(ConfigurationElement element)
+        {
+            ConfigIndexField field = (ConfigIndexField)element;
+            return field.Name;
+        }
+
+        public override bool IsReadOnly()
+        {
+            return false;
+        }
+        #endregion
+
+        /// <summary>
+        /// Adds an index field to the collection
+        /// </summary>
+        /// <param name="field"></param>
+        public void Add(ConfigIndexField field)
+        {
+            BaseAdd(field, true);
+        }
+
+        /// <summary>
+        /// Default property for accessing an IndexField definition
+        /// </summary>
+        /// <value>Field Name</value>
+        /// <returns></returns>
+        public new ConfigIndexField this[string name]
+        {
+            get
+            {
+                return (ConfigIndexField)this.BaseGet(name);
+            }
+        }
+
+    }
+}

src/Umbraco.Examine/Config/IndexSet.cs

@@ -11,48 +11,6 @@ namespace Umbraco.Examine.Config
         [ConfigurationProperty("SetName", IsRequired = true, IsKey = true)]
         public string SetName => (string)this["SetName"];
 
-        private string _indexPath = "";
-
-        /// <summary>
-        /// The folder path of where the lucene index is stored
-        /// </summary>
-        /// <value>The index path.</value>
-        /// <remarks>
-        /// This can be set at runtime but will not be persisted to the configuration file
-        /// </remarks>
-        [ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)]
-        public string IndexPath
-        {
-            get
-            {
-                if (string.IsNullOrEmpty(_indexPath))
-                    _indexPath = (string)this["IndexPath"];
-
-                return _indexPath;
-            }
-            set => _indexPath = value;
-        }
-
-        /// <summary>
-        /// Returns the DirectoryInfo object for the index path.
-        /// </summary>
-        /// <value>The index directory.</value>
-        public DirectoryInfo IndexDirectory
-        {
-            get
-            {
-                //TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too
-
-                //we need to de-couple the context
-                if (HttpContext.Current != null)
-                    return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath));
-                else if (HostingEnvironment.ApplicationID != null)
-                    return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath));
-                else
-                    return new DirectoryInfo(this.IndexPath);
-            }
-        }
-
         /// <summary>
         /// When this property is set, the indexing will only index documents that are descendants of this node.
         /// </summary>

src/Umbraco.Examine/Config/IndexSetCollection.cs

@@ -3,50 +3,6 @@
 
 namespace Umbraco.Examine.Config
 {
-    public sealed class IndexFieldCollection : ConfigurationElementCollection
-    {
-        #region Overridden methods to define collection
-        protected override ConfigurationElement CreateNewElement()
-        {
-            return new ConfigIndexField();
-        }
-        protected override object GetElementKey(ConfigurationElement element)
-        {
-            ConfigIndexField field = (ConfigIndexField)element;
-            return field.Name;
-        }
-
-        public override bool IsReadOnly()
-        {
-            return false;
-        }
-        #endregion
-
-        /// <summary>
-        /// Adds an index field to the collection
-        /// </summary>
-        /// <param name="field"></param>
-        public void Add(ConfigIndexField field)
-        {
-            BaseAdd(field, true);
-        }
-
-        /// <summary>
-        /// Default property for accessing an IndexField definition
-        /// </summary>
-        /// <value>Field Name</value>
-        /// <returns></returns>
-        public new ConfigIndexField this[string name]
-        {
-            get
-            {
-                return (ConfigIndexField)this.BaseGet(name);
-            }
-        }
-
-    }
-
-
     public sealed class IndexSetCollection : ConfigurationElementCollection
     {
         #region Overridden methods to define collection

src/Umbraco.Examine/ContentIndexPopulator.cs

@@ -0,0 +1,101 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.Services;
+using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.DatabaseModelDefinitions;
+using Umbraco.Core.Persistence.Querying;
+
+namespace Umbraco.Examine
+{
+
+    /// <summary>
+    /// Performs the data lookups required to rebuild a content index
+    /// </summary>
+    public class ContentIndexPopulator : IndexPopulator
+    {
+        private readonly IContentService _contentService;
+        private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
+
+        /// <summary>
+        /// This is a static query, it's parameters don't change so store statically
+        /// </summary>
+        private static IQuery<IContent> _publishedQuery;
+
+        private readonly bool _publishedValuesOnly;
+        private readonly int? _parentId;
+
+        /// <summary>
+        /// Default constructor to lookup all content data
+        /// </summary>
+        /// <param name="contentService"></param>
+        /// <param name="sqlContext"></param>
+        /// <param name="contentValueSetBuilder"></param>
+        public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IContentValueSetBuilder contentValueSetBuilder)
+            : this(false, null, contentService, sqlContext, contentValueSetBuilder)
+        {   
+        }
+
+        /// <summary>
+        /// Optional constructor allowing specifying custom query parameters
+        /// </summary>
+        /// <param name="publishedValuesOnly"></param>
+        /// <param name="parentId"></param>
+        /// <param name="contentService"></param>
+        /// <param name="sqlContext"></param>
+        /// <param name="contentValueSetBuilder"></param>
+        public ContentIndexPopulator(bool publishedValuesOnly, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
+        {
+            if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
+            _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
+            _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
+            if (_publishedQuery != null)
+                _publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
+            _publishedValuesOnly = publishedValuesOnly;
+            _parentId = parentId;
+
+            RegisterIndex(Constants.UmbracoIndexes.InternalIndexName);
+            RegisterIndex(Constants.UmbracoIndexes.ExternalIndexName);
+        }
+
+        protected override void PopulateIndexes(IEnumerable<IIndex> indexes)
+        {
+            const int pageSize = 10000;
+            var pageIndex = 0;
+
+            var contentParentId = -1;
+            if (_parentId.HasValue && _parentId.Value > 0)
+            {
+                contentParentId = _parentId.Value;
+            }
+            IContent[] content;
+
+            do
+            {
+                if (!_publishedValuesOnly)
+                {
+                    content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _).ToArray();
+                }
+                else
+                {
+                    //add the published filter
+                    //note: We will filter for published variants in the validator
+                    content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _,
+                        _publishedQuery, Ordering.By("Path", Direction.Ascending)).ToArray();
+                }
+
+                if (content.Length > 0)
+                {
+                    // ReSharper disable once PossibleMultipleEnumeration
+                    foreach (var index in indexes)
+                        index.IndexItems(_contentValueSetBuilder.GetValueSets(content));
+                }
+
+                pageIndex++;
+            } while (content.Length == pageSize);
+        }
+    }
+}

src/Umbraco.Examine/ContentValueSetBuilder.cs

@@ -0,0 +1,106 @@
+using Examine;
+using System.Collections.Generic;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.PropertyEditors;
+using Umbraco.Core.Services;
+using Umbraco.Core.Strings;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Builds <see cref="ValueSet"/>s for <see cref="IContent"/> items
+    /// </summary>
+    public class ContentValueSetBuilder : BaseValueSetBuilder<IContent>, IContentValueSetBuilder, IPublishedContentValueSetBuilder
+    {
+        private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
+        private readonly IUserService _userService;
+
+        public ContentValueSetBuilder(PropertyEditorCollection propertyEditors,
+            IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
+            IUserService userService,
+            bool publishedValuesOnly)
+            : base(propertyEditors, publishedValuesOnly)
+        {
+            _urlSegmentProviders = urlSegmentProviders;
+            _userService = userService;
+        }
+
+        /// <inheritdoc />
+        public override IEnumerable<ValueSet> GetValueSets(params IContent[] content)
+        {
+            //TODO: There is a lot of boxing going on here and ultimately all values will be boxed by Lucene anyways
+            // but I wonder if there's a way to reduce the boxing that we have to do or if it will matter in the end since
+            // Lucene will do it no matter what? One idea was to create a `FieldValue` struct which would contain `object`, `object[]`, `ValueType` and `ValueType[]`
+            // references and then each array is an array of `FieldValue[]` and values are assigned accordingly. Not sure if it will make a difference or not.
+
+            foreach (var c in content)
+            {
+                var isVariant = c.ContentType.VariesByCulture();
+
+                var urlValue = c.GetUrlSegment(_urlSegmentProviders); //Always add invariant urlName
+                var values = new Dictionary<string, IEnumerable<object>>
+                {
+                    {"icon", c.ContentType.Icon.Yield()},
+                    {UmbracoExamineIndexer.PublishedFieldName, new object[] {c.Published ? 1 : 0}},   //Always add invariant published value
+                    {"id", new object[] {c.Id}},
+                    {"key", new object[] {c.Key}},
+                    {"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}},
+                    {"level", new object[] {c.Level}},
+                    {"creatorID", new object[] {c.CreatorId}},
+                    {"sortOrder", new object[] {c.SortOrder}},
+                    {"createDate", new object[] {c.CreateDate}},    //Always add invariant createDate
+                    {"updateDate", new object[] {c.UpdateDate}},    //Always add invariant updateDate
+                    {"nodeName", PublishedValuesOnly               //Always add invariant nodeName
+                        ? c.PublishName.Yield()
+                        : c.Name.Yield()},            
+                    {"urlName", urlValue.Yield()},                  //Always add invariant urlName
+                    {"path", c.Path.Yield()},
+                    {"nodeType", new object[] {c.ContentType.Id}},
+                    {"creatorName", (c.GetCreatorProfile(_userService)?.Name ?? "??").Yield() },
+                    {"writerName",(c.GetWriterProfile(_userService)?.Name ?? "??").Yield() },
+                    {"writerID", new object[] {c.WriterId}},
+                    {"template", new object[] {c.Template?.Id ?? 0}},
+                    {UmbracoContentIndexer.VariesByCultureFieldName, new object[] {0}},
+                };
+
+                if (isVariant)
+                {
+                    values[UmbracoContentIndexer.VariesByCultureFieldName] = new object[] { 1 };
+
+                    foreach (var culture in c.AvailableCultures)
+                    {
+                        var variantUrl = c.GetUrlSegment(_urlSegmentProviders, culture);
+                        var lowerCulture = culture.ToLowerInvariant();
+                        values[$"urlName_{lowerCulture}"] = variantUrl.Yield();
+                        values[$"nodeName_{lowerCulture}"] = PublishedValuesOnly
+                            ? c.GetPublishName(culture).Yield()
+                            : c.GetCultureName(culture).Yield();
+                        values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = (c.IsCulturePublished(culture) ? 1 : 0).Yield<object>();
+                        values[$"updateDate_{lowerCulture}"] = PublishedValuesOnly
+                            ? c.GetPublishDate(culture).Yield<object>()
+                            : c.GetUpdateDate(culture).Yield<object>();
+                    }
+                }
+
+                foreach (var property in c.Properties)
+                {
+                    if (!property.PropertyType.VariesByCulture())
+                    {
+                        AddPropertyValue(property, null, null, values);
+                    }
+                    else
+                    {
+                        foreach (var culture in c.AvailableCultures)
+                            AddPropertyValue(property, culture.ToLowerInvariant(), null, values);
+                    }
+                }
+
+                var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
+
+                yield return vs;
+            }
+        }
+    }
+
+}

src/Umbraco.Examine/ContentValueSetValidator.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Examine.LuceneEngine.Providers;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.Services;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Used to validate a ValueSet for content/media - based on permissions, parent id, etc....
+    /// </summary>
+    public class ContentValueSetValidator : ValueSetValidator, IContentValueSetValidator
+    {
+        private readonly IPublicAccessService _publicAccessService;
+
+        private const string PathKey = "path";
+        private static readonly IEnumerable<string> ValidCategories = new[] {IndexTypes.Content, IndexTypes.Media};
+        protected override IEnumerable<string> ValidIndexCategories => ValidCategories;
+
+        public bool PublishedValuesOnly { get; }
+        public bool SupportProtectedContent { get; }
+        public int? ParentId { get; }
+
+        public bool ValidatePath(string path, string category)
+        {
+            //check if this document is a descendent of the parent
+            if (ParentId.HasValue && ParentId.Value > 0)
+            {
+                // we cannot return FAILED here because we need the value set to get into the indexer and then deal with it from there
+                // because we need to remove anything that doesn't pass by parent Id in the cases that umbraco data is moved to an illegal parent.
+                if (!path.Contains(string.Concat(",", ParentId.Value, ",")))
+                    return false;
+            }
+
+            return true;
+        }
+
+        public bool ValidateRecycleBin(string path, string category)
+        {
+            var recycleBinId = category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
+
+            //check for recycle bin
+            if (PublishedValuesOnly)
+            {
+                if (path.Contains(string.Concat(",", recycleBinId, ",")))
+                    return false;
+            }
+            return true;
+        }
+
+        public bool ValidateProtectedContent(string path, string category)
+        {
+            if (category == IndexTypes.Content
+                && !SupportProtectedContent
+                //if the service is null we can't look this up so we'll return false
+                && (_publicAccessService == null || _publicAccessService.IsProtected(path)))
+            {
+                return false;
+            }
+
+            return true;
+        }
+
+        public ContentValueSetValidator(bool publishedValuesOnly, int? parentId = null,
+            IEnumerable<string> includeItemTypes = null, IEnumerable<string> excludeItemTypes = null)
+            : this(publishedValuesOnly, true, null, parentId, includeItemTypes, excludeItemTypes)
+        {
+        }
+
+        public ContentValueSetValidator(bool publishedValuesOnly, bool supportProtectedContent,
+            IPublicAccessService publicAccessService, int? parentId = null,
+            IEnumerable<string> includeItemTypes = null, IEnumerable<string> excludeItemTypes = null)
+            : base(includeItemTypes, excludeItemTypes, null, null)
+        {
+            PublishedValuesOnly = publishedValuesOnly;
+            SupportProtectedContent = supportProtectedContent;
+            ParentId = parentId;
+            _publicAccessService = publicAccessService;
+        }
+
+        public override ValueSetValidationResult Validate(ValueSet valueSet)
+        {
+            var baseValidate = base.Validate(valueSet);
+            if (baseValidate == ValueSetValidationResult.Failed)
+                return ValueSetValidationResult.Failed;
+
+            var isFiltered = baseValidate == ValueSetValidationResult.Filtered;
+
+            //check for published content
+            if (valueSet.Category == IndexTypes.Content && PublishedValuesOnly)
+            {
+                if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published))
+                    return ValueSetValidationResult.Failed;
+
+                if (!published[0].Equals(1))
+                    return ValueSetValidationResult.Failed;
+
+                //deal with variants, if there are unpublished variants than we need to remove them from the value set
+                if (valueSet.Values.TryGetValue(UmbracoContentIndexer.VariesByCultureFieldName, out var variesByCulture)
+                    && variesByCulture.Count > 0 && variesByCulture[0].Equals(1))
+                {
+                    //so this valueset is for a content that varies by culture, now check for non-published cultures and remove those values
+                    foreach(var publishField in valueSet.Values.Where(x => x.Key.StartsWith($"{UmbracoExamineIndexer.PublishedFieldName}_")).ToList())
+                    {
+                        if (publishField.Value.Count <= 0 || !publishField.Value[0].Equals(1))
+                        {
+                            //this culture is not published, so remove all of these culture values
+                            var cultureSuffix = publishField.Key.Substring(publishField.Key.LastIndexOf('_'));
+                            foreach (var cultureField in valueSet.Values.Where(x => x.Key.InvariantEndsWith(cultureSuffix)).ToList())
+                            {
+                                valueSet.Values.Remove(cultureField.Key);
+                                isFiltered = true;
+                            }
+                        }
+                    }
+                }
+            }
+
+            //must have a 'path'
+            if (!valueSet.Values.TryGetValue(PathKey, out var pathValues)) return ValueSetValidationResult.Failed;
+            if (pathValues.Count == 0) return ValueSetValidationResult.Failed;
+            if (pathValues[0] == null) return ValueSetValidationResult.Failed;
+            if (pathValues[0].ToString().IsNullOrWhiteSpace()) return ValueSetValidationResult.Failed;
+            var path = pathValues[0].ToString();
+
+            // We need to validate the path of the content based on ParentId, protected content and recycle bin rules.
+            // We cannot return FAILED here because we need the value set to get into the indexer and then deal with it from there
+            // because we need to remove anything that doesn't pass by protected content in the cases that umbraco data is moved to an illegal parent.
+            if (!ValidatePath(path, valueSet.Category)
+                || !ValidateRecycleBin(path, valueSet.Category)
+                || !ValidateProtectedContent(path, valueSet.Category))
+                return ValueSetValidationResult.Filtered;
+
+            return isFiltered ? ValueSetValidationResult.Filtered: ValueSetValidationResult.Valid;
+        }
+    }
+}

src/Umbraco.Examine/ExamineExtensions.cs

@@ -0,0 +1,72 @@
+using System;
+using Examine.LuceneEngine.Providers;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Extension methods for the LuceneIndex
+    /// </summary>
+    internal static class ExamineExtensions
+    {
+        /// <summary>
+        /// Checks if the index can be read/opened
+        /// </summary>
+        /// <param name="indexer"></param>
+        /// <param name="ex">The exception returned if there was an error</param>
+        /// <returns></returns>
+        public static bool IsHealthy(this LuceneIndex indexer, out Exception ex)
+        {
+            try
+            {
+                using (indexer.GetIndexWriter().GetReader())
+                {
+                    ex = null;
+                    return true;
+                }
+            }
+            catch (Exception e)
+            {
+                ex = e;
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Return the number of indexed documents in Lucene
+        /// </summary>
+        /// <param name="indexer"></param>
+        /// <returns></returns>
+        public static int GetIndexDocumentCount(this LuceneIndex indexer)
+        {
+            if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher))
+                return 0;
+
+            using (searcher)
+            using (var reader = searcher.IndexReader)
+            {
+                return reader.NumDocs();
+            }
+        }
+
+        /// <summary>
+        /// Return the total number of fields in the index
+        /// </summary>
+        /// <param name="indexer"></param>
+        /// <returns></returns>
+        public static int GetIndexFieldCount(this LuceneIndex indexer)
+        {
+            if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher))
+                return 0;
+
+            using (searcher)
+            using (var reader = searcher.IndexReader)
+            {
+                return reader.GetFieldNames(IndexReader.FieldOption.ALL).Count;
+            }
+        }
+
+    }
+}

src/Umbraco.Examine/IContentValueSetBuilder.cs

@@ -0,0 +1,13 @@
+using Examine;
+using Umbraco.Core.Models;
+
+namespace Umbraco.Examine
+{
+    /// <inheritdoc />
+    /// <summary>
+    /// Marker interface for a <see cref="T:Examine.ValueSet" /> builder for supporting unpublished content
+    /// </summary>
+    public interface IContentValueSetBuilder : IValueSetBuilder<IContent>
+    {
+    }
+}

src/Umbraco.Examine/IContentValueSetValidator.cs

@@ -0,0 +1,31 @@
+using Examine;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// An extended <see cref="IValueSetValidator"/> for content indexes
+    /// </summary>
+    public interface IContentValueSetValidator : IValueSetValidator
+    {
+        /// <summary>
+        /// When set to true the index will only retain published values
+        /// </summary>
+        /// <remarks>
+        /// Any non-published values will not be put or kept in the index:
+        /// * Deleted, Trashed, non-published Content items
+        /// * non-published Variants
+        /// </remarks>
+        bool PublishedValuesOnly { get; }
+
+        /// <summary>
+        /// If true, protected content will be indexed otherwise it will not be put or kept in the index
+        /// </summary>
+        bool SupportProtectedContent { get; }
+
+        int? ParentId { get; }
+
+        bool ValidatePath(string path, string category);
+        bool ValidateRecycleBin(string path, string category);
+        bool ValidateProtectedContent(string path, string category);
+    }
+}

src/Umbraco.Examine/IIndexDiagnostics.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using Examine;
+using Umbraco.Core;
+
+namespace Umbraco.Examine
+{
+    
+
+    /// <summary>
+    /// Exposes diagnostic information about an index
+    /// </summary>
+    public interface IIndexDiagnostics
+    {
+        /// <summary>
+        /// The number of documents in the index
+        /// </summary>
+        int DocumentCount { get; }
+
+        /// <summary>
+        /// The number of fields in the index
+        /// </summary>
+        int FieldCount { get; }
+
+        /// <summary>
+        /// If the index can be open/read
+        /// </summary>
+        /// <returns>
+        /// A successful attempt if it is healthy, else a failed attempt with a message if unhealthy
+        /// </returns>
+        Attempt<string> IsHealthy();
+
+        /// <summary>
+        /// A key/value collection of diagnostic properties for the index
+        /// </summary>
+        /// <remarks>
+        /// Used to display in the UI
+        /// </remarks>
+        IReadOnlyDictionary<string, object> Metadata { get; }
+    }
+}

src/Umbraco.Examine/IIndexPopulator.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using Examine;
+
+namespace Umbraco.Examine
+{
+    
+
+    public interface IIndexPopulator
+    {
+        bool IsRegistered(string indexName);
+        void RegisterIndex(string indexName);
+
+        /// <summary>
+        /// Populate indexers 
+        /// </summary>
+        /// <param name="indexes"></param>
+        void Populate(params IIndex[] indexes);
+    }
+    
+}

src/Umbraco.Examine/IPublishedContentValueSetBuilder.cs

@@ -0,0 +1,12 @@
+using Examine;
+using Umbraco.Core.Models;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Marker interface for a <see cref="ValueSet"/> builder for only published content
+    /// </summary>
+    public interface IPublishedContentValueSetBuilder : IValueSetBuilder<IContent>
+    {
+    }
+}

src/Umbraco.Examine/IUmbracoIndexer.cs

@@ -0,0 +1,25 @@
+using Examine;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// A Marker interface for defining an Umbraco indexer
+    /// </summary>
+    public interface IUmbracoIndexer : IIndex
+    {
+        /// <summary>
+        /// When set to true Umbraco will keep the index in sync with Umbraco data automatically
+        /// </summary>
+        bool EnableDefaultEventHandler { get; }
+
+        /// <summary>
+        /// When set to true the index will only retain published values
+        /// </summary>
+        /// <remarks>
+        /// Any non-published values will not be put or kept in the index:
+        /// * Deleted, Trashed, non-published Content items
+        /// * non-published Variants
+        /// </remarks>
+        bool PublishedValuesOnly { get; }
+    }
+}

src/Umbraco.Examine/IValueSetBuilder.cs

@@ -0,0 +1,22 @@
+using Examine;
+using System.Collections.Generic;
+using Umbraco.Core.Models;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Creates a collection of <see cref="ValueSet"/> to be indexed based on a collection of <see cref="TContent"/>
+    /// </summary>
+    /// <typeparam name="TContent"></typeparam>
+    public interface IValueSetBuilder<in TContent>
+        where TContent : IContentBase
+    {
+        /// <summary>
+        /// Creates a collection of <see cref="ValueSet"/> to be indexed based on a collection of <see cref="TContent"/>
+        /// </summary>
+        /// <param name="content"></param>
+        /// <returns></returns>
+        IEnumerable<ValueSet> GetValueSets(params TContent[] content);
+    }
+
+}

src/Umbraco.Examine/IndexPopulator.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Umbraco.Core.Collections;
+
+namespace Umbraco.Examine
+{
+    public abstract class IndexPopulator : IIndexPopulator
+    {
+        private readonly ConcurrentHashSet<string> _registeredIndexes = new ConcurrentHashSet<string>();
+
+        public bool IsRegistered(string indexName)
+        {
+            return _registeredIndexes.Contains(indexName);
+        }
+
+        /// <summary>
+        /// Registers an index for this populator
+        /// </summary>
+        /// <param name="indexName"></param>
+        public void RegisterIndex(string indexName)
+        {
+            _registeredIndexes.Add(indexName);
+        }
+
+        public void Populate(params IIndex[] indexes)
+        {
+            PopulateIndexes(indexes.Where(x => IsRegistered(x.Name)));
+        }
+
+        protected abstract void PopulateIndexes(IEnumerable<IIndex> indexes);
+    }
+}

src/Umbraco.Examine/IndexRebuilder.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Examine;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Utility to rebuild all indexes ensuring minimal data queries
+    /// </summary>
+    public class IndexRebuilder
+    {
+        private readonly IEnumerable<IIndexPopulator> _populators;
+        public IExamineManager ExamineManager { get; }
+
+        public IndexRebuilder(IExamineManager examineManager, IEnumerable<IIndexPopulator> populators)
+        {
+            _populators = populators;
+            ExamineManager = examineManager;
+        }
+
+        public bool CanRebuild(string indexName)
+        {
+            return _populators.Any(x => x.IsRegistered(indexName));
+        }
+
+        public void RebuildIndex(string indexName)
+        {
+            if (!ExamineManager.TryGetIndex(indexName, out var index))
+                throw new InvalidOperationException($"No index found with name {indexName}");
+            index.CreateIndex(); // clear the index
+            foreach (var populator in _populators)
+            {
+                populator.Populate(index);
+            }
+        }
+
+        public void RebuildIndexes(bool onlyEmptyIndexes)
+        {
+            var indexes = (onlyEmptyIndexes
+                ? ExamineManager.Indexes.Where(x => !x.IndexExists())
+                : ExamineManager.Indexes).ToArray();
+
+            foreach (var index in indexes)
+            {
+                index.CreateIndex(); // clear the index
+            }
+
+            //run the populators in parallel against all indexes
+            Parallel.ForEach(_populators, populator => populator.Populate(indexes));
+        }
+
+    }
+}

src/Umbraco.Examine/IndexTypes.cs

@@ -1,10 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using Lucene.Net.Store;
-
-namespace Umbraco.Examine
+namespace Umbraco.Examine
 {
     /// <summary>
     /// The index types stored in the Lucene Index

src/Umbraco.Examine/MediaIndexPopulator.cs

@@ -0,0 +1,75 @@
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.Services;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Performs the data lookups required to rebuild a media index
+    /// </summary>
+    public class MediaIndexPopulator : IndexPopulator
+    {
+        private readonly int? _parentId;
+        private readonly IMediaService _mediaService;
+        private readonly IValueSetBuilder<IMedia> _mediaValueSetBuilder;
+
+        /// <summary>
+        /// Default constructor to lookup all content data
+        /// </summary>
+        /// <param name="mediaService"></param>
+        /// <param name="mediaValueSetBuilder"></param>
+        public MediaIndexPopulator(IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
+            : this(null, mediaService, mediaValueSetBuilder)
+        {
+        }
+
+        /// <summary>
+        /// Optional constructor allowing specifying custom query parameters
+        /// </summary>
+        /// <param name="parentId"></param>
+        /// <param name="mediaService"></param>
+        /// <param name="mediaValueSetBuilder"></param>
+        public MediaIndexPopulator(int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
+        {
+            _parentId = parentId;
+            _mediaService = mediaService;
+            _mediaValueSetBuilder = mediaValueSetBuilder;
+
+            RegisterIndex(Constants.UmbracoIndexes.InternalIndexName);
+            RegisterIndex(Constants.UmbracoIndexes.ExternalIndexName);
+        }
+
+        protected override void PopulateIndexes(IEnumerable<IIndex> indexes)
+        {
+            const int pageSize = 10000;
+            var pageIndex = 0;
+
+            var mediaParentId = -1;
+
+            if (_parentId.HasValue && _parentId.Value > 0)
+            {
+                mediaParentId = _parentId.Value;
+            }
+
+            IMedia[] media;
+
+            do
+            {
+                media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out var total).ToArray();
+
+                if (media.Length > 0)
+                {
+                    // ReSharper disable once PossibleMultipleEnumeration
+                    foreach (var index in indexes)
+                        index.IndexItems(_mediaValueSetBuilder.GetValueSets(media));
+                }
+
+                pageIndex++;
+            } while (media.Length == pageSize);
+        }
+        
+    }
+}

src/Umbraco.Examine/MediaValueSetBuilder.cs

@@ -0,0 +1,61 @@
+using Examine;
+using System.Collections.Generic;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.PropertyEditors;
+using Umbraco.Core.Services;
+using Umbraco.Core.Strings;
+
+namespace Umbraco.Examine
+{
+    public class MediaValueSetBuilder : BaseValueSetBuilder<IMedia>
+    {
+        private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
+        private readonly IUserService _userService;
+
+        public MediaValueSetBuilder(PropertyEditorCollection propertyEditors,
+            IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
+            IUserService userService)
+            : base(propertyEditors, false)
+        {
+            _urlSegmentProviders = urlSegmentProviders;
+            _userService = userService;
+        }
+
+        /// <inheritdoc />
+        public override IEnumerable<ValueSet> GetValueSets(params IMedia[] media)
+        {
+            foreach (var m in media)
+            {
+                var urlValue = m.GetUrlSegment(_urlSegmentProviders);
+                var values = new Dictionary<string, IEnumerable<object>>
+                {
+                    {"icon", m.ContentType.Icon.Yield()},
+                    {"id", new object[] {m.Id}},
+                    {"key", new object[] {m.Key}},
+                    {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
+                    {"level", new object[] {m.Level}},
+                    {"creatorID", new object[] {m.CreatorId}},
+                    {"sortOrder", new object[] {m.SortOrder}},
+                    {"createDate", new object[] {m.CreateDate}},
+                    {"updateDate", new object[] {m.UpdateDate}},
+                    {"nodeName", m.Name.Yield()},
+                    {"urlName", urlValue.Yield()},
+                    {"path", m.Path.Yield()},
+                    {"nodeType", new object[] {m.ContentType.Id}},
+                    {"creatorName", m.GetCreatorProfile(_userService).Name.Yield()}
+                };
+
+                foreach (var property in m.Properties)
+                {
+                    AddPropertyValue(property, null, null, values);
+                }
+
+                var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);
+
+                yield return vs;
+            }
+        }
+    }
+
+}

src/Umbraco.Examine/MemberIndexPopulator.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Lucene.Net.Util;
+using Umbraco.Core.Models;
+using Umbraco.Core.Services;
+
+namespace Umbraco.Examine
+{
+    public class MemberIndexPopulator : IndexPopulator
+    {
+        private readonly IMemberService _memberService;
+        private readonly IValueSetBuilder<IMember> _valueSetBuilder;
+
+        public MemberIndexPopulator(IMemberService memberService, IValueSetBuilder<IMember> valueSetBuilder)
+        {
+            _memberService = memberService;
+            _valueSetBuilder = valueSetBuilder;
+
+            RegisterIndex(Core.Constants.UmbracoIndexes.MembersIndexName);
+        }
+        protected override void PopulateIndexes(IEnumerable<IIndex> indexes)
+        {
+            const int pageSize = 1000;
+            var pageIndex = 0;
+
+            IMember[] members;
+
+            //no node types specified, do all members
+            do
+            {
+                members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
+
+                if (members.Length > 0)
+                {
+                    // ReSharper disable once PossibleMultipleEnumeration
+                    foreach (var index in indexes)
+                        index.IndexItems(_valueSetBuilder.GetValueSets(members));
+                }
+
+                pageIndex++;
+            } while (members.Length == pageSize);
+        }
+    }
+}

src/Umbraco.Examine/MemberValueSetBuilder.cs

@@ -0,0 +1,52 @@
+using Examine;
+using System.Collections.Generic;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using Umbraco.Core.PropertyEditors;
+
+namespace Umbraco.Examine
+{
+
+    public class MemberValueSetBuilder : BaseValueSetBuilder<IMember>
+    {
+        public MemberValueSetBuilder(PropertyEditorCollection propertyEditors)
+            : base(propertyEditors, false)
+        {
+        }
+
+        /// <inheritdoc />
+        public override IEnumerable<ValueSet> GetValueSets(params IMember[] members)
+        {
+            foreach (var m in members)
+            {
+                var values = new Dictionary<string, IEnumerable<object>>
+                {
+                    {"icon", m.ContentType.Icon.Yield()},
+                    {"id", new object[] {m.Id}},
+                    {"key", new object[] {m.Key}},
+                    {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
+                    {"level", new object[] {m.Level}},
+                    {"creatorID", new object[] {m.CreatorId}},
+                    {"sortOrder", new object[] {m.SortOrder}},
+                    {"createDate", new object[] {m.CreateDate}},
+                    {"updateDate", new object[] {m.UpdateDate}},
+                    {"nodeName", m.Name.Yield()},
+                    {"path", m.Path.Yield()},
+                    {"nodeType", new object[] {m.ContentType.Id}},
+                    {"loginName", m.Username.Yield()},
+                    {"email", m.Email.Yield()},
+                };
+
+                foreach (var property in m.Properties)
+                {
+                    AddPropertyValue(property, null, null, values);
+                }
+
+                var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Member, m.ContentType.Alias, values);
+
+                yield return vs;
+            }
+        }
+    }
+
+}

src/Umbraco.Examine/MemberValueSetValidator.cs

@@ -0,0 +1,32 @@
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+
+namespace Umbraco.Examine
+{
+    public class MemberValueSetValidator : ValueSetValidator
+    {
+        public MemberValueSetValidator() : base(null, null, DefaultMemberIndexFields, null)
+        {
+        }
+
+        public MemberValueSetValidator(IEnumerable<string> includeItemTypes, IEnumerable<string> excludeItemTypes)
+            : base(includeItemTypes, excludeItemTypes, DefaultMemberIndexFields, null)
+        {
+        }
+
+        public MemberValueSetValidator(IEnumerable<string> includeItemTypes, IEnumerable<string> excludeItemTypes, IEnumerable<string> includeFields, IEnumerable<string> excludeFields)
+            : base(includeItemTypes, excludeItemTypes, includeFields, excludeFields)
+        {
+        }
+
+        /// <summary>
+        /// By default these are the member fields we index
+        /// </summary>
+        public static readonly string[] DefaultMemberIndexFields = { "id", "nodeName", "updateDate", "loginName", "email" };
+
+        private static readonly IEnumerable<string> ValidCategories = new[] { IndexTypes.Member };
+        protected override IEnumerable<string> ValidIndexCategories => ValidCategories;
+        
+    }
+}

src/Umbraco.Examine/PublishedContentIndexPopulator.cs

@@ -0,0 +1,22 @@
+using Umbraco.Core.Models;
+using Umbraco.Core.Services;
+using Umbraco.Core.Persistence;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Performs the data lookups required to rebuild a content index containing only published content
+    /// </summary>
+    /// <remarks>
+    /// The published (external) index will still rebuild just fine using the default <see cref="ContentIndexPopulator"/> which is what
+    /// is used when rebuilding all indexes, but this will be used when the single index is rebuilt and will go a little bit faster
+    /// since the data query is more specific.
+    /// </remarks>
+    public class PublishedContentIndexPopulator : ContentIndexPopulator
+    {
+        public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IPublishedContentValueSetBuilder contentValueSetBuilder) :
+            base(true, null, contentService, sqlContext, contentValueSetBuilder)
+        {   
+        }
+    }
+}

src/Umbraco.Examine/Umbraco.Examine.csproj

@@ -48,30 +48,51 @@
   </ItemGroup>
   <ItemGroup>
     <!-- note: NuGet deals with transitive references now -->
-    <PackageReference Include="Examine" Version="1.0.0-beta025" />
+    <PackageReference Include="Examine" Version="1.0.0-beta046" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
     <PackageReference Include="NPoco" Version="3.9.4" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="BaseValueSetBuilder.cs" />
     <Compile Include="Config\ConfigIndexCriteria.cs" />
     <Compile Include="Config\ConfigIndexField.cs" />
+    <Compile Include="Config\IndexFieldCollection.cs" />
     <Compile Include="Config\IndexFieldCollectionExtensions.cs" />
     <Compile Include="Config\IndexSet.cs" />
     <Compile Include="Config\IndexSetCollection.cs" />
     <Compile Include="Config\IndexSets.cs" />
+    <Compile Include="ContentIndexPopulator.cs" />
+    <Compile Include="ContentValueSetBuilder.cs" />
+    <Compile Include="ExamineExtensions.cs" />
+    <Compile Include="IContentValueSetBuilder.cs" />
+    <Compile Include="IContentValueSetValidator.cs" />
+    <Compile Include="IIndexDiagnostics.cs" />
+    <Compile Include="IIndexPopulator.cs" />
+    <Compile Include="IndexPopulator.cs" />
+    <Compile Include="IndexRebuilder.cs" />
+    <Compile Include="IPublishedContentValueSetBuilder.cs" />
+    <Compile Include="IUmbracoIndexer.cs" />
+    <Compile Include="IValueSetBuilder.cs" />
+    <Compile Include="MediaIndexPopulator.cs" />
+    <Compile Include="MediaValueSetBuilder.cs" />
+    <Compile Include="MemberIndexPopulator.cs" />
+    <Compile Include="MemberValueSetBuilder.cs" />
+    <Compile Include="MemberValueSetValidator.cs" />
+    <Compile Include="PublishedContentIndexPopulator.cs" />
     <Compile Include="UmbracoExamineExtensions.cs" />
     <Compile Include="IndexTypes.cs" />
     <Compile Include="NoPrefixSimpleFsLockFactory.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="UmbracoContentIndexer.cs" />
-    <Compile Include="UmbracoContentIndexerOptions.cs" />
-    <Compile Include="UmbracoContentValueSetValidator.cs" />
+    <Compile Include="ContentValueSetValidator.cs" />
+    <Compile Include="UmbracoExamineIndexDiagnostics.cs" />
     <Compile Include="UmbracoExamineIndexer.cs" />
     <Compile Include="UmbracoExamineSearcher.cs" />
     <Compile Include="UmbracoMemberIndexer.cs" />
     <Compile Include="..\SolutionInfo.cs">
       <Link>Properties\SolutionInfo.cs</Link>
     </Compile>
+    <Compile Include="ValueSetValidator.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Umbraco.Core\Umbraco.Core.csproj">

src/Umbraco.Examine/UmbracoContentIndexer.cs

@@ -5,7 +5,6 @@ using System.ComponentModel;
 using System.Linq;
 using Examine;
 using Umbraco.Core;
-using Umbraco.Core.Models;
 using Umbraco.Core.Services;
 using Umbraco.Core.Strings;
 using Examine.LuceneEngine.Indexing;
@@ -14,14 +13,8 @@ using Lucene.Net.Analysis;
 using Lucene.Net.Store;
 using Umbraco.Core.Composing;
 using Umbraco.Core.Logging;
-using Umbraco.Core.Persistence;
-using Umbraco.Core.Persistence.DatabaseModelDefinitions;
-using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Scoping;
 using Umbraco.Examine.Config;
-using IContentService = Umbraco.Core.Services.IContentService;
-using IMediaService = Umbraco.Core.Services.IMediaService;
-
+using Examine.LuceneEngine;
 
 namespace Umbraco.Examine
 {
@@ -30,12 +23,8 @@ namespace Umbraco.Examine
     /// </summary>
     public class UmbracoContentIndexer : UmbracoExamineIndexer
     {
-        protected IContentService ContentService { get; }
-        protected IMediaService MediaService { get; }
-        protected IUserService UserService { get; }
-
-        private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
-        private int? _parentId;
+        public const string VariesByCultureFieldName = SpecialFieldPrefix + "VariesByCulture";
+        protected ILocalizationService LanguageService { get; }
 
         #region Constructors
 
@@ -45,13 +34,9 @@ namespace Umbraco.Examine
         [EditorBrowsable(EditorBrowsableState.Never)]
         public UmbracoContentIndexer()
         {
-            ContentService = Current.Services.ContentService;
-            MediaService = Current.Services.MediaService;
-            UserService = Current.Services.UserService;
+            LanguageService = Current.Services.LocalizationService;
 
-            _urlSegmentProviders = Current.UrlSegmentProviders;
-
-            InitializeQueries(Current.SqlContext);
+            //note: The validator for this config based indexer is set in the Initialize method
         }
 
         /// <summary>
@@ -62,13 +47,8 @@ namespace Umbraco.Examine
         /// <param name="luceneDirectory"></param>
         /// <param name="defaultAnalyzer"></param>
         /// <param name="profilingLogger"></param>
-        /// <param name="contentService"></param>
-        /// <param name="mediaService"></param>
-        /// <param name="userService"></param>
-        /// <param name="sqlContext"></param>
-        /// <param name="urlSegmentProviders"></param>
+        /// <param name="languageService"></param>
         /// <param name="validator"></param>
-        /// <param name="options"></param>
         /// <param name="indexValueTypes"></param>
         public UmbracoContentIndexer(
             string name,
@@ -76,36 +56,16 @@ namespace Umbraco.Examine
             Directory luceneDirectory,
             Analyzer defaultAnalyzer,
             ProfilingLogger profilingLogger,
-            IContentService contentService,
-            IMediaService mediaService,
-            IUserService userService,
-            ISqlContext sqlContext,
-            IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
-            IValueSetValidator validator,
-            UmbracoContentIndexerOptions options,
+            ILocalizationService languageService,
+            IContentValueSetValidator validator,
             IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
             : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes)
         {
             if (validator == null) throw new ArgumentNullException(nameof(validator));
-            if (options == null) throw new ArgumentNullException(nameof(options));
+            LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService));
 
-            SupportProtectedContent = options.SupportProtectedContent;
-            SupportUnpublishedContent = options.SupportUnpublishedContent;
-            ParentId = options.ParentId;
-
-            ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
-            MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
-            UserService = userService ?? throw new ArgumentNullException(nameof(userService));
-            _urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders));
-
-            InitializeQueries(sqlContext);
-        }
-
-        private void InitializeQueries(ISqlContext sqlContext)
-        {
-            if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
-            if (_publishedQuery == null)
-                _publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
+            if (validator is IContentValueSetValidator contentValueSetValidator)
+                PublishedValuesOnly = contentValueSetValidator.PublishedValuesOnly;
         }
 
         #endregion
@@ -129,25 +89,24 @@ namespace Umbraco.Examine
         /// <exception cref="T:System.InvalidOperationException">
         /// An attempt is made to call <see cref="M:System.Configuration.Provider.ProviderBase.Initialize(System.String,System.Collections.Specialized.NameValueCollection)"/> on a provider after the provider has already been initialized.
         /// </exception>
-
+        [EditorBrowsable(EditorBrowsableState.Never)]
         public override void Initialize(string name, NameValueCollection config)
         {
+            base.Initialize(name, config);
+
+            var supportUnpublished = false;
+            var supportProtected = false;
+
             //check if there's a flag specifying to support unpublished content,
             //if not, set to false;
-            if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished))
-                SupportUnpublishedContent = supportUnpublished;
-            else
-                SupportUnpublishedContent = false;
-
+            if (config["supportUnpublished"] != null)
+                bool.TryParse(config["supportUnpublished"], out supportUnpublished);
 
             //check if there's a flag specifying to support protected content,
             //if not, set to false;
-            if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected))
-                SupportProtectedContent = supportProtected;
-            else
-                SupportProtectedContent = false;
+            if (config["supportProtected"] != null)
+                bool.TryParse(config["supportProtected"], out supportProtected);
 
-            base.Initialize(name, config);
 
             //now we need to build up the indexer options so we can create our validator
             int? parentId = null;
@@ -156,44 +115,76 @@ namespace Umbraco.Examine
                 var indexSet = IndexSets.Instance.Sets[IndexSetName];
                 parentId = indexSet.IndexParentId;
             }
-            ValueSetValidator = new UmbracoContentValueSetValidator(
-                new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId),
+
+            ValueSetValidator = new ContentValueSetValidator(
+                supportUnpublished, supportProtected,
                 //Using a singleton here, we can't inject this when using config based providers and we don't use this
                 //anywhere else in this class
-                Current.Services.PublicAccessService);
+                Current.Services.PublicAccessService,
+                parentId,
+                ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes);
+
+            PublishedValuesOnly = supportUnpublished;
         }
 
         #endregion
 
-        #region Properties
-
         /// <summary>
-        /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected.
-        /// This property is ignored if SupportUnpublishedContent is set to true.
+        /// Special check for invalid paths
         /// </summary>
-        public bool SupportProtectedContent { get; protected set; }
-
-        /// <summary>
-        /// Determines if the manager will call the indexing methods when content is saved or deleted as
-        /// opposed to cache being updated.
-        /// </summary>
-        public bool SupportUnpublishedContent { get; protected set; }
-
-        /// <summary>
-        /// If set this will filter the content items allowed to be indexed
-        /// </summary>
-        public int? ParentId
+        /// <param name="values"></param>
+        /// <param name="onComplete"></param>
+        protected override void PerformIndexItems(IEnumerable<ValueSet> values, Action<IndexOperationEventArgs> onComplete)
         {
-            get => _parentId ?? ConfigIndexCriteria?.ParentNodeId;
-            protected set => _parentId = value;
+            //We don't want to re-enumerate this list, but we need to split it into 2x enumerables: invalid and valid items.
+            // The Invalid items will be deleted, these are items that have invalid paths (i.e. moved to the recycle bin, etc...)
+            // Then we'll index the Value group all together.
+            // We return 0 or 1 here so we can order the results and do the invalid first and then the valid.
+            var invalidOrValid = values.GroupBy(v =>
+            {
+                if (!v.Values.TryGetValue("path", out var paths) || paths.Count <= 0 || paths[0] == null)
+                    return 0;
+
+                //we know this is an IContentValueSetValidator
+                var validator = (IContentValueSetValidator)ValueSetValidator;
+                var path = paths[0].ToString();
+
+                return (!validator.ValidatePath(path, v.Category)
+                        || !validator.ValidateRecycleBin(path, v.Category)
+                        || !validator.ValidateProtectedContent(path, v.Category))
+                    ? 0
+                    : 1;
+            });
+
+            var hasDeletes = false;
+            var hasUpdates = false;
+            foreach (var group in invalidOrValid.OrderBy(x => x.Key))
+            {
+                if (group.Key == 0)
+                {
+                    hasDeletes = true;
+                    //these are the invalid items so we'll delete them
+                    //since the path is not valid we need to delete this item in case it exists in the index already and has now
+                    //been moved to an invalid parent.
+                    foreach (var i in group)
+                        QueueIndexOperation(new IndexOperation(new ValueSet(i.Id), IndexOperationType.Delete));
+                }
+                else
+                {
+                    hasUpdates = true;
+                    //these are the valid ones, so just index them all at once
+                    base.PerformIndexItems(group, onComplete);
+                }
+            }
+
+            if (hasDeletes && !hasUpdates || !hasDeletes && !hasUpdates)
+            {
+                //we need to manually call the completed method
+                onComplete(new IndexOperationEventArgs(this, 0));
+            }
         }
 
-        protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media};
-
-        #endregion
-
-        #region Public methods
-
+        /// <inheritdoc />
         /// <summary>
         /// Deletes a node from the index.
         /// </summary>
@@ -202,7 +193,8 @@ namespace Umbraco.Examine
         /// custom Lucene search to find all decendents and create Delete item queues for them too.
         /// </remarks>
         /// <param name="nodeId">ID of the node to delete</param>
-        public override void DeleteFromIndex(string nodeId)
+        /// <param name="onComplete"></param>
+        protected override void PerformDeleteFromIndex(string nodeId, Action<IndexOperationEventArgs> onComplete)
         {
             //find all descendants based on path
             var descendantPath = $@"\-1\,*{nodeId}\,*";
@@ -217,204 +209,32 @@ namespace Umbraco.Examine
             //need to queue a delete item for each one found
             foreach (var r in results)
             {
-                QueueIndexOperation(new IndexOperation(IndexItem.ForId(r.Id),  IndexOperationType.Delete));
+                QueueIndexOperation(new IndexOperation(new ValueSet(r.Id), IndexOperationType.Delete));
             }
 
-            base.DeleteFromIndex(nodeId);
+            base.PerformDeleteFromIndex(nodeId, onComplete);
         }
-        #endregion
-
-        #region Protected
 
         /// <summary>
-        /// This is a static query, it's parameters don't change so store statically
+        /// Overridden to ensure that the variant system fields have the right value types
         /// </summary>
-        private static IQuery<IContent> _publishedQuery;
-
-        protected override void PerformIndexAll(string type)
+        /// <param name="x"></param>
+        /// <param name="indexValueTypesFactory"></param>
+        /// <returns></returns>
+        protected override FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
         {
-            const int pageSize = 10000;
-            var pageIndex = 0;
-
-            switch (type)
+            //fixme: languages are dynamic so although this will work on startup it wont work when languages are edited
+            foreach(var lang in LanguageService.GetAllLanguages())
             {
-                case IndexTypes.Content:
-                    var contentParentId = -1;
-                    if (ParentId.HasValue && ParentId.Value > 0)
-                    {
-                        contentParentId = ParentId.Value;
-                    }
-                    IContent[] content;
-
-                    do
-                    {
-                        long total;
-
-                        IEnumerable<IContent> descendants;
-                        if (SupportUnpublishedContent)
-                        {
-                            descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
-                        }
-                        else
-                        {
-                            //add the published filter
-                            descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
-                                _publishedQuery, Ordering.By("Path", Direction.Ascending));
-                        }
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
-                        {
-                            content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
-                        }
-                        else
-                        {
-                            content = descendants.ToArray();
-                        }
-
-                        IndexItems(GetValueSets(_urlSegmentProviders, UserService, content));
-
-                        pageIndex++;
-                    } while (content.Length == pageSize);
-
-                    break;
-                case IndexTypes.Media:
-                    var mediaParentId = -1;
-
-                    if (ParentId.HasValue && ParentId.Value > 0)
-                    {
-                        mediaParentId = ParentId.Value;
-                    }
-
-                    // merge note: 7.5 changes this to use mediaService.GetPagedXmlEntries but we cannot merge the
-                    // change here as mediaService does not provide access to Xml in v8 - and actually Examine should
-                    // not assume that Umbraco provides Xml at all.
-
-                    IMedia[] media;
-
-                    do
-                    {
-                        var descendants = MediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out _);
-
-                        //if specific types are declared we need to post filter them
-                        //TODO: Update the service layer to join the cmsContentType table so we can query by content type too
-                        if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
-                        {
-                            media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
-                        }
-                        else
-                        {
-                            media = descendants.ToArray();
-                        }
-
-                        IndexItems(GetValueSets(_urlSegmentProviders, UserService, media));
-
-                        pageIndex++;
-                    } while (media.Length == pageSize);
-
-                    break;
-            }
-        }
-
-        public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IContent[] content)
-        {
-            foreach (var c in content)
-            {
-                var urlValue = c.GetUrlSegment(urlSegmentProviders); // for now, index with invariant culture
-                var values = new Dictionary<string, object[]>
+                foreach (var field in UmbracoIndexFieldDefinitions)
                 {
-                    {"icon", new object[] {c.ContentType.Icon}},
-                    {PublishedFieldName, new object[] {c.Published ? 1 : 0}},
-                    {"id", new object[] {c.Id}},
-                    {"key", new object[] {c.Key}},
-                    {"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}},
-                    {"level", new object[] {c.Level}},
-                    {"creatorID", new object[] {c.CreatorId}},
-                    {"sortOrder", new object[] {c.SortOrder}},
-                    {"createDate", new object[] {c.CreateDate}},
-                    {"updateDate", new object[] {c.UpdateDate}},
-                    {"nodeName", new object[] {c.Name}},
-                    {"urlName", new object[] {urlValue}},
-                    {"path", new object[] {c.Path}},
-                    {"nodeType", new object[] {c.ContentType.Id}},
-                    {"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}},
-                    {"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}},
-                    {"writerID", new object[] {c.WriterId}},
-                    {"template", new object[] {c.Template?.Id ?? 0}}
-                };
-
-                foreach (var property in c.Properties)
-                {
-                    //only add the value if its not null or empty (we'll check for string explicitly here too)
-                    //fixme support variants with language id
-                    var val = property.GetValue("", ""); // for now, index the invariant values
-                    switch (val)
-                    {
-                        case null:
-                            continue;
-                        case string strVal when strVal.IsNullOrWhiteSpace() == false:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                        default:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                    }
+                    var def = new FieldDefinition($"{field.Name}_{lang.IsoCode.ToLowerInvariant()}", field.Type);
+                    FieldDefinitionCollection.TryAdd(def.Name, def);
                 }
-
-                var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
-
-                yield return vs;
             }
+
+            return base.CreateFieldValueTypes(indexValueTypesFactory);
         }
-
-        public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IMedia[] media)
-        {
-            foreach (var m in media)
-            {
-                var urlValue = m.GetUrlSegment(urlSegmentProviders);
-                var values = new Dictionary<string, object[]>
-                {
-                    {"icon", new object[] {m.ContentType.Icon}},
-                    {"id", new object[] {m.Id}},
-                    {"key", new object[] {m.Key}},
-                    {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
-                    {"level", new object[] {m.Level}},
-                    {"creatorID", new object[] {m.CreatorId}},
-                    {"sortOrder", new object[] {m.SortOrder}},
-                    {"createDate", new object[] {m.CreateDate}},
-                    {"updateDate", new object[] {m.UpdateDate}},
-                    {"nodeName", new object[] {m.Name}},
-                    {"urlName", new object[] {urlValue}},
-                    {"path", new object[] {m.Path}},
-                    {"nodeType", new object[] {m.ContentType.Id}},
-                    {"creatorName", new object[] {m.GetCreatorProfile(userService).Name}}
-                };
-
-                foreach (var property in m.Properties)
-                {
-                    //only add the value if its not null or empty (we'll check for string explicitly here too)
-                    var val = property.GetValue();
-                    switch (val)
-                    {
-                        case null:
-                            continue;
-                        case string strVal when strVal.IsNullOrWhiteSpace() == false:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                        default:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                    }
-                }
-
-                var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);
-
-                yield return vs;
-            }
-        }
-
-
-        #endregion
+        
     }
 }

src/Umbraco.Examine/UmbracoContentIndexerOptions.cs

@@ -1,23 +0,0 @@
-using System;
-
-namespace Umbraco.Examine
-{   
-
-    /// <summary>
-    /// Options used to configure the umbraco content indexer
-    /// </summary>
-    public class UmbracoContentIndexerOptions
-    {
-        public bool SupportUnpublishedContent { get; private set; }
-        public bool SupportProtectedContent { get; private set; }
-        //TODO: We should make this a GUID! But to do that we sort of need to store the 'Path' as a comma list of GUIDs instead of int
-        public int? ParentId { get; private set; }
-
-        public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent, int? parentId)
-        {
-            SupportUnpublishedContent = supportUnpublishedContent;
-            SupportProtectedContent = supportProtectedContent;
-            ParentId = parentId;
-        }
-    }
-}

src/Umbraco.Examine/UmbracoContentValueSetValidator.cs

@@ -1,72 +0,0 @@
-using System;
-using System.Linq;
-using Examine;
-using Examine.LuceneEngine.Providers;
-using Umbraco.Core;
-using Umbraco.Core.Services;
-
-namespace Umbraco.Examine
-{
-    /// <summary>
-    /// Used to validate a ValueSet for content - based on permissions, parent id, etc....
-    /// </summary>
-    public class UmbracoContentValueSetValidator : IValueSetValidator
-    {
-        private readonly UmbracoContentIndexerOptions _options;
-        private readonly IPublicAccessService _publicAccessService;
-        private const string PathKey = "path";
-        public UmbracoContentValueSetValidator(UmbracoContentIndexerOptions options, IPublicAccessService publicAccessService)
-        {
-            _options = options;
-            _publicAccessService = publicAccessService;
-        }
-
-        public bool Validate(ValueSet valueSet)
-        {
-            //check for published content
-            if (valueSet.Category == IndexTypes.Content
-                && valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName))
-            {
-                var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1);
-                //we don't support unpublished and the item is not published return false
-                if (_options.SupportUnpublishedContent == false && published == false)
-                {
-                    return false;
-                }
-            }
-
-            //must have a 'path'
-            if (valueSet.Values.ContainsKey(PathKey) == false) return false;
-            var path = valueSet.Values[PathKey]?[0].ToString() ?? string.Empty;
-
-            // Test for access if we're only indexing published content
-            // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content
-            if (valueSet.Category == IndexTypes.Content
-                && _options.SupportUnpublishedContent == false
-                && _options.SupportProtectedContent == false
-                && _publicAccessService.IsProtected(path))
-            {
-                return false;
-            }
-
-            //check if this document is a descendent of the parent
-            if (_options.ParentId.HasValue && _options.ParentId.Value > 0)
-            {
-                if (path.IsNullOrWhiteSpace()) return false;
-                if (path.Contains(string.Concat(",", _options.ParentId.Value, ",")) == false)
-                    return false;
-            }
-
-            //check for recycle bin
-            if (_options.SupportUnpublishedContent == false)
-            {
-                if (path.IsNullOrWhiteSpace()) return false;
-                var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
-                if (path.Contains(string.Concat(",", recycleBinId, ",")))
-                    return false;
-            }
-
-            return true;
-        }
-    }
-}

src/Umbraco.Examine/UmbracoExamineExtensions.cs

@@ -76,17 +76,6 @@ namespace Umbraco.Examine
             return fieldQuery;
         }
 
-        /// <summary>
-        /// Used to replace any available tokens in the index path before the lucene directory is assigned to the path
-        /// </summary>
-        /// <param name="indexSet"></param>
-        internal static void ReplaceTokensInIndexPath(this IndexSet indexSet)
-        {
-            if (indexSet == null) return;
-            indexSet.IndexPath = indexSet.IndexPath
-                .Replace("{machinename}", NetworkHelper.FileSafeMachineName)
-                .Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars(""))
-                .EnsureEndsWith('/');
-        }
+        
     }
-}
+}

src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs

@@ -0,0 +1,97 @@
+using System.Collections.Generic;
+using System.Linq;
+using Lucene.Net.Store;
+using Umbraco.Core;
+using Umbraco.Core.IO;
+using Umbraco.Core.Logging;
+
+namespace Umbraco.Examine
+{
+    public class UmbracoExamineIndexDiagnostics : IIndexDiagnostics
+    {
+        private readonly UmbracoExamineIndexer _index;
+        private readonly ILogger _logger;
+
+        public UmbracoExamineIndexDiagnostics(UmbracoExamineIndexer index, ILogger logger)
+        {
+            _index = index;
+            _logger = logger;
+        }
+
+        public int DocumentCount
+        {
+            get
+            {
+                try
+                {
+                    return _index.GetIndexDocumentCount();
+                }
+                catch (AlreadyClosedException)
+                {
+                    _logger.Warn(typeof(UmbracoContentIndexer), "Cannot get GetIndexDocumentCount, the writer is already closed");
+                    return 0;
+                }
+            }
+        }
+
+        public int FieldCount
+        {
+            get
+            {
+                try
+                {
+                    return _index.GetIndexFieldCount();
+                }
+                catch (AlreadyClosedException)
+                {
+                    _logger.Warn(typeof(UmbracoContentIndexer), "Cannot get GetIndexFieldCount, the writer is already closed");
+                    return 0;
+                }
+            }
+        }
+
+        public Attempt<string> IsHealthy()
+        {
+            var isHealthy = _index.IsHealthy(out var indexError);
+            return isHealthy ? Attempt<string>.Succeed() : Attempt.Fail(indexError.Message);
+        }
+
+        public virtual IReadOnlyDictionary<string, object> Metadata
+        {
+            get
+            {
+                var d = new Dictionary<string, object>
+                {
+                    [nameof(UmbracoExamineIndexer.CommitCount)] = _index.CommitCount,
+                    [nameof(UmbracoExamineIndexer.DefaultAnalyzer)] = _index.DefaultAnalyzer.GetType().Name,
+                    [nameof(UmbracoExamineIndexer.DirectoryFactory)] = _index.DirectoryFactory,
+                    [nameof(UmbracoExamineIndexer.EnableDefaultEventHandler)] = _index.EnableDefaultEventHandler,
+                    [nameof(UmbracoExamineIndexer.LuceneIndexFolder)] =
+                        _index.LuceneIndexFolder == null
+                            ? string.Empty
+                            : _index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'),
+                    [nameof(UmbracoExamineIndexer.PublishedValuesOnly)] = _index.PublishedValuesOnly,
+                    //There's too much info here
+                    //[nameof(UmbracoExamineIndexer.FieldDefinitionCollection)] = _index.FieldDefinitionCollection,
+                };
+
+                if (_index.ValueSetValidator is ValueSetValidator vsv)
+                {
+                    d[nameof(ValueSetValidator.IncludeItemTypes)] = vsv.IncludeItemTypes;
+                    d[nameof(ContentValueSetValidator.ExcludeItemTypes)] = vsv.ExcludeItemTypes;
+                    d[nameof(ContentValueSetValidator.IncludeFields)] = vsv.IncludeFields;
+                    d[nameof(ContentValueSetValidator.ExcludeFields)] = vsv.ExcludeFields;
+                }
+
+                if (_index.ValueSetValidator is ContentValueSetValidator cvsv)
+                {
+                    d[nameof(ContentValueSetValidator.PublishedValuesOnly)] = cvsv.PublishedValuesOnly;
+                    d[nameof(ContentValueSetValidator.SupportProtectedContent)] = cvsv.SupportProtectedContent;
+                    d[nameof(ContentValueSetValidator.ParentId)] = cvsv.ParentId;
+                }
+
+                return d.Where(x => x.Value != null).ToDictionary(x => x.Key, x => x.Value);
+            }
+        }
+    }
+}

src/Umbraco.Examine/UmbracoExamineIndexer.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
-using System.IO;
 using System.Linq;
 using Examine.LuceneEngine.Providers;
 using Lucene.Net.Analysis;
@@ -11,19 +10,20 @@ using Umbraco.Core;
 using Examine;
 using Examine.LuceneEngine;
 using Examine.LuceneEngine.Indexing;
+using Lucene.Net.Store;
 using Umbraco.Core.Composing;
 using Umbraco.Core.Logging;
-using Umbraco.Core.Xml;
 using Umbraco.Examine.Config;
 using Directory = Lucene.Net.Store.Directory;
 
 namespace Umbraco.Examine
 {
+
     /// <summary>
     /// An abstract provider containing the basic functionality to be able to query against
     /// Umbraco data.
     /// </summary>
-    public abstract class UmbracoExamineIndexer : LuceneIndexer
+    public abstract class UmbracoExamineIndexer : LuceneIndex, IUmbracoIndexer, IIndexDiagnostics
     {
         // note
         // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call
@@ -34,14 +34,15 @@ namespace Umbraco.Examine
         /// <summary>
         /// Used to store the path of a content object
         /// </summary>
-        public const string IndexPathFieldName = "__Path";
-        public const string NodeKeyFieldName = "__Key";
-        public const string IconFieldName = "__Icon";
-        public const string PublishedFieldName = "__Published";
+        public const string IndexPathFieldName = SpecialFieldPrefix + "Path";
+        public const string NodeKeyFieldName = SpecialFieldPrefix + "Key";
+        public const string IconFieldName = SpecialFieldPrefix + "Icon";
+        public const string PublishedFieldName = SpecialFieldPrefix + "Published";
+
         /// <summary>
         /// The prefix added to a field when it is duplicated in order to store the original raw value.
         /// </summary>
-        public const string RawFieldPrefix = "__Raw_";
+        public const string RawFieldPrefix = SpecialFieldPrefix + "Raw_";
 
         /// <summary>
         /// Constructor for config provider based indexes
@@ -52,25 +53,18 @@ namespace Umbraco.Examine
         {
             ProfilingLogger = Current.ProfilingLogger;
             _configBased = true;
-
-            //This is using the config so we'll validate based on that
-            ValueSetValidator = new ValueSetValidatorDelegate(set =>
-            {
-
-                //check if this document is of a correct type of node type alias
-                if (ConfigIndexCriteria.IncludeItemTypes.Any())
-                    if (!ConfigIndexCriteria.IncludeItemTypes.Contains(set.ItemType))
-                        return false;
-
-                //if this node type is part of our exclusion list, do not validate
-                if (ConfigIndexCriteria.ExcludeItemTypes.Any())
-                    if (ConfigIndexCriteria.ExcludeItemTypes.Contains(set.ItemType))
-                        return false;
-
-                return true;
-            });
         }
 
+        /// <summary>
+        /// Create a new <see cref="UmbracoExamineIndexer"/>
+        /// </summary>
+        /// <param name="name"></param>
+        /// <param name="fieldDefinitions"></param>
+        /// <param name="luceneDirectory"></param>
+        /// <param name="defaultAnalyzer"></param>
+        /// <param name="profilingLogger"></param>
+        /// <param name="validator"></param>
+        /// <param name="indexValueTypes"></param>
         protected UmbracoExamineIndexer(
             string name,
             IEnumerable<FieldDefinition> fieldDefinitions,
@@ -82,6 +76,12 @@ namespace Umbraco.Examine
             : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes)
         {
             ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger));
+
+            //try to set the value of `LuceneIndexFolder` for diagnostic reasons
+            if (luceneDirectory is FSDirectory fsDir)
+                LuceneIndexFolder = fsDir.Directory;
+
+            _diagnostics = new UmbracoExamineIndexDiagnostics(this, ProfilingLogger.Logger);
         }
 
         private readonly bool _configBased = false;
@@ -91,7 +91,7 @@ namespace Umbraco.Examine
         /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene
         /// for retreival after searching.
         /// </summary>
-        internal static readonly FieldDefinition[] UmbracoIndexFields =
+        public static readonly FieldDefinition[] UmbracoIndexFieldDefinitions =
         {
             new FieldDefinition("parentID", FieldDefinitionTypes.Integer),
             new FieldDefinition("level", FieldDefinitionTypes.Integer),
@@ -110,6 +110,10 @@ namespace Umbraco.Examine
             new FieldDefinition("urlName", FieldDefinitionTypes.InvariantCultureIgnoreCase),
             new FieldDefinition("path", FieldDefinitionTypes.Raw),
 
+            new FieldDefinition("email", FieldDefinitionTypes.EmailAddress),
+
+            new FieldDefinition(PublishedFieldName, FieldDefinitionTypes.Raw),
+            new FieldDefinition(NodeKeyFieldName, FieldDefinitionTypes.Raw),
             new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw),
             new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw)
         };
@@ -119,17 +123,21 @@ namespace Umbraco.Examine
         /// <summary>
         /// Overridden to ensure that the umbraco system field definitions are in place
         /// </summary>
-        /// <param name="x"></param>
         /// <param name="indexValueTypesFactory"></param>
         /// <returns></returns>
-        protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
+        protected override FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
         {
-            foreach (var field in UmbracoIndexFields)
+            //if config based then ensure the value types else it's assumed these were passed in via ctor
+            if (_configBased)
             {
-                FieldDefinitionCollection.TryAdd(field.Name, field);
+                foreach (var field in UmbracoIndexFieldDefinitions)
+                {
+                    FieldDefinitionCollection.TryAdd(field.Name, field);
+                }
             }
+            
 
-            return base.CreateFieldValueTypes(x, indexValueTypesFactory);
+            return base.CreateFieldValueTypes(indexValueTypesFactory);
         }
 
         /// <summary>
@@ -137,10 +145,7 @@ namespace Umbraco.Examine
         /// </summary>
         public bool EnableDefaultEventHandler { get; set; } = true;
 
-        /// <summary>
-        /// the supported indexable types
-        /// </summary>
-        protected abstract IEnumerable<string> SupportedTypes { get; }
+        public bool PublishedValuesOnly { get; protected set; } = false;
 
         protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; }
 
@@ -170,38 +175,35 @@ namespace Umbraco.Examine
             }
 
             //Need to check if the index set or IndexerData is specified...
-            if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0))
+            if (config["indexSet"] == null && FieldDefinitionCollection.Count == 0)
             {
                 //if we don't have either, then we'll try to set the index set by naming conventions
                 var found = false;
-                if (name.EndsWith("Indexer"))
+
+                var possibleSuffixes = new[] {"Index", "Indexer"};
+                foreach (var suffix in possibleSuffixes)
                 {
-                    var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet";
+                    if (!name.EndsWith(suffix)) continue;
+
+                    var setNameByConvension = name.Remove(name.LastIndexOf(suffix, StringComparison.Ordinal)) + "IndexSet";
                     //check if we can assign the index set by naming convention
                     var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
 
-                    if (set != null)
+                    if (set == null) continue;
+
+                    //we've found an index set by naming conventions :)
+                    IndexSetName = set.SetName;
+
+                    var indexSet = IndexSets.Instance.Sets[IndexSetName];
+
+                    //get the index criteria and ensure folder
+                    ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
+                    foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
                     {
-                        //we've found an index set by naming conventions :)
-                        IndexSetName = set.SetName;
-
-                        var indexSet = IndexSets.Instance.Sets[IndexSetName];
-
-                        //if tokens are declared in the path, then use them (i.e. {machinename} )
-                        indexSet.ReplaceTokensInIndexPath();
-
-                        //get the index criteria and ensure folder
-                        ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
-                        foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
-                        {
-                            FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
-                        }
-
-                        //now set the index folder
-                        LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
-
-                        found = true;
+                        FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
                     }
+                    found = true;
+                    break;
                 }
 
                 if (!found)
@@ -222,86 +224,33 @@ namespace Umbraco.Examine
 
                     var indexSet = IndexSets.Instance.Sets[IndexSetName];
 
-                    //if tokens are declared in the path, then use them (i.e. {machinename} )
-                    indexSet.ReplaceTokensInIndexPath();
-
                     //get the index criteria and ensure folder
                     ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
                     foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
                     {
                         FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
                     }
-
-                    //now set the index folder
-                    LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
                 }
             }
 
             base.Initialize(name, config);
         }
-
+        
         #endregion
 
-
         /// <summary>
         /// override to check if we can actually initialize.
         /// </summary>
         /// <remarks>
         /// This check is required since the base examine lib will try to rebuild on startup
         /// </remarks>
-        public override void RebuildIndex()
-        {
-            if (CanInitialize())
-            {
-                ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index");
-                using (new SafeCallContext())
-                {
-                    base.RebuildIndex();
-                }
-            }
-        }
-
-        /// <summary>
-        /// override to check if we can actually initialize.
-        /// </summary>
-        /// <remarks>
-        /// This check is required since the base examine lib will try to rebuild on startup
-        /// </remarks>
-        public override void IndexAll(string type)
+        protected override void PerformDeleteFromIndex(string nodeId, Action<IndexOperationEventArgs> onComplete)
         {
             if (CanInitialize())
             {
                 using (new SafeCallContext())
                 {
-                    base.IndexAll(type);
-                }
-            }
-        }
-
-        public override void IndexItems(IEnumerable<ValueSet> nodes)
-        {
-            if (CanInitialize())
-            {
-                using (new SafeCallContext())
-                {
-                    base.IndexItems(nodes);
-                }
-            }
-        }
-
-        /// <summary>
-        /// override to check if we can actually initialize.
-        /// </summary>
-        /// <remarks>
-        /// This check is required since the base examine lib will try to rebuild on startup
-        /// </remarks>
-        public override void DeleteFromIndex(string nodeId)
-        {
-            if (CanInitialize())
-            {
-                using (new SafeCallContext())
-                {
-                    base.DeleteFromIndex(nodeId);
+                    base.PerformDeleteFromIndex(nodeId, onComplete);
                 }
             }
         }
@@ -317,17 +266,6 @@ namespace Umbraco.Examine
             return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
         }
 
-        /// <summary>
-        /// Reindexes all supported types
-        /// </summary>
-        protected override void PerformIndexRebuild()
-        {
-            foreach (var t in SupportedTypes)
-            {
-                IndexAll(t);
-            }
-        }
-
         /// <summary>
         /// overridden for logging
         /// </summary>
@@ -339,17 +277,20 @@ namespace Umbraco.Examine
         }
 
         /// <summary>
-        /// This ensures that the special __Raw_ fields are indexed
+        /// This ensures that the special __Raw_ fields are indexed correctly
         /// </summary>
         /// <param name="docArgs"></param>
         protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs)
         {
             var d = docArgs.Document;
 
-            foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix)))
+            foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix)).ToList())
             {
                 if (f.Value.Count > 0)
                 {
+                    //remove the original value so we can store it the correct way
+                    d.RemoveField(f.Key);
+
                     d.Add(new Field(
                         f.Key,
                         f.Value[0].ToString(),
@@ -359,26 +300,21 @@ namespace Umbraco.Examine
                 }
             }
 
-            ProfilingLogger.Logger.Debug(GetType(),
-                "Write lucene doc id:{DocumentId}, category:{DocumentCategory}, type:{DocumentItemType}",
-                docArgs.ValueSet.Id,
-                docArgs.ValueSet.Category,
-                docArgs.ValueSet.ItemType);
-
-
             base.OnDocumentWriting(docArgs);
         }
 
         /// <summary>
         /// Overridden for logging.
         /// </summary>
-        protected override void AddDocument(Document doc, IndexItem item, IndexWriter writer)
+        protected override void AddDocument(Document doc, ValueSet valueSet, IndexWriter writer)
         {
             ProfilingLogger.Logger.Debug(GetType(),
-                "AddDocument {DocumentId} with type {DocumentItemType}",
-                item.ValueSet.Id,
-                item.ValueSet.ItemType);
-            base.AddDocument(doc, item, writer);
+                "Write lucene doc id:{DocumentId}, category:{DocumentCategory}, type:{DocumentItemType}",
+                valueSet.Id,
+                valueSet.Category,
+                valueSet.ItemType);
+
+            base.AddDocument(doc, valueSet, writer);
         }
 
         protected override void OnTransformingIndexValues(IndexingItemEventArgs e)
@@ -386,39 +322,16 @@ namespace Umbraco.Examine
             base.OnTransformingIndexValues(e);
 
             //ensure special __Path field
-            var path = e.IndexItem.ValueSet.GetValue("path");
+            var path = e.ValueSet.GetValue("path");
             if (path != null)
             {
-                e.IndexItem.ValueSet.Set(IndexPathFieldName, path);
-            }
-
-            //strip html of all users fields if we detect it has HTML in it.
-            //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return
-            //the value of the field 'as-is'.
-            foreach (var value in e.IndexItem.ValueSet.Values.ToList()) //ToList here to make a diff collection else we'll get collection modified errors
-            {
-                if (value.Value == null) continue;
-
-                if (value.Value.Count > 0)
-                {
-                    if (value.Value.First() is string str)
-                    {
-                        if (XmlHelper.CouldItBeXml(str))
-                        {
-                            //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later
-                            e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List<object> { str };
-                            //now replace the original value with the stripped html
-                            //TODO: This should be done with an analzer?!
-                            e.IndexItem.ValueSet.Values[value.Key] = new List<object> { str.StripHtml() };
-                        }
-                    }
-                }
+                e.ValueSet.Set(IndexPathFieldName, path);
             }
 
             //icon
-            if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false)
+            if (e.ValueSet.Values.TryGetValue("icon", out var icon) && e.ValueSet.Values.ContainsKey(IconFieldName) == false)
             {
-                e.IndexItem.ValueSet.Values[IconFieldName] = icon;
+                e.ValueSet.Values[IconFieldName] = icon;
             }
         }
 
@@ -431,5 +344,16 @@ namespace Umbraco.Examine
                 indexSet.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(),
                 indexSet.IndexParentId);
         }
+
+        #region IIndexDiagnostics
+
+        private readonly UmbracoExamineIndexDiagnostics _diagnostics;
+
+        public int DocumentCount => _diagnostics.DocumentCount;
+        public int FieldCount => _diagnostics.FieldCount;
+        public Attempt<string> IsHealthy() => _diagnostics.IsHealthy();
+        public virtual IReadOnlyDictionary<string, object> Metadata => _diagnostics.Metadata;
+
+        #endregion
     }
 }

src/Umbraco.Examine/UmbracoExamineSearcher.cs

@@ -1,156 +1,52 @@
-using System;
-using System.ComponentModel;
-using System.IO;
-using System.Linq;
-using Umbraco.Core;
-using Examine.LuceneEngine.Providers;
-using Lucene.Net.Analysis;
-using Lucene.Net.Index;
-using Umbraco.Core.Composing;
-using Umbraco.Examine.Config;
-using Directory = Lucene.Net.Store.Directory;
+//using System;
+//using System.ComponentModel;
+//using System.IO;
+//using System.Linq;
+//using Umbraco.Core;
+//using Examine.LuceneEngine.Providers;
+//using Lucene.Net.Analysis;
+//using Lucene.Net.Index;
+//using Umbraco.Core.Composing;
+//using Umbraco.Examine.Config;
+//using Directory = Lucene.Net.Store.Directory;
+//using Examine.LuceneEngine;
 
+//namespace Umbraco.Examine
+//{
+//    /// <summary>
+//    /// An Examine searcher which uses Lucene.Net as the
+//    /// </summary>
+//    public class UmbracoExamineSearcher : LuceneSearcher
+//    {
+//        /// <summary>
+//        /// Constructor to allow for creating an indexer at runtime
+//        /// </summary>
+//        /// <param name="name"></param>
+//        /// <param name="luceneDirectory"></param>
+//        /// <param name="analyzer"></param>
+//        public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
+//            : base(name, luceneDirectory, analyzer, fieldValueTypeCollection)
+//        {
+//        }
 
-namespace Umbraco.Examine
-{
-    /// <summary>
-    /// An Examine searcher which uses Lucene.Net as the
-    /// </summary>
-    public class UmbracoExamineSearcher : LuceneSearcher
-    {
-        private readonly bool _configBased = false;
+//        /// <inheritdoc />
+//        public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
+//            : base(name, writer, analyzer, fieldValueTypeCollection)
+//        {
+//        }
 
-        /// <summary>
-        /// Default constructor for config based construction
-        /// </summary>
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public UmbracoExamineSearcher()
-        {
-            _configBased = true;
-        }
+//        /// <summary>
+//        /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias
+//        /// </summary>
+//        /// <returns></returns>
+//        public override string[] GetAllIndexedFields()
+//        {
+//            var fields = base.GetAllIndexedFields();
+//            return fields
+//                .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName)
+//                .Where(x => x != LuceneIndex.ItemTypeFieldName)
+//                .ToArray();
+//        }
 
-        /// <summary>
-        /// Constructor to allow for creating an indexer at runtime
-        /// </summary>
-        /// <param name="name"></param>
-        /// <param name="indexPath"></param>
-        /// <param name="analyzer"></param>
-        public UmbracoExamineSearcher(string name, DirectoryInfo indexPath, Analyzer analyzer)
-            : base(name, indexPath, analyzer)
-        {
-            _configBased = false;
-        }
-
-        /// <summary>
-        /// Constructor to allow for creating an indexer at runtime
-        /// </summary>
-        /// <param name="name"></param>
-        /// <param name="luceneDirectory"></param>
-        /// <param name="analyzer"></param>
-        public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer)
-            : base(name, luceneDirectory, analyzer)
-        {
-            _configBased = false;
-        }
-
-        /// <inheritdoc />
-        public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer) : base(name, writer, analyzer)
-        {
-            _configBased = false;
-        }
-
-        /// <summary>
-        /// Name of the Lucene.NET index set
-        /// </summary>
-        public string IndexSetName { get; private set; }
-
-        /// <summary>
-        /// Method used for initializing based on a configuration based searcher
-        /// </summary>
-        /// <param name="name"></param>
-        /// <param name="config"></param>
-        public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
-        {
-            if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name));
-            
-            //We need to check if we actually can initialize, if not then don't continue
-            if (CanInitialize() == false)
-            {
-                return;
-            }
-
-            //need to check if the index set is specified, if it's not, we'll see if we can find one by convension
-            //if the folder is not null and the index set is null, we'll assume that this has been created at runtime.
-            //NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look.
-            var luceneDirectory = GetLuceneDirectory();
-            if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null))
-            {
-                //if we don't have either, then we'll try to set the index set by naming convensions
-                var found = false;
-                if (name.EndsWith("Searcher"))
-                {
-                    var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet";
-                    //check if we can assign the index set by naming convension
-                    var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
-
-                    if (set != null)
-                    {
-                        set.ReplaceTokensInIndexPath();
-
-                        //we've found an index set by naming convensions :)
-                        IndexSetName = set.SetName;
-                        found = true;
-                    }
-                }
-
-                if (!found)
-                    throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration");
-
-                //get the folder to index
-                LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
-            }
-            else if (config["indexSet"] != null && luceneDirectory == null)
-            {
-                if (IndexSets.Instance.Sets[config["indexSet"]] == null)
-                    throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist");
-
-                IndexSetName = config["indexSet"];
-
-                var indexSet = IndexSets.Instance.Sets[IndexSetName];
-
-                indexSet.ReplaceTokensInIndexPath();
-
-                //get the folder to index
-                LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index"));
-            }
-
-            base.Initialize(name, config);
-
-        }
-
-        /// <summary>
-        /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes
-        /// </summary>
-
-        protected bool CanInitialize()
-        {
-            // only affects indexers that are config file based, if an index was created via code then
-            // this has no effect, it is assumed the index would not be created if it could not be initialized
-            return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
-        }
-
-        /// <summary>
-        /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias
-        /// </summary>
-        /// <returns></returns>
-        public override string[] GetAllIndexedFields()
-        {
-            var fields = base.GetAllIndexedFields();
-            return fields
-                .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName)
-                .Where(x => x != LuceneIndexer.ItemTypeFieldName)
-                .ToArray();
-        }
-
-    }
-}
+//    }
+//}

src/Umbraco.Examine/UmbracoMemberIndexer.cs

@@ -5,6 +5,7 @@ using Umbraco.Core.Models;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Services;
 using System.Collections.Generic;
+using System.Collections.Specialized;
 using System.ComponentModel;
 using Examine;
 using Examine.LuceneEngine;
@@ -23,15 +24,12 @@ namespace Umbraco.Examine
     /// </summary>
     public class UmbracoMemberIndexer : UmbracoExamineIndexer
     {
-        private readonly IMemberService _memberService;
-
         /// <summary>
         /// Constructor for config/provider based indexes
         /// </summary>
         [EditorBrowsable(EditorBrowsableState.Never)]
         public UmbracoMemberIndexer()
         {
-            _memberService = Current.Services.MemberService;
         }
 
         /// <summary>
@@ -42,7 +40,6 @@ namespace Umbraco.Examine
         /// <param name="luceneDirectory"></param>
         /// <param name="profilingLogger"></param>
         /// <param name="validator"></param>
-        /// <param name="memberService"></param>
         /// <param name="analyzer"></param>
         public UmbracoMemberIndexer(
             string name, 
@@ -50,118 +47,29 @@ namespace Umbraco.Examine
             Directory luceneDirectory,
             Analyzer analyzer,
             ProfilingLogger profilingLogger,
-            IValueSetValidator validator,
-            IMemberService memberService) :
+            IValueSetValidator validator = null) :
             base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
         {
-            _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService));
         }
 
+        public override void Initialize(string name, NameValueCollection config)
+        {
+            base.Initialize(name, config);
+
+            ValueSetValidator = new MemberValueSetValidator(ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes);
+        }
 
         /// <summary>
         /// Overridden to ensure that the umbraco system field definitions are in place
         /// </summary>
-        /// <param name="x"></param>
         /// <param name="indexValueTypesFactory"></param>
         /// <returns></returns>
-        protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
+        protected override FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
         {
             var keyDef = new FieldDefinition("__key", FieldDefinitionTypes.Raw);
             FieldDefinitionCollection.TryAdd(keyDef.Name, keyDef);
 
-            return base.CreateFieldValueTypes(x, indexValueTypesFactory);
-        }
-
-        /// <inheritdoc />
-        protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Member};
-
-        /// <summary>
-        /// Reindex all members
-        /// </summary>
-        /// <param name="type"></param>
-        protected override void PerformIndexAll(string type)
-        {
-            //This only supports members
-            if (SupportedTypes.Contains(type) == false)
-                return;
-
-            const int pageSize = 1000;
-            var pageIndex = 0;
-
-            IMember[] members;
-
-            if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
-            {
-                //if there are specific node types then just index those
-                foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes)
-                {
-                    do
-                    {
-                        members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
-
-                        IndexItems(GetValueSets(members));
-
-                        pageIndex++;
-                    } while (members.Length == pageSize);
-                }
-            }
-            else
-            {
-                //no node types specified, do all members
-                do
-                {
-                    members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
-
-                    IndexItems(GetValueSets(members));
-
-                    pageIndex++;
-                } while (members.Length == pageSize);
-            }
-        }
-
-        public static IEnumerable<ValueSet> GetValueSets(params IMember[] members)
-        {
-            foreach (var m in members)
-            {
-                var values = new Dictionary<string, object[]>
-                {
-                    {"icon", new object[] {m.ContentType.Icon}},
-                    {"id", new object[] {m.Id}},
-                    {"key", new object[] {m.Key}},
-                    {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
-                    {"level", new object[] {m.Level}},
-                    {"creatorID", new object[] {m.CreatorId}},
-                    {"sortOrder", new object[] {m.SortOrder}},
-                    {"createDate", new object[] {m.CreateDate}},
-                    {"updateDate", new object[] {m.UpdateDate}},
-                    {"nodeName", new object[] {m.Name}},
-                    {"path", new object[] {m.Path}},
-                    {"nodeType", new object[] {m.ContentType.Id}},
-                    {"loginName", new object[] {m.Username}},
-                    {"email", new object[] {m.Email}},
-                };
-
-                foreach (var property in m.Properties)
-                {
-                    //only add the value if its not null or empty (we'll check for string explicitly here too)
-                    var val = property.GetValue();
-                    switch (val)
-                    {
-                        case null:
-                            continue;
-                        case string strVal when strVal.IsNullOrWhiteSpace() == false:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                        default:
-                            values.Add(property.Alias, new[] { val });
-                            break;
-                    }
-                }
-
-                var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values);
-
-                yield return vs;
-            }
+            return base.CreateFieldValueTypes(indexValueTypesFactory);
         }
 
         /// <summary>
@@ -172,21 +80,12 @@ namespace Umbraco.Examine
         {
             base.OnTransformingIndexValues(e);
 
-            if (e.IndexItem.ValueSet.Values.TryGetValue("key", out var key) && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false)
+            if (e.ValueSet.Values.TryGetValue("key", out var key) && e.ValueSet.Values.ContainsKey("__key") == false)
             {
                 //double __ prefix means it will be indexed as culture invariant
-                e.IndexItem.ValueSet.Values["__key"] = key;
+                e.ValueSet.Values["__key"] = key;
             }
 
-            if (e.IndexItem.ValueSet.Values.TryGetValue("email", out var email) && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false)
-            {
-                if (email.Count > 0)
-                {
-                    //will be indexed as full text (the default anaylyzer)
-                    e.IndexItem.ValueSet.Values["_searchEmail"] = new List<object> { email[0]?.ToString().Replace(".", " ").Replace("@", " ") };
-                }
-                
-            }
         }
 
     }

src/Umbraco.Examine/ValueSetValidator.cs

@@ -0,0 +1,100 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Examine;
+using Examine.LuceneEngine.Providers;
+using Umbraco.Core;
+
+namespace Umbraco.Examine
+{
+    /// <summary>
+    /// Performing basic validation of a value set
+    /// </summary>
+    public class ValueSetValidator : IValueSetValidator
+    {
+        public ValueSetValidator(
+            IEnumerable<string> includeItemTypes,
+            IEnumerable<string> excludeItemTypes,
+            IEnumerable<string> includeFields,
+            IEnumerable<string> excludeFields)
+        {
+            IncludeItemTypes = includeItemTypes;
+            ExcludeItemTypes = excludeItemTypes;
+            IncludeFields = includeFields;
+            ExcludeFields = excludeFields;
+            ValidIndexCategories = null;
+        }
+
+        protected virtual IEnumerable<string> ValidIndexCategories { get; }
+
+        /// <summary>
+        /// Optional inclusion list of content types to index
+        /// </summary>
+        /// <remarks>
+        /// All other types will be ignored if they do not match this list
+        /// </remarks>
+        public IEnumerable<string> IncludeItemTypes { get; }
+
+        /// <summary>
+        /// Optional exclusion list of content types to ignore
+        /// </summary>
+        /// <remarks>
+        /// Any content type alias matched in this will not be included in the index
+        /// </remarks>
+        public IEnumerable<string> ExcludeItemTypes { get; }
+
+        /// <summary>
+        /// Optional inclusion list of index fields to index
+        /// </summary>
+        /// <remarks>
+        /// If specified, all other fields in a <see cref="ValueSet"/> will be filtered
+        /// </remarks>
+        public IEnumerable<string> IncludeFields { get; }
+
+        /// <summary>
+        /// Optional exclusion list of index fields
+        /// </summary>
+        /// <remarks>
+        /// If specified, all fields matching these field names will be filtered from the <see cref="ValueSet"/>
+        /// </remarks>
+        public IEnumerable<string> ExcludeFields { get; }
+
+        public virtual ValueSetValidationResult Validate(ValueSet valueSet)
+        {
+            if (ValidIndexCategories != null && !ValidIndexCategories.InvariantContains(valueSet.Category))
+                return ValueSetValidationResult.Failed;
+
+            //check if this document is of a correct type of node type alias
+            if (IncludeItemTypes != null && !IncludeItemTypes.InvariantContains(valueSet.ItemType))
+                return ValueSetValidationResult.Failed;
+
+            //if this node type is part of our exclusion list
+            if (ExcludeItemTypes != null && ExcludeItemTypes.InvariantContains(valueSet.ItemType))
+                return ValueSetValidationResult.Failed;
+
+            var isFiltered = false;
+
+            //filter based on the fields provided (if any)
+            if (IncludeFields != null || ExcludeFields != null)
+            {
+                foreach (var key in valueSet.Values.Keys.ToList())
+                {
+                    if (IncludeFields != null && !IncludeFields.InvariantContains(key))
+                    {
+                        valueSet.Values.Remove(key); //remove any value with a key that doesn't match the inclusion list
+                        isFiltered = true;
+                    }
+
+                    if (ExcludeFields != null && ExcludeFields.InvariantContains(key))
+                    {
+                        valueSet.Values.Remove(key); //remove any value with a key that matches the exclusion list
+                        isFiltered = true;
+                    }
+
+                }
+            }
+
+            return isFiltered ? ValueSetValidationResult.Filtered : ValueSetValidationResult.Valid;
+        }
+    }
+}

src/Umbraco.Tests.Benchmarks/EnumeratorBenchmarks.cs

@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using BenchmarkDotNet.Attributes;
+
+namespace Umbraco.Tests.Benchmarks
+{
+    [MemoryDiagnoser]
+    public class EnumeratorBenchmarks
+    {
+        [Benchmark(Baseline = true)]
+        public void WithArray()
+        {
+            foreach (var t in EnumerateOneWithArray(1)) ;
+        }
+
+        [Benchmark]
+        public void WithYield()
+        {
+            foreach (var t in EnumerateOneWithYield(1)) ;
+        }
+
+        private IEnumerable<T> EnumerateOneWithArray<T>(T o) => new [] { o };
+
+        private IEnumerable<T> EnumerateOneWithYield<T>(T o)
+        {
+            yield return o;
+        }
+    }
+}

src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj

@@ -50,6 +50,7 @@
     <Compile Include="Config\QuickRunConfigAttribute.cs" />
     <Compile Include="Config\QuickRunWithMemoryDiagnoserConfigAttribute.cs" />
     <Compile Include="CtorInvokeBenchmarks.cs" />
+    <Compile Include="EnumeratorBenchmarks.cs" />
     <Compile Include="LinqCastBenchmarks.cs" />
     <Compile Include="ModelToSqlExpressionHelperBenchmarks.cs" />
     <Compile Include="Program.cs" />

src/Umbraco.Tests/Models/Collections/PropertyCollectionTests.cs

@@ -78,7 +78,7 @@ namespace Umbraco.Tests.Models.Collections
         [Test]
         public void PropertyGroups_Collection_FirstOrDefault_Returns_Null()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
 
             Assert.That(contentType.PropertyGroups, Is.Not.Null);
             Assert.That(contentType.PropertyGroups.FirstOrDefault(x => x.Name.InvariantEquals("Content")) == null, Is.False);

src/Umbraco.Tests/Models/ContentExtensionsTests.cs

@@ -16,7 +16,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_Reset_Clears_SavedPublishedState()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             content.PublishedState = PublishedState.Publishing;
@@ -29,7 +29,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_OnlyIfActuallyChanged_Content()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // if you assign a content property with its value it is not dirty
@@ -51,7 +51,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_OnlyIfActuallyChanged_User()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             var prop = content.Properties.First();
 
@@ -75,7 +75,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_UpdateDate()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             var prop = content.Properties.First();
 
@@ -98,7 +98,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_WasDirty_ContentProperty()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             content.ResetDirtyProperties(false);
             Assert.IsFalse(content.IsDirty());
@@ -125,7 +125,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_WasDirty_ContentSortOrder()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             content.ResetDirtyProperties(false);
             Assert.IsFalse(content.IsDirty());
@@ -152,7 +152,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void DirtyProperty_WasDirty_UserProperty()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             var prop = content.Properties.First();
             content.ResetDirtyProperties(false);

src/Umbraco.Tests/Models/ContentTests.cs

@@ -128,7 +128,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void All_Dirty_Properties_Get_Reset()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             content.ResetDirtyProperties(false);
@@ -144,7 +144,7 @@ namespace Umbraco.Tests.Models
         public void Can_Verify_Mocked_Content()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -157,7 +157,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_Property_Value()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -173,7 +173,7 @@ namespace Umbraco.Tests.Models
         public void Can_Set_Property_Value_As_String()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -189,7 +189,7 @@ namespace Umbraco.Tests.Models
         public void Can_Clone_Content_With_Reset_Identity()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             content.Id = 10;
             content.Key = new Guid("29181B97-CB8F-403F-86DE-5FEB497F4800");
@@ -218,7 +218,7 @@ namespace Umbraco.Tests.Models
         public void Can_Deep_Clone_Perf_Test()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             var i = 200;
@@ -269,7 +269,7 @@ namespace Umbraco.Tests.Models
         public void Can_Deep_Clone()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
             contentType.Variations = ContentVariation.Culture;
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
@@ -394,7 +394,7 @@ namespace Umbraco.Tests.Models
         public void Remember_Dirty_Properties()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
             contentType.Variations = ContentVariation.Culture;
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
@@ -475,7 +475,7 @@ namespace Umbraco.Tests.Models
             var ss = new SerializationService(new JsonNetSerializer());
 
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
             var i = 200;
@@ -530,7 +530,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_Property_Value_Through_Anonymous_Object()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -548,7 +548,7 @@ namespace Umbraco.Tests.Models
         public void Can_Verify_Dirty_Property_On_Content()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -564,7 +564,7 @@ namespace Umbraco.Tests.Models
         public void Can_Add_PropertyGroup_On_ContentType()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -579,7 +579,7 @@ namespace Umbraco.Tests.Models
         public void Can_Remove_PropertyGroup_From_ContentType()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.ResetDirtyProperties();
 
             // Act
@@ -594,7 +594,7 @@ namespace Umbraco.Tests.Models
         public void Can_Add_PropertyType_To_Group_On_ContentType()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -616,7 +616,7 @@ namespace Umbraco.Tests.Models
         public void Can_Add_New_Property_To_New_PropertyType()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -638,7 +638,7 @@ namespace Umbraco.Tests.Models
         public void Can_Add_New_Property_To_New_PropertyType_In_New_PropertyGroup()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act
@@ -669,7 +669,7 @@ namespace Umbraco.Tests.Models
         public void Can_Update_PropertyType_Through_Content_Properties()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             // Act - note that the PropertyType's properties like SortOrder is not updated through the Content object
@@ -689,7 +689,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_ContentType_On_Content()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var simpleContentType = MockedContentTypes.CreateSimpleContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
@@ -708,7 +708,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_ContentType_On_Content_And_Set_Property_Value()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var simpleContentType = MockedContentTypes.CreateSimpleContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
@@ -725,7 +725,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_ContentType_On_Content_And_Still_Get_Old_Properties()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var simpleContentType = MockedContentTypes.CreateSimpleContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
@@ -744,7 +744,7 @@ namespace Umbraco.Tests.Models
         public void Can_Change_ContentType_On_Content_And_Clear_Old_PropertyTypes()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var simpleContentType = MockedContentTypes.CreateSimpleContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
@@ -760,7 +760,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void Can_Verify_Content_Is_Published()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             var content = MockedContent.CreateTextpageContent(contentType, "Textpage", -1);
 
             content.ResetDirtyProperties();
@@ -794,7 +794,7 @@ namespace Umbraco.Tests.Models
         public void Adding_PropertyGroup_To_ContentType_Results_In_Dirty_Entity()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.ResetDirtyProperties();
 
             // Act
@@ -811,7 +811,7 @@ namespace Umbraco.Tests.Models
         public void After_Committing_Changes_Was_Dirty_Is_True()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.ResetDirtyProperties(); //reset
 
             // Act
@@ -828,7 +828,7 @@ namespace Umbraco.Tests.Models
         public void After_Committing_Changes_Was_Dirty_Is_True_On_Changed_Property()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.ResetDirtyProperties(); //reset
             var content = MockedContent.CreateTextpageContent(contentType, "test", -1);
             content.ResetDirtyProperties();
@@ -859,7 +859,7 @@ namespace Umbraco.Tests.Models
         public void If_Not_Committed_Was_Dirty_Is_False()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
 
             // Act
             contentType.Alias = "newAlias";
@@ -873,7 +873,7 @@ namespace Umbraco.Tests.Models
         public void Detect_That_A_Property_Is_Removed()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             Assert.That(contentType.WasPropertyDirty("HasPropertyTypeBeenRemoved"), Is.False);
 
             // Act
@@ -887,7 +887,7 @@ namespace Umbraco.Tests.Models
         public void Adding_PropertyType_To_PropertyGroup_On_ContentType_Results_In_Dirty_Entity()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.ResetDirtyProperties();
 
             // Act
@@ -974,7 +974,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void Can_Avoid_Circular_Dependencies_In_Composition()
         {
-            var textPage = MockedContentTypes.CreateTextpageContentType();
+            var textPage = MockedContentTypes.CreateTextPageContentType();
             var parent = MockedContentTypes.CreateSimpleContentType("parent", "Parent", null, true);
             var meta = MockedContentTypes.CreateMetaContentType();
             var mixin1 = MockedContentTypes.CreateSimpleContentType("mixin1", "Mixin1", new PropertyTypeCollection(true,

src/Umbraco.Tests/Models/ContentTypeTests.cs

@@ -33,7 +33,7 @@ namespace Umbraco.Tests.Models
         [Test]
         public void Can_Deep_Clone_Content_Type_With_Reset_Identities()
         {
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
 
             var i = 200;
@@ -105,7 +105,7 @@ namespace Umbraco.Tests.Models
         public void Can_Deep_Clone_Content_Type_Perf_Test()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
 
             var i = 200;
@@ -155,7 +155,7 @@ namespace Umbraco.Tests.Models
         public void Can_Deep_Clone_Content_Type()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
 
             var i = 200;
@@ -256,7 +256,7 @@ namespace Umbraco.Tests.Models
             var ss = new SerializationService(new JsonNetSerializer());
 
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             contentType.Id = 99;
 
             var i = 200;

src/Umbraco.Tests/Models/ContentXmlTest.cs

@@ -18,7 +18,7 @@ namespace Umbraco.Tests.Models
         public void Can_Generate_Xml_Representation_Of_Content()
         {
             // Arrange
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate); // else, FK violation on contentType!
             ServiceContext.ContentTypeService.Save(contentType);
 

src/Umbraco.Tests/Models/Mapping/ContentTypeModelMappingTests.cs

@@ -427,7 +427,7 @@ namespace Umbraco.Tests.Models.Mapping
 
             // setup the mocks to return the data we want to test against...
 
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             MockedContentTypes.EnsureAllIds(contentType, 8888);
 
             //Act

src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs

@@ -964,7 +964,7 @@ namespace Umbraco.Tests.Persistence.Repositories
             ServiceContext.ContentTypeService.Save(simpleContentType);
 
             //Create and Save ContentType "textPage" -> (NodeDto.NodeIdSeed + 1)
-            ContentType textpageContentType = MockedContentTypes.CreateTextpageContentType();
+            ContentType textpageContentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.ContentTypeService.Save(textpageContentType);
         }
     }

src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs

@@ -23,6 +23,7 @@ using Umbraco.Tests.Testing;
 using LightInject;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Services;
+using Umbraco.Core.PropertyEditors;
 
 namespace Umbraco.Tests.PublishedContent
 {
@@ -112,11 +113,13 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ensure_Children_Sorted_With_Examine()
         {
-            using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
-            {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
 
-                indexer.RebuildIndex();
+            using (var luceneDir = new RandomIdRamDirectory())
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
+            {
+                rebuilder.Populate(indexer);
 
                 var searcher = indexer.GetSearcher();
                 var ctx = GetUmbracoContext("/test");
@@ -137,14 +140,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Do_Not_Find_In_Recycle_Bin()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
                 //include unpublished content since this uses the 'internal' indexer, it's up to the media cache to filter
-                options: new UmbracoContentIndexerOptions(true, false, null)))
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 var searcher = indexer.GetSearcher();
                 var ctx = GetUmbracoContext("/test");
@@ -183,12 +187,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Children_With_Examine()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 var searcher = indexer.GetSearcher();
                 var ctx = GetUmbracoContext("/test");
@@ -208,12 +214,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Descendants_With_Examine()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 var searcher = indexer.GetSearcher();
                 var ctx = GetUmbracoContext("/test");
@@ -233,12 +241,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void DescendantsOrSelf_With_Examine()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 var searcher = indexer.GetSearcher();
                 var ctx = GetUmbracoContext("/test");
@@ -258,12 +268,15 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void Ancestors_With_Examine()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 var ctx = GetUmbracoContext("/test");
                 var searcher = indexer.GetSearcher();
@@ -280,11 +293,14 @@ namespace Umbraco.Tests.PublishedContent
         [Test]
         public void AncestorsOrSelf_With_Examine()
         {
+            var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(true)))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
+                rebuilder.Populate(indexer);
 
 
                 var ctx = GetUmbracoContext("/test");

src/Umbraco.Tests/Services/ContentServicePerformanceTest.cs

@@ -65,9 +65,9 @@ namespace Umbraco.Tests.Services
             // ... NOPE, made even more nice changes, it is now...
             // 4452ms !!!!!!!
 
-            var contentType1 = MockedContentTypes.CreateTextpageContentType("test1", "test1");
-            var contentType2 = MockedContentTypes.CreateTextpageContentType("test2", "test2");
-            var contentType3 = MockedContentTypes.CreateTextpageContentType("test3", "test3");
+            var contentType1 = MockedContentTypes.CreateTextPageContentType("test1", "test1");
+            var contentType2 = MockedContentTypes.CreateTextPageContentType("test2", "test2");
+            var contentType3 = MockedContentTypes.CreateTextPageContentType("test3", "test3");
             ServiceContext.ContentTypeService.Save(new[] { contentType1, contentType2, contentType3 });
             contentType1.AllowedContentTypes = new[]
             {
@@ -291,7 +291,7 @@ namespace Umbraco.Tests.Services
         public void CreateTestData()
         {
             //Create and Save ContentType "textpage" -> NodeDto.NodeIdSeed
-            ContentType contentType = MockedContentTypes.CreateTextpageContentType();
+            ContentType contentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.ContentTypeService.Save(contentType);
         }
     }

src/Umbraco.Tests/Services/ContentServiceTests.cs

@@ -92,7 +92,7 @@ namespace Umbraco.Tests.Services
             var contentService = ServiceContext.ContentService;
             var contentTypeService = ServiceContext.ContentTypeService;
 
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
             contentTypeService.Save(contentType);
 
@@ -118,7 +118,7 @@ namespace Umbraco.Tests.Services
             var contentService = ServiceContext.ContentService;
             var contentTypeService = ServiceContext.ContentTypeService;
 
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
             contentTypeService.Save(contentType);
 
@@ -142,7 +142,7 @@ namespace Umbraco.Tests.Services
             var contentService = ServiceContext.ContentService;
             var contentTypeService = ServiceContext.ContentTypeService;
 
-            var contentType = MockedContentTypes.CreateTextpageContentType();
+            var contentType = MockedContentTypes.CreateTextPageContentType();
             ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
             contentTypeService.Save(contentType);
 
@@ -170,10 +170,10 @@ namespace Umbraco.Tests.Services
             var contentService = ServiceContext.ContentService;
             var contentTypeService = ServiceContext.ContentTypeService;
 
-            var ct1 = MockedContentTypes.CreateTextpageContentType("ct1");
+            var ct1 = MockedContentTypes.CreateTextPageContentType("ct1");
             ServiceContext.FileService.SaveTemplate(ct1.DefaultTemplate);
             contentTypeService.Save(ct1);
-            var ct2 = MockedContentTypes.CreateTextpageContentType("ct2");
+            var ct2 = MockedContentTypes.CreateTextPageContentType("ct2");
             ServiceContext.FileService.SaveTemplate(ct2.DefaultTemplate);
             contentTypeService.Save(ct2);
 

src/Umbraco.Tests/Services/ContentTypeServiceTests.cs

@@ -611,7 +611,7 @@ namespace Umbraco.Tests.Services
         [Test]
         public void Deleting_PropertyType_Removes_The_Property_From_Content()
         {
-            IContentType contentType1 = MockedContentTypes.CreateTextpageContentType("test1", "Test1");
+            IContentType contentType1 = MockedContentTypes.CreateTextPageContentType("test1", "Test1");
             ServiceContext.FileService.SaveTemplate(contentType1.DefaultTemplate);
             ServiceContext.ContentTypeService.Save(contentType1);
             IContent contentItem = MockedContent.CreateTextpageContent(contentType1, "Testing", -1);
@@ -633,11 +633,11 @@ namespace Umbraco.Tests.Services
         [Test]
         public void Rebuild_Content_Xml_On_Alias_Change()
         {
-            var contentType1 = MockedContentTypes.CreateTextpageContentType("test1", "Test1");
+            var contentType1 = MockedContentTypes.CreateTextPageContentType("test1", "Test1");
             ServiceContext.FileService.SaveTemplate(contentType1.DefaultTemplate);
             ServiceContext.ContentTypeService.Save(contentType1);
 
-            var contentType2 = MockedContentTypes.CreateTextpageContentType("test2", "Test2");
+            var contentType2 = MockedContentTypes.CreateTextPageContentType("test2", "Test2");
             ServiceContext.FileService.SaveTemplate(contentType2.DefaultTemplate);
             ServiceContext.ContentTypeService.Save(contentType2);
 
@@ -701,7 +701,7 @@ namespace Umbraco.Tests.Services
         [Test]
         public void Rebuild_Content_Xml_On_Property_Removal()
         {
-            var contentType1 = MockedContentTypes.CreateTextpageContentType("test1", "Test1");
+            var contentType1 = MockedContentTypes.CreateTextPageContentType("test1", "Test1");
             ServiceContext.FileService.SaveTemplate(contentType1.DefaultTemplate);
             ServiceContext.ContentTypeService.Save(contentType1);
             var contentItems1 = MockedContent.CreateTextpageContent(contentType1, -1, 10).ToArray();

src/Umbraco.Tests/TestHelpers/ControllerTesting/TestControllerActivatorBase.cs

@@ -169,12 +169,10 @@ namespace Umbraco.Tests.TestHelpers.ControllerTesting
                 Mock.Of<IPublishedContent>(),
                 mockedTypedContent,
                 Mock.Of<ITagQuery>(),
-                Mock.Of<IDataTypeService>(),
                 Mock.Of<ICultureDictionary>(),
                 Mock.Of<IUmbracoComponentRenderer>(),
                 membershipHelper,
-                serviceContext,
-                CacheHelper.NoCache);
+                serviceContext);
 
             return CreateController(controllerType, request, umbHelper);
         }

src/Umbraco.Tests/TestHelpers/Entities/MockedContentTypes.cs

@@ -26,7 +26,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
             return contentType;
         }
 
-        public static ContentType CreateTextpageContentType(string alias = "textPage", string name = "Text Page")
+        public static ContentType CreateTextPageContentType(string alias = "textPage", string name = "Text Page")
         {
             var contentType = new ContentType(-1)
                                   {

src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs

@@ -6,64 +6,38 @@ namespace Umbraco.Tests.TestHelpers.Stubs
 {
     internal class TestExamineManager : IExamineManager
     {
-        private readonly ConcurrentDictionary<string, IIndexer> _indexers = new ConcurrentDictionary<string, IIndexer>();
+        private readonly ConcurrentDictionary<string, IIndex> _indexers = new ConcurrentDictionary<string, IIndex>();
         private readonly ConcurrentDictionary<string, ISearcher> _searchers = new ConcurrentDictionary<string, ISearcher>();
 
-        public void AddIndexer(string name, IIndexer indexer)
+        public void AddIndex(IIndex indexer)
         {
-            _indexers.TryAdd(name, indexer);
+            _indexers.TryAdd(indexer.Name, indexer);
         }
 
-        public void AddSearcher(string name, ISearcher searcher)
+        public void AddSearcher(ISearcher searcher)
         {
-            _searchers.TryAdd(name, searcher);
+            _searchers.TryAdd(searcher.Name, searcher);
         }
-
-        public void DeleteFromIndexes(string nodeId)
-        {
-            //noop
-        }
-
-        public void DeleteFromIndexes(string nodeId, IEnumerable<IIndexer> providers)
-        {
-            //noop
-        }
-
+        
         public void Dispose()
         {
             //noop
         }
 
-        public IIndexer GetIndexer(string indexerName)
+        public bool TryGetIndex(string indexName, out IIndex index)
         {
-            return _indexers.TryGetValue(indexerName, out var indexer) ? indexer : null;
+            return _indexers.TryGetValue(indexName, out index);
         }
 
-        public ISearcher GetRegisteredSearcher(string searcherName)
+        public bool TryGetSearcher(string searcherName, out ISearcher searcher)
         {
-            return _searchers.TryGetValue(searcherName, out var indexer) ? indexer : null;
+            return _searchers.TryGetValue(searcherName, out searcher);
         }
 
-        public void IndexAll(string indexCategory)
-        {
-            //noop
-        }
+        public IEnumerable<IIndex> Indexes => _indexers.Values;
 
-        public void IndexItems(ValueSet[] nodes)
-        {
-            //noop
-        }
+        public IEnumerable<ISearcher> RegisteredSearchers => _searchers.Values;
 
-        public void IndexItems(ValueSet[] nodes, IEnumerable<IIndexer> providers)
-        {
-            //noop
-        }
-
-        public void RebuildIndexes()
-        {
-            //noop
-        }
-
-        public IReadOnlyDictionary<string, IIndexer> IndexProviders => _indexers;
+        public IReadOnlyDictionary<string, IIndex> IndexProviders => _indexers;
     }
 }

src/Umbraco.Tests/Testing/TestingTests/MockTests.cs

@@ -62,12 +62,10 @@ namespace Umbraco.Tests.Testing.TestingTests
                 Mock.Of<IPublishedContent>(),
                 Mock.Of<IPublishedContentQuery>(),
                 Mock.Of<ITagQuery>(),
-                Mock.Of<IDataTypeService>(),
                 Mock.Of<ICultureDictionary>(),
                 Mock.Of<IUmbracoComponentRenderer>(),
                 new MembershipHelper(umbracoContext, Mock.Of<MembershipProvider>(), Mock.Of<RoleProvider>()),
-                new ServiceContext(),
-                CacheHelper.CreateDisabledCacheHelper());
+                new ServiceContext());
             Assert.Pass();
         }
 

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -77,7 +77,7 @@
   <ItemGroup>
     <PackageReference Include="AutoMapper" Version="7.0.1" />
     <PackageReference Include="Castle.Core" Version="4.2.1" />
-    <PackageReference Include="Examine" Version="1.0.0-beta025" />
+    <PackageReference Include="Examine" Version="1.0.0-beta046" />
     <PackageReference Include="HtmlAgilityPack">
       <Version>1.8.9</Version>
     </PackageReference>
@@ -207,6 +207,7 @@
     <Compile Include="Testing\Objects\Accessors\TestUmbracoContextAccessor.cs" />
     <Compile Include="CoreThings\UdiTests.cs" />
     <Compile Include="UmbracoExamine\RandomIdRamDirectory.cs" />
+    <Compile Include="UmbracoExamine\UmbracoContentValueSetValidatorTests.cs" />
     <Compile Include="Web\AngularIntegration\AngularAntiForgeryTests.cs" />
     <Compile Include="Web\AngularIntegration\ContentModelSerializationTests.cs" />
     <Compile Include="Web\AngularIntegration\JsInitializationTests.cs" />
@@ -464,7 +465,6 @@
     <Compile Include="UmbracoExamine\IndexInitializer.cs" />
     <Compile Include="UmbracoExamine\IndexTest.cs" />
     <Compile Include="UmbracoExamine\SearchTests.cs" />
-    <Compile Include="UmbracoExamine\TestDataService.cs" />
     <Compile Include="UmbracoExamine\TestFiles.Designer.cs">
       <AutoGen>True</AutoGen>
       <DesignTime>True</DesignTime>

src/Umbraco.Tests/UmbracoExamine/EventsTest.cs

@@ -1,13 +1,16 @@
 using System;
 using System.Linq;
 using Examine;
+using LightInject;
 using Lucene.Net.Store;
 using NUnit.Framework;
 using Umbraco.Tests.Testing;
 using Umbraco.Examine;
+using Umbraco.Core.PropertyEditors;
 
 namespace Umbraco.Tests.UmbracoExamine
 {
+
     [TestFixture]
     [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
     public class EventsTest : ExamineBaseTest
@@ -16,13 +19,13 @@ namespace Umbraco.Tests.UmbracoExamine
         public void Events_Ignoring_Node()
         {
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
                 //make parent id 999 so all are ignored
-                options: new UmbracoContentIndexerOptions(false, false, 999)))
+                validator: new ContentValueSetValidator(false, 999)))
             using (indexer.ProcessNonAsync())
             {
                 var searcher = indexer.GetSearcher();
-                
+
                 var contentService = new ExamineDemoDataContentService();
                 //get a node from the data repo
                 var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
@@ -31,9 +34,9 @@ namespace Umbraco.Tests.UmbracoExamine
                                           .First();
 
                 var valueSet = node.ConvertToValueSet(IndexTypes.Content);
-                indexer.IndexItems(new[] {valueSet});
+                indexer.IndexItems(new[] { valueSet });
 
-                var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile());
+                var found = searcher.Search(searcher.CreateCriteria().Id((string)node.Attribute("id")).Compile());
 
                 Assert.AreEqual(0, found.TotalItemCount);
             }

src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs

@@ -13,6 +13,7 @@ using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Querying;
+using Umbraco.Core.PropertyEditors;
 using Umbraco.Core.Scoping;
 using Umbraco.Core.Services;
 using Umbraco.Core.Strings;
@@ -28,177 +29,158 @@ namespace Umbraco.Tests.UmbracoExamine
     /// </summary>
     internal static class IndexInitializer
     {
+        public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors, bool publishedValuesOnly)
+        {
+            var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService(), publishedValuesOnly);
+            return contentValueSetBuilder;
+        }
+
+        public static ContentIndexPopulator GetContentIndexRebuilder(PropertyEditorCollection propertyEditors, IContentService contentService, ISqlContext sqlContext, bool publishedValuesOnly)
+        {
+            var contentValueSetBuilder = GetContentValueSetBuilder(propertyEditors, publishedValuesOnly);
+            var contentIndexDataSource = new ContentIndexPopulator(true, null, contentService, sqlContext, contentValueSetBuilder);
+            return contentIndexDataSource;
+        }
+
+        public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService)
+        {
+            var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService());
+            var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
+            return mediaIndexDataSource;
+        }
+
+        public static IContentService GetMockContentService()
+        {
+            long longTotalRecs;
+            var demoData = new ExamineDemoDataContentService();
+
+            var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]")
+                .Root
+                .Elements()
+                .Select(x => Mock.Of<IContent>(
+                    m =>
+                        m.Id == (int)x.Attribute("id") &&
+                        m.ParentId == (int)x.Attribute("parentID") &&
+                        m.Level == (int)x.Attribute("level") &&
+                        m.CreatorId == 0 &&
+                        m.SortOrder == (int)x.Attribute("sortOrder") &&
+                        m.CreateDate == (DateTime)x.Attribute("createDate") &&
+                        m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
+                        m.Name == (string)x.Attribute("nodeName") &&
+                        m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
+                        m.Path == (string)x.Attribute("path") &&
+                        m.Properties == new PropertyCollection() &&
+                        m.ContentType == Mock.Of<IContentType>(mt =>
+                            mt.Icon == "test" &&
+                            mt.Alias == x.Name.LocalName &&
+                            mt.Id == (int)x.Attribute("nodeType"))))
+                .ToArray();
+
+
+            return Mock.Of<IContentService>(
+                x => x.GetPagedDescendants(
+                         It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<IQuery<IContent>>(), It.IsAny<Ordering>())
+                     == allRecs);
+        }
+
+        public static IUserService GetMockUserService()
+        {
+            return Mock.Of<IUserService>(x => x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
+        }
+
+        public static IMediaService GetMockMediaService()
+        {
+            long totalRecs;
+
+            var demoData = new ExamineDemoDataMediaService();
+
+            var allRecs = demoData.GetLatestMediaByXpath("//node")
+                .Root
+                .Elements()
+                .Select(x => Mock.Of<IMedia>(
+                    m =>
+                        m.Id == (int)x.Attribute("id") &&
+                        m.ParentId == (int)x.Attribute("parentID") &&
+                        m.Level == (int)x.Attribute("level") &&
+                        m.CreatorId == 0 &&
+                        m.SortOrder == (int)x.Attribute("sortOrder") &&
+                        m.CreateDate == (DateTime)x.Attribute("createDate") &&
+                        m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
+                        m.Name == (string)x.Attribute("nodeName") &&
+                        m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
+                        m.Path == (string)x.Attribute("path") &&
+                        m.Properties == new PropertyCollection() &&
+                        m.ContentType == Mock.Of<IMediaType>(mt =>
+                            mt.Alias == (string)x.Attribute("nodeTypeAlias") &&
+                            mt.Id == (int)x.Attribute("nodeType"))))
+                .ToArray();
+
+            // MOCK!
+            var mediaServiceMock = new Mock<IMediaService>();
+
+            mediaServiceMock
+                .Setup(x => x.GetPagedDescendants(
+                        It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out totalRecs, It.IsAny<IQuery<IMedia>>(), It.IsAny<Ordering>())
+                ).Returns(() => allRecs);
+
+            //mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs))
+            //    .Returns(() => allRecs.Select(x => x.ToXml()));
+
+            return mediaServiceMock.Object;
+        }
+
+        public static ILocalizationService GetMockLocalizationService()
+        {
+            return Mock.Of<ILocalizationService>(x => x.GetAllLanguages() == Array.Empty<ILanguage>());
+        }
+
+        public static IMediaTypeService GetMockMediaTypeService()
+        {
+            var mediaTypeServiceMock = new Mock<IMediaTypeService>();
+            mediaTypeServiceMock.Setup(x => x.GetAll())
+                .Returns(new List<IMediaType>
+                {
+                    new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"},
+                    new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"}
+                });
+            return mediaTypeServiceMock.Object;
+        }
+
         public static UmbracoContentIndexer GetUmbracoIndexer(
             ProfilingLogger profilingLogger,
             Directory luceneDir,
-            ISqlContext sqlContext,
             Analyzer analyzer = null,
-            IContentService contentService = null,
-            IMediaService mediaService = null,
-            IMemberService memberService = null,
-            IUserService userService = null,
-            IContentTypeService contentTypeService = null,
-            IMediaTypeService mediaTypeService = null,
-            UmbracoContentIndexerOptions options = null)
+            ILocalizationService languageService = null,
+            IContentValueSetValidator validator = null)
         {
-            if (contentService == null)
-            {
-                long longTotalRecs;
-                var demoData = new ExamineDemoDataContentService();
-
-                var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]")
-                    .Root
-                    .Elements()
-                    .Select(x => Mock.Of<IContent>(
-                        m =>
-                            m.Id == (int)x.Attribute("id") &&
-                            m.ParentId == (int)x.Attribute("parentID") &&
-                            m.Level == (int)x.Attribute("level") &&
-                            m.CreatorId == 0 &&
-                            m.SortOrder == (int)x.Attribute("sortOrder") &&
-                            m.CreateDate == (DateTime)x.Attribute("createDate") &&
-                            m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
-                            m.Name == (string)x.Attribute("nodeName") &&
-                            m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
-                            m.Path == (string)x.Attribute("path") &&
-                            m.Properties == new PropertyCollection() &&
-                            m.ContentType == Mock.Of<IContentType>(mt =>
-                                mt.Icon == "test" &&
-                                mt.Alias == x.Name.LocalName &&
-                                mt.Id == (int)x.Attribute("nodeType"))))
-                    .ToArray();
-
-
-                contentService = Mock.Of<IContentService>(
-                    x => x.GetPagedDescendants(
-                        It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<IQuery<IContent>>(), It.IsAny<Ordering>())
-                        ==
-                        allRecs);
-            }
-            if (userService == null)
-            {
-                userService = Mock.Of<IUserService>(x => x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
-            }
-
-            if (mediaService == null)
-            {
-                long totalRecs;
-
-                var demoData = new ExamineDemoDataMediaService();
-
-                var allRecs = demoData.GetLatestMediaByXpath("//node")
-                    .Root
-                    .Elements()
-                    .Select(x => Mock.Of<IMedia>(
-                        m =>
-                            m.Id == (int) x.Attribute("id") &&
-                            m.ParentId == (int) x.Attribute("parentID") &&
-                            m.Level == (int) x.Attribute("level") &&
-                            m.CreatorId == 0 &&
-                            m.SortOrder == (int) x.Attribute("sortOrder") &&
-                            m.CreateDate == (DateTime) x.Attribute("createDate") &&
-                            m.UpdateDate == (DateTime) x.Attribute("updateDate") &&
-                            m.Name == (string) x.Attribute("nodeName") &&
-                            m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
-                            m.Path == (string) x.Attribute("path") &&
-                            m.Properties == new PropertyCollection() &&
-                            m.ContentType == Mock.Of<IMediaType>(mt =>
-                                mt.Alias == (string) x.Attribute("nodeTypeAlias") &&
-                                mt.Id == (int) x.Attribute("nodeType"))))
-                    .ToArray();
-
-                // MOCK!
-                var mediaServiceMock = new Mock<IMediaService>();
-
-                mediaServiceMock
-                    .Setup(x => x.GetPagedDescendants(
-                            It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out totalRecs, It.IsAny<IQuery<IMedia>>(), It.IsAny<Ordering>())
-                    ).Returns(() => allRecs);
-
-                //mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs))
-                //    .Returns(() => allRecs.Select(x => x.ToXml()));
-
-                mediaService = mediaServiceMock.Object;
-
-            }
+            if (languageService == null)
+                languageService = GetMockLocalizationService();
 
             if (analyzer == null)
-            {
                 analyzer = new StandardAnalyzer(Version.LUCENE_30);
-            }
 
-            //var indexSet = new IndexSet();
-      //      var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies);
-
-            //var i = new UmbracoContentIndexer(indexCriteria,
-            //                                  luceneDir, //custom lucene directory
-      //                                        dataService,
-      //                                        contentService,
-      //                                        mediaService,
-      //                                        dataTypeService,
-      //                                        userService,
-      //                                        new[] { new DefaultUrlSegmentProvider() },
-            //                                  analyzer,
-            //                                  false);
-
-            //i.IndexSecondsInterval = 1;
-
-            if (options == null)
-            {
-                options = new UmbracoContentIndexerOptions(false, false, null);
-            }
-
-            if (mediaTypeService == null)
-            {
-                var mediaTypeServiceMock = new Mock<IMediaTypeService>();
-                mediaTypeServiceMock.Setup(x => x.GetAll())
-                    .Returns(new List<IMediaType>
-                    {
-                        new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"},
-                        new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"}
-                    });
-                mediaTypeService = mediaTypeServiceMock.Object;
-            }
-
-            // fixme oops?!
-            //var query = new Mock<IQuery<IContent>>();
-            //query
-            //    .Setup(x => x.GetWhereClauses())
-            //    .Returns(new List<Tuple<string, object[]>> { new Tuple<string, object[]>($"{Constants.DatabaseSchema.Tables.Document}.published", new object[] { 1 }) });
+            if (validator == null)
+                validator = new ContentValueSetValidator(true);
             
-            //scopeProvider
-            //    .Setup(x => x.Query<IContent>())
-            //    .Returns(query.Object);
-
             var i = new UmbracoContentIndexer(
                 "testIndexer",
-                Enumerable.Empty<FieldDefinition>(),
+                UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
                 luceneDir,
                 analyzer,
                 profilingLogger,
-                contentService,
-                mediaService,
-                userService,
-                sqlContext,
-                new[] {new DefaultUrlSegmentProvider()},
-                new UmbracoContentValueSetValidator(options, Mock.Of<IPublicAccessService>()),
-                options);
+                languageService,
+                validator);
 
             i.IndexingError += IndexingError;
 
             return i;
         }
 
-        public static LuceneSearcher GetLuceneSearcher(Directory luceneDir)
-        {
-            return new LuceneSearcher("testSearcher", luceneDir, new StandardAnalyzer(Version.LUCENE_29));
-        }
-
-        public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir)
-        {
-            var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
-            return i;
-        }
+        //public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir)
+        //{
+        //    var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
+        //    return i;
+        //}
 
 
         internal static void IndexingError(object sender, IndexingErrorEventArgs e)

src/Umbraco.Tests/UmbracoExamine/IndexTest.cs

@@ -8,6 +8,13 @@ using Lucene.Net.Store;
 using NUnit.Framework;
 using Umbraco.Tests.Testing;
 using Umbraco.Examine;
+using Umbraco.Core.PropertyEditors;
+using LightInject;
+using Umbraco.Tests.TestHelpers.Entities;
+using Umbraco.Core.Models;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using System;
 
 namespace Umbraco.Tests.UmbracoExamine
 {
@@ -21,16 +28,116 @@ namespace Umbraco.Tests.UmbracoExamine
     {
 
         [Test]
-        public void Rebuild_Index()
+        public void Index_Property_Data_With_Value_Indexer()
         {
+            var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Container.GetInstance<PropertyEditorCollection>(), false);
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(false)))
             using (indexer.ProcessNonAsync())
             {
+                indexer.CreateIndex();
+
+                var contentType = MockedContentTypes.CreateBasicContentType();
+                contentType.AddPropertyType(new PropertyType("test", ValueStorageType.Ntext)
+                {
+                    Alias = "grid",
+                    Name = "Grid",
+                    PropertyEditorAlias = Core.Constants.PropertyEditors.Aliases.Grid
+                });
+                var content = MockedContent.CreateBasicContent(contentType);
+                content.Id = 555;
+                content.Path = "-1,555";
+                var gridVal = new GridValue
+                {
+                    Name = "n1",
+                    Sections = new List<GridValue.GridSection>
+                    {
+                        new GridValue.GridSection
+                        {
+                            Grid = "g1",
+                            Rows = new List<GridValue.GridRow>
+                            {
+                                new GridValue.GridRow
+                                {
+                                    Id = Guid.NewGuid(),
+                                    Name = "row1",
+                                    Areas = new List<GridValue.GridArea>
+                                    {
+                                        new GridValue.GridArea
+                                        {
+                                            Grid = "g2",
+                                            Controls = new List<GridValue.GridControl>
+                                            {
+                                                new GridValue.GridControl
+                                                {
+                                                    Editor = new GridValue.GridEditor
+                                                    {
+                                                        Alias = "editor1",
+                                                        View = "view1"
+                                                    },
+                                                    Value = "value1"
+                                                },
+                                                new GridValue.GridControl
+                                                {
+                                                    Editor = new GridValue.GridEditor
+                                                    {
+                                                        Alias = "editor1",
+                                                        View = "view1"
+                                                    },
+                                                    Value = "value2"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                };
+
+                var json = JsonConvert.SerializeObject(gridVal);
+                content.Properties["grid"].SetValue(json);
+
+                var valueSet = contentValueSetBuilder.GetValueSets(content);
+                indexer.IndexItems(valueSet);
+
+                var searcher = indexer.GetSearcher();
+
+                var results = searcher.Search(searcher.CreateCriteria().Id(555).Compile());
+                Assert.AreEqual(1, results.TotalItemCount);
+
+                var result = results.First();
+                Assert.IsTrue(result.Values.ContainsKey("grid.row1"));
+                Assert.AreEqual("value1", result.AllValues["grid.row1"][0]);
+                Assert.AreEqual("value2", result.AllValues["grid.row1"][1]);
+                Assert.IsTrue(result.Values.ContainsKey("grid"));
+                Assert.AreEqual("value1 value2 ", result["grid"]);
+                Assert.IsTrue(result.Values.ContainsKey($"{UmbracoExamineIndexer.RawFieldPrefix}grid"));
+                Assert.AreEqual(json, result[$"{UmbracoExamineIndexer.RawFieldPrefix}grid"]);
+            }
+        }
+
+        [Test]
+        public void Rebuild_Index()
+        {
+            var contentRebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext, false);
+            var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
+
+            using (var luceneDir = new RandomIdRamDirectory())
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(false)))
+            using (indexer.ProcessNonAsync())
+            {
+                contentRebuilder.RegisterIndex(indexer.Name);
+                mediaRebuilder.RegisterIndex(indexer.Name);
+
                 var searcher = indexer.GetSearcher();
 
                 //create the whole thing
-                indexer.RebuildIndex();
+                contentRebuilder.Populate(indexer);
+                mediaRebuilder.Populate(indexer);
 
                 var result = searcher.Search(searcher.CreateCriteria().All().Compile());
 
@@ -45,23 +152,27 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Index_Protected_Content_Not_Indexed()
         {
+            var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext, false);
+
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
             using (indexer.ProcessNonAsync())
             using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher())
             {
                 //create the whole thing
-                indexer.RebuildIndex();
+                rebuilder.Populate(indexer);
+
 
                 var protectedQuery = new BooleanQuery();
                 protectedQuery.Add(
                     new BooleanClause(
-                        new TermQuery(new Term(LuceneIndexer.CategoryFieldName, IndexTypes.Content)),
+                        new TermQuery(new Term(LuceneIndex.CategoryFieldName, IndexTypes.Content)),
                         Occur.MUST));
 
                 protectedQuery.Add(
                     new BooleanClause(
-                        new TermQuery(new Term(LuceneIndexer.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
+                        new TermQuery(new Term(LuceneIndex.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
                         Occur.MUST));
 
                 var collector = TopScoreDocCollector.Create(100, true);
@@ -76,10 +187,11 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
         {
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
                 //make parent id 1116
-                options: new UmbracoContentIndexerOptions(false, false, 1116)))
+                validator: new ContentValueSetValidator(false, 1116)))
             using (indexer.ProcessNonAsync())
             {
                 var searcher = indexer.GetSearcher();
@@ -95,7 +207,7 @@ namespace Umbraco.Tests.UmbracoExamine
                 Assert.AreEqual("-1,1111,2222,2112", currPath);
 
                 //ensure it's indexed
-                indexer.IndexItems(new []{ node.ConvertToValueSet(IndexTypes.Media) });
+                indexer.IndexItem(node.ConvertToValueSet(IndexTypes.Media));
 
                 //it will not exist because it exists under 2222
                 var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile());
@@ -119,9 +231,9 @@ namespace Umbraco.Tests.UmbracoExamine
         public void Index_Move_Media_To_Non_Indexable_ParentID()
         {
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
+            using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
                 //make parent id 2222
-                options: new UmbracoContentIndexerOptions(false, false, 2222)))
+                validator: new ContentValueSetValidator(false, 2222)))
             using (indexer1.ProcessNonAsync())
             {
                 var searcher = indexer1.GetSearcher();
@@ -137,7 +249,7 @@ namespace Umbraco.Tests.UmbracoExamine
                 Assert.AreEqual("-1,1111,2222,2112", currPath);
 
                 //ensure it's indexed
-                indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) });
+                indexer1.IndexItem(node.ConvertToValueSet(IndexTypes.Media));
 
 
 
@@ -168,17 +280,20 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Index_Reindex_Content()
         {
+            var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext, false);
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
+                validator: new ContentValueSetValidator(false)))
             using (indexer.ProcessNonAsync())
             {
+                rebuilder.RegisterIndex(indexer.Name);
+
                 var searcher = indexer.GetSearcher();
 
                 //create the whole thing
-                indexer.RebuildIndex();
+                rebuilder.Populate(indexer);
 
-
-                var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
+                var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Compile());
                 Assert.AreEqual(21, result.TotalItemCount);
 
                 //delete all content
@@ -189,15 +304,13 @@ namespace Umbraco.Tests.UmbracoExamine
 
 
                 //ensure it's all gone
-                result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
+                result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Compile());
                 Assert.AreEqual(0, result.TotalItemCount);
 
                 //call our indexing methods
-                indexer.IndexAll(IndexTypes.Content);
+                rebuilder.Populate(indexer);
 
-
-
-                result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
+                result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Compile());
                 Assert.AreEqual(21, result.TotalItemCount);
             }
         }
@@ -208,15 +321,17 @@ namespace Umbraco.Tests.UmbracoExamine
         [Test]
         public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
         {
+
+            var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext, false);
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
             using (indexer.ProcessNonAsync())
             {
                 var searcher = indexer.GetSearcher();
 
                 //create the whole thing
-                indexer.RebuildIndex();
-
+                rebuilder.Populate(indexer);
 
                 //now delete a node that has children
 

src/Umbraco.Tests/UmbracoExamine/SearchTests.cs

@@ -1,17 +1,19 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using LightInject;
 using Examine;
-using Lucene.Net.Store;
 using NUnit.Framework;
 using Examine.LuceneEngine.SearchCriteria;
 using Moq;
 using Umbraco.Core.Models;
-using Umbraco.Core.Persistence.DatabaseModelDefinitions;
+using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Services;
 using Umbraco.Examine;
 using Umbraco.Tests.Testing;
+using Umbraco.Core.PropertyEditors;
+using Umbraco.Core.Strings;
 
 namespace Umbraco.Tests.UmbracoExamine
 {
@@ -53,13 +55,17 @@ namespace Umbraco.Tests.UmbracoExamine
                     ==
                     allRecs);
 
+            var propertyEditors = Container.GetInstance<PropertyEditorCollection>();
+            var rebuilder = IndexInitializer.GetContentIndexRebuilder(propertyEditors, contentService, ScopeProvider.SqlContext, true);
+
             using (var luceneDir = new RandomIdRamDirectory())
-            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, contentService: contentService))
+            using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
             using (indexer.ProcessNonAsync())
             {
-                indexer.RebuildIndex();
+                rebuilder.RegisterIndex(indexer.Name);
+                indexer.CreateIndex();
+                rebuilder.Populate(indexer);
                 
-
                 var searcher = indexer.GetSearcher();
 
                 var numberSortedCriteria = searcher.CreateCriteria()
@@ -80,12 +86,12 @@ namespace Umbraco.Tests.UmbracoExamine
             }
         }
 
-        private bool IsSortedByNumber(IEnumerable<SearchResult> results)
+        private bool IsSortedByNumber(IEnumerable<ISearchResult> results)
         {
             var currentSort = 0;
             foreach (var searchResult in results)
             {
-                var sort = int.Parse(searchResult.Fields["sortOrder"]);
+                var sort = int.Parse(searchResult.Values["sortOrder"]);
                 if (currentSort >= sort)
                 {
                     return false;

src/Umbraco.Tests/UmbracoExamine/TestDataService.cs

@@ -1,32 +0,0 @@
-//using System.IO;
-//using Umbraco.Tests.TestHelpers;
-//using UmbracoExamine.DataServices;
-
-//namespace Umbraco.Tests.UmbracoExamine
-//{
-//    public class TestDataService : IDataService
-//    {
-
-//        public TestDataService()
-//        {
-//            ContentService = new TestContentService();
-//            LogService = new TestLogService();
-//            MediaService = new TestMediaService();
-//        }
-
-//        #region IDataService Members
-
-//        public IContentService ContentService { get; internal set; }
-
-//        public ILogService LogService { get; internal set; }
-
-//        public IMediaService MediaService { get; internal set; }
-
-//        public string MapPath(string virtualPath)
-//        {
-//            return new DirectoryInfo(TestHelper.CurrentAssemblyDirectory) + "\\" + virtualPath.Replace("/", "\\");
-//        }
-
-//        #endregion
-//    }
-//}

src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs

@@ -0,0 +1,300 @@
+using Examine;
+using NUnit.Framework;
+using Umbraco.Examine;
+using Moq;
+using Umbraco.Core.Services;
+using System.Collections.Generic;
+using Umbraco.Core;
+using Umbraco.Core.Models;
+using System;
+using System.Linq;
+
+namespace Umbraco.Tests.UmbracoExamine
+{
+    [TestFixture]
+    public class UmbracoContentValueSetValidatorTests
+    {
+        [Test]
+        public void Invalid_Category()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);            
+
+            result = validator.Validate(ValueSet.FromObject("777", IndexTypes.Media, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", "invalid-category", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+        }
+
+        [Test]
+        public void Must_Have_Path()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Parent_Id()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>(), 555);
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,444" }));
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777,999" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Inclusion_Field_List()
+        {
+            var validator = new ValueSetValidator(null, null,
+                new[] { "hello", "world" },
+                null);
+
+            var valueSet = ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555", world = "your oyster" });
+            var result = validator.Validate(valueSet);
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            Assert.IsFalse(valueSet.Values.ContainsKey("path"));
+            Assert.IsTrue(valueSet.Values.ContainsKey("hello"));
+            Assert.IsTrue(valueSet.Values.ContainsKey("world"));
+        }
+
+        [Test]
+        public void Exclusion_Field_List()
+        {
+            var validator = new ValueSetValidator(null, null,
+                null,
+                new[] { "hello", "world" });
+
+            var valueSet = ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555", world = "your oyster" });
+            var result = validator.Validate(valueSet);
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            Assert.IsTrue(valueSet.Values.ContainsKey("path"));
+            Assert.IsFalse(valueSet.Values.ContainsKey("hello"));
+            Assert.IsFalse(valueSet.Values.ContainsKey("world"));
+        }
+
+        [Test]
+        public void Inclusion_Exclusion_Field_List()
+        {
+            var validator = new ValueSetValidator(null, null,
+                new[] { "hello", "world" },
+                new[] { "world" });
+
+            var valueSet = ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555", world = "your oyster" });
+            var result = validator.Validate(valueSet);
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            Assert.IsFalse(valueSet.Values.ContainsKey("path"));
+            Assert.IsTrue(valueSet.Values.ContainsKey("hello"));
+            Assert.IsFalse(valueSet.Values.ContainsKey("world"));
+        }
+
+        [Test]
+        public void Inclusion_Type_List()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>(),
+                includeItemTypes: new List<string> { "include-content" });
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Exclusion_Type_List()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>(),
+                excludeItemTypes: new List<string> { "exclude-content" });
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+        }
+
+        [Test]
+        public void Inclusion_Exclusion_Type_List()
+        {
+            var validator = new ContentValueSetValidator(false, true, Mock.Of<IPublicAccessService>(),
+                includeItemTypes: new List<string> { "include-content", "exclude-content" },
+                excludeItemTypes: new List<string> { "exclude-content" });
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Recycle_Bin_Content()
+        {
+            var validator = new ContentValueSetValidator(true, false, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555,777" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoExamineIndexer.PublishedFieldName] = 1
+                }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Recycle_Bin_Media()
+        {
+            var validator = new ContentValueSetValidator(true, false, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Media, new { hello = "world", path = "-1,-21,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Media, new { hello = "world", path = "-1,-21,555,777" }));
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Media, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+        }
+
+        [Test]
+        public void Published_Only()
+        {
+            var validator = new ContentValueSetValidator(true, true, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoExamineIndexer.PublishedFieldName] = 0
+                }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoExamineIndexer.PublishedFieldName] = 1
+                }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+
+        [Test]
+        public void Published_Only_With_Variants()
+        {
+            var validator = new ContentValueSetValidator(true, true, Mock.Of<IPublicAccessService>());
+
+            var result = validator.Validate(new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoContentIndexer.VariesByCultureFieldName] = 1,
+                    [UmbracoExamineIndexer.PublishedFieldName] = 0
+                }));
+            Assert.AreEqual(ValueSetValidationResult.Failed, result);
+
+            result = validator.Validate(new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoContentIndexer.VariesByCultureFieldName] = 1,
+                    [UmbracoExamineIndexer.PublishedFieldName] = 1
+                }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+
+            var valueSet = new ValueSet("555", IndexTypes.Content,
+                new Dictionary<string, object>
+                {
+                    ["hello"] = "world",
+                    ["path"] = "-1,555",
+                    [UmbracoContentIndexer.VariesByCultureFieldName] = 1,
+                    [$"{UmbracoExamineIndexer.PublishedFieldName}_en-us"] = 1,
+                    ["hello_en-us"] = "world",
+                    ["title_en-us"] = "my title",
+                    [$"{UmbracoExamineIndexer.PublishedFieldName}_es-es"] = 0,
+                    ["hello_es-ES"] = "world",
+                    ["title_es-ES"] = "my title",
+                    [UmbracoExamineIndexer.PublishedFieldName] = 1
+                });
+            Assert.AreEqual(10, valueSet.Values.Count());
+            Assert.IsTrue(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es"));
+            Assert.IsTrue(valueSet.Values.ContainsKey("hello_es-ES"));
+            Assert.IsTrue(valueSet.Values.ContainsKey("title_es-ES"));
+
+            result = validator.Validate(valueSet);
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            Assert.AreEqual(7, valueSet.Values.Count()); //filtered to 7 values (removes es-es values)
+            Assert.IsFalse(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es"));
+            Assert.IsFalse(valueSet.Values.ContainsKey("hello_es-ES"));
+            Assert.IsFalse(valueSet.Values.ContainsKey("title_es-ES"));
+        }
+
+        [Test]
+        public void Non_Protected()
+        {
+            var publicAccessService = new Mock<IPublicAccessService>();
+            publicAccessService.Setup(x => x.IsProtected("-1,555"))
+                .Returns(Attempt.Succeed(new PublicAccessEntry(Guid.NewGuid(), 555, 444, 333, Enumerable.Empty<PublicAccessRule>())));
+            publicAccessService.Setup(x => x.IsProtected("-1,777"))
+                .Returns(Attempt.Fail<PublicAccessEntry>());
+            var validator = new ContentValueSetValidator(false, false, publicAccessService.Object);
+
+            var result = validator.Validate(ValueSet.FromObject("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
+            Assert.AreEqual(ValueSetValidationResult.Filtered, result);
+
+            result = validator.Validate(ValueSet.FromObject("777", IndexTypes.Content, new { hello = "world", path = "-1,777" }));
+            Assert.AreEqual(ValueSetValidationResult.Valid, result);
+        }
+    }
+}

src/Umbraco.Tests/Web/Mvc/SurfaceControllerTests.cs

@@ -125,12 +125,10 @@ namespace Umbraco.Tests.Web.Mvc
                                                          //return mock of IPublishedContent for any call to GetById
                                                          Mock.Of<IPublishedContent>(content => content.Id == 2)),
                 Mock.Of<ITagQuery>(),
-                Mock.Of<IDataTypeService>(),
                 Mock.Of<ICultureDictionary>(),
                 Mock.Of<IUmbracoComponentRenderer>(),
                 new MembershipHelper(umbracoContext, Mock.Of<MembershipProvider>(), Mock.Of<RoleProvider>()),
-                new ServiceContext(),
-                CacheHelper.CreateDisabledCacheHelper());
+                new ServiceContext());
 
             var ctrl = new TestSurfaceController { UmbracoContext = umbracoContext, Umbraco = helper };
             var result = ctrl.GetContent(2) as PublishedContentResult;

src/Umbraco.Web.UI.Client/package.json

@@ -44,7 +44,7 @@
     "autoprefixer": "9.3.1",
     "gulp-clean-css": "4.0.0",
     "cssnano": "4.1.7",
-    "gulp-connect": "5.6.1",
+    "gulp": "3.9.1",
     "gulp-babel": "8.0.0",
     "gulp-concat": "2.6.1",
     "gulp-connect": "5.6.1",
@@ -60,14 +60,13 @@
     "gulp-wrap": "0.14.0",
     "gulp-wrap-js": "0.4.1",
     "jasmine-core": "3.3.0",
-    "karma": "^3.1.3",
+    "karma": "3.1.1",
     "karma-jasmine": "2.0.1",
     "karma-phantomjs-launcher": "1.0.4",
-    "less": "^3.9.0",
+    "less": "3.9.0",
     "lodash": "4.17.11",
     "merge-stream": "1.0.1",
-    "run-sequence": "^2.2.1",
-    "marked": "^0.5.2",
-    "event-stream": "3.3.4"
+    "run-sequence": "2.2.1",
+    "marked": "^0.5.2"
   }
 }

src/Umbraco.Web.UI.Client/src/views/dashboard/developer/developerdashboardintro.html

@@ -1,14 +0,0 @@
-<h3>Start here</h3>
-<h4>This section contains the tools to add advanced features to your Umbraco site</h4>
-    <p>From here you can explore and install packages, create macros, add data types, and much more. Start by exploring the below links or videos.</p>
-
-<h4>Find out more:</h4>
-
-<ul>
-    <li>Find the answers to your Umbraco questions in our <a class="btn-link -underline href="https://our.umbraco.com/documentation/" target="_blank">Documentation</a></li>
-    <li>Ask a question in the <a class="btn-link -underline href="https://our.umbraco.com/" target="_blank">Community Forum</a></li>
-    <li>Find an add-on <a class="btn-link -underline href="https://our.umbraco.com/projects" target="_blank">package</a> to help you get going quickly</li>
-    <li>Watch our <a class="btn-link -underline href="https://umbraco.tv" target="_blank">tutorial videos</a> (some are free, some require a subscription)</li>
-    <li>Find out about our <a class="btn-link -underline href="https://umbraco.com/products" target="_blank">productivity boosting tools and commercial support</a></li>
-    <li>Find out about real-life <a class="btn-link -underline href="https://umbraco.com/training/" target="_blank">training and certification</a> opportunities</li>
-</ul>

src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js

@@ -0,0 +1,187 @@
+function ExamineManagementController($scope, umbRequestHelper, $http, $q, $timeout) {
+
+    var vm = this;
+
+    vm.indexerDetails = [];
+    vm.searcherDetails = [];
+    vm.loading = true;
+    vm.viewState = "list";
+    vm.selectedIndex = null;
+    vm.selectedSearcher = null;
+    vm.searchResults = null;
+
+    vm.showSearchResultDialog = showSearchResultDialog;
+    vm.showIndexInfo = showIndexInfo;
+    vm.showSearcherInfo = showSearcherInfo;
+    vm.search = search;
+    vm.toggle = toggle;
+    vm.rebuildIndex = rebuildIndex;
+    vm.setViewState = setViewState;
+    vm.nextSearchResultPage = nextSearchResultPage;
+    vm.prevSearchResultPage = prevSearchResultPage;
+    vm.goToPageSearchResultPage = goToPageSearchResultPage;
+
+    vm.infoOverlay = null;
+
+    function showSearchResultDialog(values) {
+        if (vm.searchResults) {
+            vm.searchResults.overlay = {
+                title: "Field values",
+                searchResultValues: values,
+                view: "views/dashboard/settings/examinemanagementresults.html",
+                close: function () {
+                    vm.searchResults.overlay = null;
+                }
+            };
+        }
+    }
+
+    function nextSearchResultPage(pageNumber) {
+        search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber);
+    }
+    function prevSearchResultPage(pageNumber) {
+        search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber);
+    }
+    function goToPageSearchResultPage(pageNumber) {
+        search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber);
+    }
+
+    function setViewState(state) {
+        vm.searchResults = null;
+        vm.viewState = state;
+    }
+
+    function showIndexInfo(index) {
+        vm.selectedIndex = index;
+        setViewState("index-details");
+    }
+
+    function showSearcherInfo(searcher) {
+        vm.selectedSearcher = searcher;
+        setViewState("searcher-details");
+    }
+
+    function checkProcessing(index, checkActionName) {
+        umbRequestHelper.resourcePromise(
+                $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
+                    checkActionName,
+                    { indexName: index.name })),
+                'Failed to check index processing')
+            .then(function(data) {
+
+                if (data !== null && data !== "null") {
+
+                    //copy all resulting properties
+                    for (var k in data) {
+                        index[k] = data[k];
+                    }
+                    index.isProcessing = false;
+                } else {
+                    $timeout(() => {
+                            //don't continue if we've tried 100 times
+                            if (index.processingAttempts < 100) {
+                                checkProcessing(index, checkActionName);
+                                //add an attempt
+                                index.processingAttempts++;
+                            } else {
+                                //we've exceeded 100 attempts, stop processing
+                                index.isProcessing = false;
+                            }
+                        },
+                        1000);
+                }
+            });
+    }
+
+    function search(searcher, e, pageNumber) {
+
+        //deal with accepting pressing the enter key
+        if (e && e.keyCode !== 13) {
+            return;
+        }
+
+        if (!searcher) {
+            throw "searcher parameter is required";
+        }
+
+        searcher.isProcessing = true;
+
+        umbRequestHelper.resourcePromise(
+                $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
+                    "GetSearchResults",
+                    {
+                        searcherName: searcher.name,
+                        query: encodeURIComponent(vm.searchText),
+                        pageIndex: pageNumber ? (pageNumber - 1) : 0
+                    })),
+                'Failed to search')
+            .then(searchResults => {
+                searcher.isProcessing = false;
+                vm.searchResults = searchResults
+                vm.searchResults.pageNumber = pageNumber ? pageNumber : 1;
+                //20 is page size
+                vm.searchResults.totalPages = Math.ceil(vm.searchResults.totalRecords / 20);
+            });
+    }
+
+    function toggle(provider, propName) {
+        if (provider[propName] !== undefined) {
+            provider[propName] = !provider[propName];
+        } else {
+            provider[propName] = true;
+        }
+    }
+
+    function rebuildIndex(index) {
+        if (confirm("This will cause the index to be rebuilt. " +
+            "Depending on how much content there is in your site this could take a while. " +
+            "It is not recommended to rebuild an index during times of high website traffic " +
+            "or when editors are editing content.")) {
+
+            index.isProcessing = true;
+            index.processingAttempts = 0;
+
+            umbRequestHelper.resourcePromise(
+                    $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
+                        "PostRebuildIndex",
+                        { indexName: index.name })),
+                    'Failed to rebuild index')
+                .then(function() {
+
+                    //rebuilding has started, nothing is returned accept a 200 status code.
+                    //lets poll to see if it is done.
+                    $timeout(() => { checkProcessing(index, "PostCheckRebuildIndex"), 1000 });
+
+                });
+        }
+    }
+
+    function init() {
+        //go get the data
+
+        //combine two promises and execute when they are both done
+        $q.all([
+
+                //get the indexer details
+                umbRequestHelper.resourcePromise(
+                    $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", "GetIndexerDetails")),
+                    'Failed to retrieve indexer details')
+                .then(function (data) {
+                    vm.indexerDetails = data;
+                }),
+
+                //get the searcher details
+                umbRequestHelper.resourcePromise(
+                    $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", "GetSearcherDetails")),
+                    'Failed to retrieve searcher details')
+                .then(data => {
+                    vm.searcherDetails = data;
+                })
+            ])
+            .then(() => { vm.loading = false });
+    }
+
+    init();
+}
+
+angular.module("umbraco").controller("Umbraco.Dashboard.ExamineManagementController", ExamineManagementController);

src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html

@@ -1,286 +1,411 @@
-<div id="examineManagement" ng-controller="Umbraco.Dashboard.ExamineMgmtController">
+<div id="examineManagement" ng-controller="Umbraco.Dashboard.ExamineManagementController as vm">
 
-    <umb-box>
-        <umb-box-content>
-            <h3 class="bold">Examine Management</h3>
-        </umb-box-content>
-    </umb-box>
+    <div ng-if="vm.viewState === 'list'">
+        <umb-box>
+            <umb-box-content>
+                <h3 class="bold">Examine Management</h3>
+            </umb-box-content>
+        </umb-box>
+    </div>
 
-    <div ng-show="loading">
+    <div ng-show="vm.loading">
         <umb-load-indicator></umb-load-indicator>
     </div>
 
-    <div ng-hide="loading" class="umb-healthcheck-group__details">
+    <div ng-hide="vm.loading">
+        <div ng-if="vm.viewState === 'list'" class="umb-healthcheck-group__details">
 
-        <div class="umb-healthcheck-group__details-group">
+            <div class="umb-healthcheck-group__details-group">
 
-            <div class="umb-healthcheck-group__details-group-title">
-                <div class="umb-healthcheck-group__details-group-name">Indexers</div>
-            </div>
+                <div class="umb-healthcheck-group__details-group-title">
+                    <div class="umb-healthcheck-group__details-group-name">Indexers</div>
+                </div>
 
-            <div class="umb-healthcheck-group__details-checks">
-                <div class="umb-healthcheck-group__details-check">
-                    <div class="umb-healthcheck-group__details-check-title">
-                        <div class="umb-healthcheck-group__details-check-name">Manage Examine's indexes</div>
-                        <div class="umb-healthcheck-group__details-check-description">Allows you to view the details of each index and provides some tools for managing the indexes</div>
-                    </div>
-
-                    <div class="umb-healthcheck-group__details-status" ng-repeat="indexer in indexerDetails">
-
-                        <div class="umb-healthcheck-group__details-status-icon-container">
-                            <i class="umb-healthcheck-status-icon" ng-class="{'icon-check color-green' : indexer.isHealthy, 'icon-delete color-red' : !indexer.isHealthy}"></i>
+                <div class="umb-healthcheck-group__details-checks">
+                    <div class="umb-healthcheck-group__details-check">
+                        <div class="umb-healthcheck-group__details-check-title">
+                            <div class="umb-healthcheck-group__details-check-name">Manage Examine's indexes</div>
+                            <div class="umb-healthcheck-group__details-check-description">Allows you to view the details of each index and provides some tools for managing the indexes</div>
                         </div>
 
-                        <div class="umb-healthcheck-group__details-status-content">
-                            <div class="umb-healthcheck-group__details-status-text">
-                                <div ng-show="!indexer.isHealthy">
-                                    {{indexer.name}}
-                                </div>
-                                <a class="btn-link -underline" href="" ng-click="toggle(indexer, 'showProperties')" ng-show="indexer.isHealthy">
-                                    {{indexer.name}}
-                                </a>
-                                <div ng-if="!indexer.isHealthy" class="text-error">
-                                    The index cannot be read and will need to be rebuilt
+                        <div class="umb-healthcheck-group__details-status" ng-repeat="indexer in vm.indexerDetails">
+
+                            <div class="umb-healthcheck-group__details-status-icon-container">
+                                <i class="umb-healthcheck-status-icon"
+                                   ng-class="{'icon-check color-green' : indexer.isHealthy, 'icon-delete color-red' : !indexer.isHealthy}"></i>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status-content">
+                                <div class="umb-healthcheck-group__details-status-text">
+                                    <a class="btn-link -underline" href="" ng-click="vm.showIndexInfo(indexer)">
+                                        {{indexer.name}}
+                                    </a>
                                 </div>
                             </div>
 
-                            <div class="umb-healthcheck-group__details-status-actions" ng-if="!indexer.isHealthy">
-                                <div class="umb-healthcheck-group__details-status-action">
-                                    <umb-button
-                                        ng-if="!indexer.isProcessing && (!indexer.processingAttempts || indexer.processingAttempts < 100)"
-                                        type="button"
-                                        button-style="success"
-                                        action="rebuildIndex(indexer)"
-                                        label="Rebuild index">
-                                    </umb-button>
-                                </div>
-                            </div>
-
-                            <div class="umb-healthcheck-group__details-status-actions" ng-show="indexer.isHealthy && indexer.showProperties">
-                                <ul>
-                                    <li>
-                                        <a href="" ng-click="toggle(indexer, 'showTools')">Index info & tools</a>
-
-                                        <div ng-show="indexer.showTools && indexer.isLuceneIndex">
-                                            <div>
-                                                <br />
-
-                                                <div ng-show="!indexer.isProcessing && (!indexer.processingAttempts || indexer.processingAttempts < 100)"
-                                                    class="umb-healthcheck-group__details-status-action">
-                                                    <umb-button type="button" button-style="success" action="rebuildIndex(indexer)" label="Rebuild index"></umb-button>
-                                                </div>
-
-                                                <div ng-show="indexer.processingAttempts >= 100">
-                                                    The process is taking longer than expected, check the umbraco log to see if there have been any errors during this operation
-                                                </div>
-                                            </div>
-                                            <table class="table table-bordered table-condensed">
-                                                <caption>&nbsp;</caption>
-                                                <tr>
-                                                    <th>Documents in index</th>
-                                                    <td>{{indexer.documentCount}}</td>
-                                                </tr>
-                                                <tr>
-                                                    <th>Fields in index</th>
-                                                    <td>{{indexer.fieldCount}}</td>
-                                                </tr>                                            
-                                            </table>
-                                        </div>
-                                    </li>
-                                    <li ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0 || indexer.indexCriteria.ExcludeNodeTypes.length > 0 || indexer.indexCriteria.ParentNodeId">
-                                        <a href="" ng-click="toggle(indexer, 'showNodeTypes')">Node types</a>
-                                        <table ng-show="indexer.showNodeTypes" class="table table-bordered table-condensed">
-                                            <tr ng-show="indexer.indexCriteria.IncludeNodeTypes.length > 0">
-                                                <th>Include node types</th>
-                                                <td>{{indexer.indexCriteria.IncludeNodeTypes | json}}</td>
-                                            </tr>
-                                            <tr ng-show="indexer.indexCriteria.ExcludeNodeTypes.length > 0">
-                                                <th>Exclude node types</th>
-                                                <td>{{indexer.indexCriteria.ExcludeNodeTypes | json}}</td>
-                                            </tr>
-                                            <tr ng-show="indexer.indexCriteria.ParentNodeId">
-                                                <th>Parent node id</th>
-                                                <td>{{indexer.indexCriteria.ParentNodeId}}</td>
-                                            </tr>
-                                        </table>
-                                    </li>
-                                    <li ng-show="indexer.indexCriteria.StandardFields.length > 0">
-                                        <a href="" ng-click="toggle(indexer, 'showSystemFields')">System fields</a>
-                                        <table ng-show="indexer.showSystemFields" class="table table-bordered table-condensed">
-                                            <caption>&nbsp;</caption>
-                                            <thead>
-                                                <tr>
-                                                    <th>Name</th>
-                                                    <th>Enable sorting</th>
-                                                    <th>Type</th>
-                                                </tr>
-                                            </thead>
-                                            <tbody>
-                                                <tr ng-repeat="field in indexer.indexCriteria.StandardFields">
-                                                    <th>{{field.Name}}</th>
-                                                    <td>{{field.EnableSorting}}</td>
-                                                    <td>{{field.Type}}</td>
-                                                </tr>
-                                            </tbody>
-                                        </table>
-                                    </li>
-                                    <li ng-show="indexer.indexCriteria.UserFields.length > 0">
-                                        <a href="" ng-click="toggle(indexer, 'showUserFields')">User fields</a>
-                                        <table ng-show="indexer.showUserFields" class="table table-bordered table-condensed">
-                                            <caption>&nbsp;</caption>
-                                            <thead>
-                                                <tr>
-                                                    <th>Name</th>
-                                                    <th>Enable sorting</th>
-                                                    <th>Type</th>
-                                                </tr>
-                                            </thead>
-                                            <tbody>
-                                                <tr ng-repeat="field in indexer.indexCriteria.UserFields">
-                                                    <th>{{field.Name}}</th>
-                                                    <td>{{field.EnableSorting}}</td>
-                                                    <td>{{field.Type}}</td>
-                                                </tr>
-                                            </tbody>
-                                        </table>
-                                    </li>
-                                    <li>
-                                        <a href="" ng-click="toggle(indexer, 'showProviderProperties')">Provider properties</a>
-                                        <table ng-show="indexer.showProviderProperties" class="table table-bordered table-condensed">
-                                            <caption>&nbsp;</caption>
-                                            <tr ng-repeat="(key, val) in indexer.providerProperties track by $index">
-                                                <th>{{key}}</th>
-                                                <td>{{val}}</td>
-                                            </tr>
-                                        </table>
-                                    </li>
-                                </ul>
-                            </div>
-                        </div>
-
-                        <div ng-show="indexer.isProcessing">
-                            <div class="umb-healthcheck-group__details-status-overlay"></div>
-                            <umb-load-indicator></umb-load-indicator>
                         </div>
                     </div>
                 </div>
+
+            </div>
+
+            <br />
+
+            <div class="umb-healthcheck-group__details-group">
+
+                <div class="umb-healthcheck-group__details-group-title">
+                    <div class="umb-healthcheck-group__details-group-name">Searchers</div>
+                </div>
+
+                <div class="umb-healthcheck-group__details-checks">
+                    <div class="umb-healthcheck-group__details-check">
+                        <div class="umb-healthcheck-group__details-check-title">
+                            <div class="umb-healthcheck-group__details-check-name">Configured Searchers</div>
+                            <div class="umb-healthcheck-group__details-check-description">Shows properties and tools for any configured Searcher (i.e. such as a multi-index searcher)</div>
+                        </div>
+
+                        <div class="umb-healthcheck-group__details-status" ng-repeat="searcher in vm.searcherDetails">
+
+                            <div class="umb-healthcheck-group__details-status-icon-container">
+                                <i class="umb-healthcheck-status-icon icon-info"></i>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status-content">
+                                <div class="umb-healthcheck-group__details-status-text">
+                                    <a class="btn-link -underline" href="" ng-click="vm.showSearcherInfo(searcher)">
+                                        {{searcher.name}}
+                                    </a>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+
             </div>
 
         </div>
 
-        <br />
+        <div ng-if="vm.viewState === 'searcher-details'">
 
-        <div class="umb-healthcheck-group__details-group">
+            <umb-editor-sub-header>
+                <umb-editor-sub-header-content-left>
+                    <a class="umb-healthcheck-back-link" href="" ng-click="vm.setViewState('list');">&larr; Back to overview</a>
+                </umb-editor-sub-header-content-left>
+            </umb-editor-sub-header>
 
-            <div class="umb-healthcheck-group__details-group-title">
-                <div class="umb-healthcheck-group__details-group-name">Searchers</div>
-            </div>
+            <div class="umb-healthcheck-group__details">
 
-            <div class="umb-healthcheck-group__details-checks">
-                <div class="umb-healthcheck-group__details-check">
-                    <div class="umb-healthcheck-group__details-check-title">
-                        <div class="umb-healthcheck-group__details-check-name">Search indexes</div>
-                        <div class="umb-healthcheck-group__details-check-description">Allows you to search the indexes and view the searcher properties</div>
+                <div class="umb-healthcheck-group__details-group">
+
+                    <div class="umb-healthcheck-group__details-group-title">
+                        <div class="umb-healthcheck-group__details-group-name">{{ vm.selectedSearcher.name }}</div>
                     </div>
 
-                    <div class="umb-healthcheck-group__details-status" ng-repeat="searcher in searcherDetails">
+                    <div class="umb-healthcheck-group__details-checks">
 
-                        <div class="umb-healthcheck-group__details-status-icon-container">
-                            <i class="umb-healthcheck-status-icon icon-info"></i>
-                        </div>
-                        
-                        <div class="umb-healthcheck-group__details-status-content">
-                            <div class="umb-healthcheck-group__details-status-text">
-                                <a class="btn-link -underline" href="" ng-click="toggle(searcher, 'showProperties')">
-                                    {{searcher.name}}
-                                </a>
+                        <!-- Search Tool -->
+
+                        <div class="umb-healthcheck-group__details-check">
+
+                            <div class="umb-healthcheck-group__details-check-title">
+                                <div class="umb-healthcheck-group__details-check-name">Search</div>
+                                <div class="umb-healthcheck-group__details-check-description">Search the index and view the results</div>
                             </div>
-                            
-                            <div class="umb-healthcheck-group__details-status-actions" ng-show="searcher.showProperties">
-                                <ul>
-                                    <li class="search-tools">
 
-                                        <a href="" ng-click="toggle(searcher, 'showTools')">Search tools</a>
+                            <div class="umb-healthcheck-group__details-status">
 
-                                        <div ng-show="searcher.showTools">
-                                            <a class="hide" href="" ng-click="closeSearch(searcher)" ng-show="searcher.isSearching">Hide search results</a>
+                                <div class="umb-healthcheck-group__details-status-content">
 
-                                            <br />
-
-                                            <form>
+                                    <div class="umb-healthcheck-group__details-status-actions">
+                                        <div class="umb-healthcheck-group__details-status-action">
+                                            <ng-form name="searchTools">
 
                                                 <div class="row form-search">
-                                                    <div class="span8 input-append">
-                                                        <input type="text" class="search-query" ng-model="searcher.searchText" no-dirty-check />
-                                                        <button type="submit" class="btn btn-info" ng-click="search(searcher)" ng-disabled="searcher.isProcessing">Search</button>
+                                                    <div>
+                                                        <input type="text" class="search-query"
+                                                               ng-model="vm.searchText" no-dirty-check
+                                                               ng-keypress="vm.search(vm.selectedSearcher, $event)" />
+
+                                                        <umb-button disabled="vm.selectedSearcher.isProcessing"
+                                                                    type="button"
+                                                                    button-style="success"
+                                                                    action="vm.search(vm.selectedSearcher)"
+                                                                    label="Search">
+                                                        </umb-button>
+
                                                     </div>
                                                 </div>
-                                                <div class="row">
-                                                    <label for="{{searcher.name}}-textSearch" class="radio inline">
-                                                        <input type="radio" name="searchType" id="{{searcher.name}}-textSearch" value="text" ng-model="searcher.searchType" no-dirty-check />
-                                                        Text Search
-                                                    </label>
-                                                    <label for="{{searcher.name}}-luceneSearch" class="radio inline">
-                                                        <input type="radio" name="searchType" id="{{searcher.name}}-luceneSearch" value="lucene" ng-model="searcher.searchType" no-dirty-check />
-                                                        Lucene Search
-                                                    </label>
+
+                                                <div ng-if="!vm.selectedSearcher.isProcessing && vm.searchResults">
+                                                    <br />
+
+                                                    <table class="table table-bordered table-condensed">
+                                                        <thead>
+                                                            <tr>
+                                                                <th class="score">Score</th>
+                                                                <th class="id">Id</th>
+                                                                <th>Name</th>
+                                                            </tr>
+                                                        </thead>
+                                                        <tbody>
+                                                            <tr ng-repeat="result in vm.searchResults.results track by $index">
+                                                                <td>{{result.score}}</td>
+                                                                <td>{{result.id}}</td>
+                                                                <td>
+                                                                    <span>{{result.values['nodeName']}}</span>&nbsp;
+                                                                    <a class="color-green" href="" ng-click="vm.showSearchResultDialog(result.values)">
+                                                                        <em>({{result.fieldCount}} fields)</em>
+                                                                    </a>
+                                                                </td>
+                                                            </tr>
+                                                        </tbody>
+                                                    </table>
+
+                                                    <div class="flex justify-center">
+                                                        <umb-pagination page-number="vm.searchResults.pageNumber"
+                                                                        total-pages="vm.searchResults.totalPages"
+                                                                        on-next="vm.nextSearchResultPage"
+                                                                        on-prev="vm.prevSearchResultPage"
+                                                                        on-go-to-page="vm.goToPageSearchResultPage">
+                                                        </umb-pagination>
+                                                    </div>
+
                                                 </div>
 
+                                            </ng-form>
+                                        </div>
+                                    </div>
 
-                                            </form>
+                                </div>
 
-                                            <div class="search-results" ng-show="searcher.isSearching">
+                            </div>
 
-                                                <div ng-show="indexer.isProcessing" class="umb-loader-wrapper" ng-show="indexer.isProcessing">
-                                                    <div class="umb-loader"></div>
+                        </div>
+
+                    </div>
+
+                </div>
+
+            </div>
+        </div>
+
+        <div ng-if="vm.viewState === 'index-details'">
+
+            <umb-editor-sub-header>
+                <umb-editor-sub-header-content-left>
+                    <a class="umb-healthcheck-back-link" href="" ng-click="vm.setViewState('list');">&larr; Back to overview</a>
+                </umb-editor-sub-header-content-left>
+            </umb-editor-sub-header>
+
+
+            <div class="umb-healthcheck-group__details">
+
+                <div class="umb-healthcheck-group__details-group">
+
+                    <div class="umb-healthcheck-group__details-group-title">
+                        <div class="umb-healthcheck-group__details-group-name">{{ vm.selectedIndex.name }}</div>
+                    </div>
+
+                    <div class="umb-healthcheck-group__details-checks">
+
+                        <!-- Health Status -->
+
+                        <div class="umb-healthcheck-group__details-check">
+
+                            <div class="umb-healthcheck-group__details-check-title">
+                                <div class="umb-healthcheck-group__details-check-name">Health status</div>
+                                <div class="umb-healthcheck-group__details-check-description">The health status of the index and if it can be read</div>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status">
+
+                                <div class="umb-healthcheck-group__details-status-icon-container">
+                                    <i class="umb-healthcheck-status-icon"
+                                       ng-class="{'icon-check color-green' : vm.selectedIndex.isHealthy, 'icon-delete color-red' : !vm.selectedIndex.isHealthy}"></i>
+                                </div>
+
+                                <div class="umb-healthcheck-group__details-status-content">
+
+                                    <div class="umb-healthcheck-group__details-status-text">
+                                        <div>{{vm.selectedIndex.healthStatus}}</div>
+                                        <div ng-show="!vm.selectedIndex" class="color-red">
+                                            The index cannot be read and will need to be rebuilt
+                                        </div>
+                                        <!--<div ng-if="status.description" ng-bind-html="status.description"></div>-->
+                                    </div>
+                                </div>
+
+                            </div>
+
+                        </div>
+
+                        <!-- Search Tool -->
+
+                        <div class="umb-healthcheck-group__details-check">
+
+                            <div class="umb-healthcheck-group__details-check-title">
+                                <div class="umb-healthcheck-group__details-check-name">Search</div>
+                                <div class="umb-healthcheck-group__details-check-description">Search the index and view the results</div>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status">
+
+                                <div class="umb-healthcheck-group__details-status-content">
+
+                                    <div class="umb-healthcheck-group__details-status-actions">
+                                        <div class="umb-healthcheck-group__details-status-action">
+                                            <ng-form name="searchTools">
+
+                                                <div class="row form-search">
+                                                    <div>
+                                                        <input type="text" class="search-query"
+                                                               ng-model="vm.searchText" no-dirty-check
+                                                               ng-keypress="vm.search(vm.selectedIndex, $event)" />
+
+                                                        <umb-button disabled="vm.selectedIndex.isProcessing"
+                                                                    type="button"
+                                                                    button-style="success"
+                                                                    action="vm.search(vm.selectedIndex)"
+                                                                    label="Search">
+                                                        </umb-button>
+
+                                                    </div>
                                                 </div>
 
-                                                <table ng-hide="searcher.isProcessing" class="table table-bordered table-condensed">
-                                                    <thead>
-                                                        <tr>
-                                                            <th class="score">Score</th>
-                                                            <th class="id">Id</th>
-                                                            <th>Values</th>
-                                                        </tr>
-                                                    </thead>
-                                                    <tbody>
-                                                        <tr ng-repeat="result in searcher.searchResults">
-                                                            <td>{{result.Score}}</td>
-                                                            <td>{{result.Id}}</td>
-                                                            <td>
-                                                                <span ng-repeat="(key,val) in result.Fields track by $index">
-                                                                    <span class=""><em>{{key}}</em>:</span>
-                                                                    <span class="text-info">{{val}}</span>
-                                                                </span>
-                                                            </td>
-                                                        </tr>
-                                                    </tbody>
-                                                </table>
+                                                <div ng-if="!vm.selectedIndex.isProcessing && vm.searchResults">
+                                                    <br />
+
+                                                    <table class="table table-bordered table-condensed">
+                                                        <thead>
+                                                            <tr>
+                                                                <th class="score">Score</th>
+                                                                <th class="id">Id</th>
+                                                                <th>Name</th>
+                                                            </tr>
+                                                        </thead>
+                                                        <tbody>
+                                                            <tr ng-repeat="result in vm.searchResults.results track by $index">
+                                                                <td>{{result.score}}</td>
+                                                                <td>{{result.id}}</td>
+                                                                <td>
+                                                                    <span>{{result.values['nodeName']}}</span>&nbsp;
+                                                                    <a class="color-green" href="" ng-click="vm.showSearchResultDialog(result.values)">
+                                                                        <em>({{result.fieldCount}} fields)</em>
+                                                                    </a>
+                                                                </td>
+                                                            </tr>
+                                                        </tbody>
+                                                    </table>
+
+                                                    <div class="flex justify-center">
+                                                        <umb-pagination page-number="vm.searchResults.pageNumber"
+                                                                        total-pages="vm.searchResults.totalPages"
+                                                                        on-next="vm.nextSearchResultPage"
+                                                                        on-prev="vm.prevSearchResultPage"
+                                                                        on-go-to-page="vm.goToPageSearchResultPage">
+                                                        </umb-pagination>
+                                                    </div>
+
+                                                </div>
+
+                                            </ng-form>
+                                        </div>
+                                    </div>
+
+                                </div>
+
+                            </div>
+
+                        </div>
+
+                        <!-- Index Stats -->
+
+                        <div class="umb-healthcheck-group__details-check">
+
+                            <div class="umb-healthcheck-group__details-check-title">
+                                <div class="umb-healthcheck-group__details-check-name">Index info</div>
+                                <div class="umb-healthcheck-group__details-check-description">Lists the properties of the index</div>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status">
+
+                                <div class="umb-healthcheck-group__details-status-content">
+
+                                    <table class="table table-bordered table-condensed">
+                                        <caption>&nbsp;</caption>
+                                        <tr ng-repeat="(key, val) in vm.selectedIndex.providerProperties track by $index">
+                                            <th>{{key}}</th>
+                                            <td>{{val}}</td>
+                                        </tr>
+                                    </table>
+
+                                </div>
+
+                            </div>
+
+                        </div>
+
+                        <!-- Rebuild -->
+
+                        <div class="umb-healthcheck-group__details-check">
+
+                            <div class="umb-healthcheck-group__details-check-title">
+                                <div class="umb-healthcheck-group__details-check-name">Tools</div>
+                                <div class="umb-healthcheck-group__details-check-description">Tools to manage the index</div>
+                            </div>
+
+                            <div class="umb-healthcheck-group__details-status">
+
+                                <div class="umb-healthcheck-group__details-status-content">
+
+                                    <div class="umb-healthcheck-group__details-status-actions">
+                                        <div class="umb-healthcheck-group__details-status-action">
+                                            <ng-form name="indexTools">
+
+                                                <umb-button ng-show="!vm.selectedIndex.isProcessing && (!vm.selectedIndex.processingAttempts || vm.selectedIndex.processingAttempts < 100)"
+                                                            disabled="!vm.selectedIndex.canRebuild"
+                                                            type="button"
+                                                            button-style="success"
+                                                            action="vm.rebuildIndex(vm.selectedIndex)"
+                                                            label="Rebuild index">
+                                                </umb-button>
+
+                                                <div ng-show="vm.selectedIndex.isProcessing">
+                                                    <div class="umb-healthcheck-group__details-status-overlay"></div>
+                                                    <umb-load-indicator></umb-load-indicator>
+                                                </div>
+
+                                                <div ng-show="vm.selectedIndex.processingAttempts >= 100">
+                                                    The process is taking longer than expected, check the umbraco log to see if there have been any errors during this operation
+                                                </div>
+
+                                            </ng-form>
+
+                                            <div class="umb-healthcheck-group__details-status-action-description" ng-show="!vm.selectedIndex.canRebuild">
+                                                This index cannot be rebuilt because it has no assigned <code>IIndexPopulator</code>
                                             </div>
                                         </div>
-                                    </li>
-                                    <li>
-                                        <a href="" ng-click="toggle(searcher, 'showProviderProperties')">Provider properties</a>
-                                        <table ng-show="searcher.showProviderProperties" class="table table-bordered table-condensed">
-                                            <caption>&nbsp;</caption>
-                                            <tr ng-repeat="(key, val) in searcher.providerProperties track by $index">
-                                                <th>{{key}}</th>
-                                                <td>{{val}}</td>
-                                            </tr>
-                                        </table>
-                                    </li>
-                                </ul>
+                                    </div>
+
+                                </div>
+
                             </div>
+
                         </div>
+
                     </div>
+
                 </div>
+
             </div>
 
         </div>
-
-
     </div>
 
-
+    <umb-overlay ng-if="vm.searchResults.overlay"
+                 position="center"
+                 view="vm.searchResults.overlay.view"
+                 model="vm.searchResults.overlay">
+    </umb-overlay>
 
 </div>

src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagementresults.html

@@ -0,0 +1,18 @@
+<div>
+
+    <table class="table table-bordered table-condensed">
+        <thead>
+        <tr>
+            <th class="score">Field</th>
+            <th class="id">Value</th>
+        </tr>
+        </thead>
+        <tbody>
+        <tr ng-repeat="(key, val) in model.searchResultValues track by key">
+            <td>{{key}}</td>
+            <td>{{val}}</td>
+        </tr>
+        </tbody>
+    </table>
+
+</div>

src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemgmt.controller.js

@@ -1,128 +0,0 @@
-function ExamineMgmtController($scope, umbRequestHelper, $log, $http, $q, $timeout) {
-
-    $scope.indexerDetails = [];
-    $scope.searcherDetails = [];
-    $scope.loading = true;
-
-    function checkProcessing(indexer, checkActionName) {
-        umbRequestHelper.resourcePromise(
-                $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
-                    checkActionName,
-                    { indexerName: indexer.name })),
-                'Failed to check index processing')
-            .then(function(data) {
-
-                if (data !== null && data !== "null") {
-
-                    //copy all resulting properties
-                    for (var k in data) {
-                        indexer[k] = data[k];
-                    }
-                    indexer.isProcessing = false;
-                } else {
-                    $timeout(function() {
-                            //don't continue if we've tried 100 times
-                            if (indexer.processingAttempts < 100) {
-                                checkProcessing(indexer, checkActionName);
-                                //add an attempt
-                                indexer.processingAttempts++;
-                            } else {
-                                //we've exceeded 100 attempts, stop processing
-                                indexer.isProcessing = false;
-                            }
-                        },
-                        1000);
-                }
-            });
-    }
-
-    $scope.search = function(searcher, e) {
-        if (e && e.keyCode !== 13) {
-            return;
-        }
-
-        umbRequestHelper.resourcePromise(
-                $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
-                    "GetSearchResults",
-                    {
-                        searcherName: searcher.name,
-                        query: encodeURIComponent(searcher.searchText),
-                        queryType: searcher.searchType
-                    })),
-                'Failed to search')
-            .then(function(searchResults) {
-                searcher.isSearching = true;
-                searcher.searchResults = searchResults;
-            });
-    }
-
-    $scope.toggle = function(provider, propName) {
-        if (provider[propName] !== undefined) {
-            provider[propName] = !provider[propName];
-        } else {
-            provider[propName] = true;
-        }
-    }
-
-    $scope.rebuildIndex = function(indexer) {
-        if (confirm("This will cause the index to be rebuilt. " +
-            "Depending on how much content there is in your site this could take a while. " +
-            "It is not recommended to rebuild an index during times of high website traffic " +
-            "or when editors are editing content.")) {
-
-            indexer.isProcessing = true;
-            indexer.processingAttempts = 0;
-
-            umbRequestHelper.resourcePromise(
-                    $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
-                        "PostRebuildIndex",
-                        { indexerName: indexer.name })),
-                    'Failed to rebuild index')
-                .then(function() {
-
-                    //rebuilding has started, nothing is returned accept a 200 status code.
-                    //lets poll to see if it is done.
-                    $timeout(function() {
-                            checkProcessing(indexer, "PostCheckRebuildIndex");
-                        },
-                        1000);
-
-                });
-        }
-    }
-
-    $scope.closeSearch = function(searcher) {
-        searcher.isSearching = true;
-    }
-
-    //go get the data
-
-    //combine two promises and execute when they are both done
-    $q.all([
-
-            //get the indexer details
-            umbRequestHelper.resourcePromise(
-                $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", "GetIndexerDetails")),
-                'Failed to retrieve indexer details')
-            .then(function(data) {
-                $scope.indexerDetails = data;
-            }),
-
-            //get the searcher details
-            umbRequestHelper.resourcePromise(
-                $http.get(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", "GetSearcherDetails")),
-                'Failed to retrieve searcher details')
-            .then(function(data) {
-                $scope.searcherDetails = data;
-                for (var s in $scope.searcherDetails) {
-                    $scope.searcherDetails[s].searchType = "text";
-                }
-            })
-        ])
-        .then(function() {
-            //all init loading is complete
-            $scope.loading = false;
-        });
-}
-
-angular.module("umbraco").controller("Umbraco.Dashboard.ExamineMgmtController", ExamineMgmtController);

src/Umbraco.Web.UI/Global.asax

@@ -1,2 +1 @@
 <%@ Application Inherits="Umbraco.Web.UmbracoApplication" Language="C#" %>
-          

src/Umbraco.Web.UI/Umbraco.Web.UI.csproj

@@ -88,10 +88,9 @@
     <PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
     <PackageReference Include="ClientDependency" Version="1.9.7" />
     <PackageReference Include="ClientDependency-Mvc5" Version="1.8.0.0" />
-    <PackageReference Include="Examine" Version="1.0.0-beta025" />
+    <PackageReference Include="Examine" Version="1.0.0-beta046" />
     <PackageReference Include="ImageProcessor.Web" Version="4.9.3.25" />
     <PackageReference Include="ImageProcessor.Web.Config" Version="2.4.1.19" />
-    <PackageReference Include="Lucene.Net.Contrib" Version="3.0.3" />
     <PackageReference Include="Microsoft.AspNet.Identity.Owin" Version="2.2.2" />
     <PackageReference Include="Microsoft.AspNet.Mvc" Version="5.2.6" />
     <PackageReference Include="Microsoft.AspNet.WebApi" Version="5.2.6" />

src/Umbraco.Web.UI/config/ExamineIndex.Release.config

@@ -7,22 +7,4 @@ Index/Search providers can be defined in the UmbracoSettings.config
 More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
 -->
 <ExamineLuceneIndexSets>
-  <!-- The internal index set used by Umbraco back-office - DO NOT REMOVE -->
-  <IndexSet SetName="InternalIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/Internal/"/>
-
-  <!-- The internal index set used by Umbraco back-office for indexing members - DO NOT REMOVE -->
-  <IndexSet SetName="InternalMemberIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/InternalMember/">
-    <IndexAttributeFields>
-      <add Name="id" />
-      <add Name="nodeName"/>
-      <add Name="updateDate" />
-      <add Name="writerName" />
-      <add Name="loginName" />
-      <add Name="email" />
-      <add Name="nodeTypeAlias" />
-    </IndexAttributeFields>
-  </IndexSet>
-    
-  <!-- Default Indexset for external searches, this indexes all fields on all types of nodes-->
-  <IndexSet SetName="ExternalIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/External/" />
-</ExamineLuceneIndexSets>
+</ExamineLuceneIndexSets>

src/Umbraco.Web.UI/config/ExamineIndex.config

@@ -4,27 +4,11 @@ Umbraco examine is an extensible indexer and search engine.
 This configuration file can be extended to create your own index sets.
 Index/Search providers can be defined in the UmbracoSettings.config
 
-More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
+More information and documentation can be found on Our:
+https://our.umbraco.com/Documentation/Reference/Searching/Examine/
+https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/
+https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/
+
 -->
 <ExamineLuceneIndexSets>
-  
-  <!-- The internal index set used by Umbraco back-office - DO NOT REMOVE -->
-  <IndexSet SetName="InternalIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/Internal/" />
-
-  <!-- The internal index set used by Umbraco back-office for indexing members - DO NOT REMOVE -->
-  <IndexSet SetName="InternalMemberIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/InternalMember/">
-    <IndexAttributeFields>
-      <add Name="id" />
-      <add Name="nodeName"/>
-      <add Name="updateDate" />
-      <add Name="writerName" />
-      <add Name="loginName" />
-      <add Name="email" />
-      <add Name="nodeTypeAlias" /> 
-    </IndexAttributeFields>
-  </IndexSet>
-  
-  <!-- Default Indexset for external searches, this indexes all fields on all types of nodes-->
-  <IndexSet SetName="ExternalIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/External/" />
-  
-</ExamineLuceneIndexSets>
+</ExamineLuceneIndexSets>

src/Umbraco.Web.UI/config/ExamineSettings.Release.config

@@ -7,34 +7,14 @@ Index sets can be defined in the ExamineIndex.config if you're using the standar
 More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
 -->
 <Examine>
+    
   <ExamineIndexProviders>
     <providers>
-      <add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
-           supportUnpublished="true"
-           supportProtected="true"
-           analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
-
-      <add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
-           supportUnpublished="true"
-           supportProtected="true"
-           analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
-
-        <!-- default external indexer, which excludes protected and unpublished pages-->
-        <add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
-
     </providers>
   </ExamineIndexProviders>
 
   <ExamineSearchProviders>
     <providers>
-      <add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
-           analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
-
-      <add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine" />
-
-      <add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
-           analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
-
     </providers>
   </ExamineSearchProviders>
 

src/Umbraco.Web.UI/config/ExamineSettings.config

@@ -4,38 +4,20 @@ Umbraco examine is an extensible indexer and search engine.
 This configuration file can be extended to add your own search/index providers.
 Index sets can be defined in the ExamineIndex.config if you're using the standard provider model.
 
-More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
+More information and documentation can be found on Our:
+https://our.umbraco.com/Documentation/Reference/Searching/Examine/
+https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/
+https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/
 -->
 <Examine>
   <ExamineIndexProviders>
     <providers>
-      <add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
-           supportUnpublished="true"
-           supportProtected="true"
-           analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
-
-      <add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
-           supportUnpublished="true"
-           supportProtected="true"
-           analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
-
-      <!-- default external indexer, which excludes protected and unpublished pages-->
-      <add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
-
     </providers>
   </ExamineIndexProviders>
 
   <ExamineSearchProviders>
     <providers>
-      <add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
-           analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
-
-      <add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"/>
-
-      <add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
-           analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
-
     </providers>
   </ExamineSearchProviders>
 
-</Examine>
+</Examine> 

src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs

@@ -11,6 +11,7 @@ using Umbraco.Core.Scoping;
 using Umbraco.Core.Services;
 using Umbraco.Core.Services.Changes;
 using Umbraco.Core.Sync;
+using Umbraco.Examine;
 using Umbraco.Web.Cache;
 using Umbraco.Web.Composing;
 using Umbraco.Web.Routing;
@@ -48,7 +49,7 @@ namespace Umbraco.Web.Components
         private BackgroundTaskRunner<IBackgroundTask> _processTaskRunner;
         private bool _started;
         private IBackgroundTask[] _tasks;
-        private IExamineManager _examineManager;
+        private IndexRebuilder _indexRebuilder;
 
         public override void Compose(Composition composition)
         {
@@ -87,13 +88,13 @@ namespace Umbraco.Web.Components
                             //rebuild indexes if the server is not synced
                             // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
                             // indexes then they can adjust this logic themselves.
-                            () => ExamineComponent.RebuildIndexes(false, _examineManager, _logger)
+                            () => ExamineComponent.RebuildIndexes(_indexRebuilder, _logger, false, 5000)
                         }
                     });
             });
         }
 
-        public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineManager examineManager)
+        public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder)
         {
             _registrar = serverRegistrar as DatabaseServerRegistrar;
             if (_registrar == null) throw new Exception("panic: registar.");
@@ -101,12 +102,10 @@ namespace Umbraco.Web.Components
             _messenger = serverMessenger as BatchedDatabaseServerMessenger;
             if (_messenger == null) throw new Exception("panic: messenger");
 
-            _messenger.Startup();
-
             _runtime = runtime;
             _logger = logger;
             _registrationService = registrationService;
-            _examineManager = examineManager;
+            _indexRebuilder = indexRebuilder;
 
             _touchTaskRunner = new BackgroundTaskRunner<IBackgroundTask>("ServerRegistration",
                 new BackgroundTaskRunnerOptions { AutoStart = true }, logger);
@@ -115,6 +114,9 @@ namespace Umbraco.Web.Components
 
             //We will start the whole process when a successful request is made
             UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce;
+
+            // must come last, as it references some _variables
+            _messenger.Startup();
         }
 
         /// <summary>

src/Umbraco.Web/Editors/EntityController.cs

@@ -15,6 +15,7 @@ using Umbraco.Core.Models;
 using Constants = Umbraco.Core.Constants;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using System.Web.Http.Controllers;
+using Examine;
 using Umbraco.Core.Models.Entities;
 using Umbraco.Core.Xml;
 using Umbraco.Web.Models.Mapping;
@@ -53,12 +54,13 @@ namespace Umbraco.Web.Editors
             }
         }
 
-        private readonly UmbracoTreeSearcher _treeSearcher = new UmbracoTreeSearcher();
+        private readonly UmbracoTreeSearcher _treeSearcher;
         private readonly SearchableTreeCollection _searchableTreeCollection;
 
-        public EntityController(SearchableTreeCollection searchableTreeCollection)
+        public EntityController(SearchableTreeCollection searchableTreeCollection, UmbracoTreeSearcher treeSearcher)
         {
             _searchableTreeCollection = searchableTreeCollection;
+            _treeSearcher = treeSearcher;
         }
 
         /// <summary>
@@ -121,9 +123,8 @@ namespace Umbraco.Web.Editors
                 return result;
 
             var allowedSections = Security.CurrentUser.AllowedSections.ToArray();
-            var searchableTrees = _searchableTreeCollection.AsReadOnlyDictionary();
 
-            foreach (var searchableTree in searchableTrees)
+            foreach (var searchableTree in _searchableTreeCollection.SearchableApplicationTrees)
             {
                 if (allowedSections.Contains(searchableTree.Value.AppAlias))
                 {
@@ -411,22 +412,18 @@ namespace Umbraco.Web.Editors
             Direction orderDirection = Direction.Ascending,
             string filter = "")
         {
-            int intId;
-
-            if (int.TryParse(id, out intId))
+            if (int.TryParse(id, out var intId))
             {
                 return GetPagedChildren(intId, type, pageNumber, pageSize, orderBy, orderDirection, filter);
             }
 
-            Guid guidId;
-            if (Guid.TryParse(id, out guidId))
+            if (Guid.TryParse(id, out _))
             {
                 //Not supported currently
                 throw new HttpResponseException(HttpStatusCode.NotFound);
             }
 
-            Udi udiId;
-            if (Udi.TryParse(id, out udiId))
+            if (Udi.TryParse(id, out _))
             {
                 //Not supported currently
                 throw new HttpResponseException(HttpStatusCode.NotFound);
@@ -444,8 +441,7 @@ namespace Umbraco.Web.Editors
             //the EntityService cannot search members of a certain type, this is currently not supported and would require
             //quite a bit of plumbing to do in the Services/Repository, we'll revert to a paged search
 
-            long total;
-            var searchResult = _treeSearcher.ExamineSearch(Umbraco, filter ?? "", type, pageSize, pageNumber - 1, out total, id);
+            var searchResult = _treeSearcher.ExamineSearch(filter ?? "", type, pageSize, pageNumber - 1, out long total, id);
 
             return new PagedResult<EntityBasic>(total, pageNumber, pageSize)
             {
@@ -481,8 +477,7 @@ namespace Umbraco.Web.Editors
             var objectType = ConvertToObjectType(type);
             if (objectType.HasValue)
             {
-                long totalRecords;
-                var entities = Services.EntityService.GetPagedChildren(id, objectType.Value, pageNumber - 1, pageSize, out totalRecords, orderBy, orderDirection, filter);
+                var entities = Services.EntityService.GetPagedChildren(id, objectType.Value, pageNumber - 1, pageSize, out var totalRecords, orderBy, orderDirection, filter);
 
                 if (totalRecords == 0)
                 {
@@ -595,15 +590,11 @@ namespace Umbraco.Web.Editors
         /// <param name="entityType"></param>
         /// <param name="searchFrom"></param>
         /// <returns></returns>
-        private IEnumerable<SearchResultItem> ExamineSearch(string query, UmbracoEntityTypes entityType, string searchFrom = null)
+        private IEnumerable<SearchResultEntity> ExamineSearch(string query, UmbracoEntityTypes entityType, string searchFrom = null)
         {
-            long total;
-            return _treeSearcher.ExamineSearch(Umbraco, query, entityType, 200, 0, out total, searchFrom);
+            return _treeSearcher.ExamineSearch(query, entityType, 200, 0, out _, searchFrom);
         }
 
-
-
-
         private IEnumerable<EntityBasic> GetResultForChildren(int id, UmbracoEntityTypes entityType)
         {
             var objectType = ConvertToObjectType(entityType);

src/Umbraco.Web/Editors/ExamineManagementController.cs

@@ -9,12 +9,18 @@ using System.Web.Http;
 using Examine;
 using Examine.LuceneEngine;
 using Examine.LuceneEngine.Providers;
+using Lucene.Net.Analysis;
+using Lucene.Net.QueryParsers;
 using Umbraco.Core;
 using Umbraco.Core.Cache;
 using Umbraco.Core.Composing;
 using Umbraco.Core.Logging;
+using Umbraco.Examine;
+using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Mvc;
 using Umbraco.Web.Search;
+using SearchResult = Umbraco.Web.Models.ContentEditing.SearchResult;
+using Version = Lucene.Net.Util.Version;
 
 namespace Umbraco.Web.Editors
 {
@@ -24,26 +30,26 @@ namespace Umbraco.Web.Editors
         private readonly IExamineManager _examineManager;
         private readonly ILogger _logger;
         private readonly IRuntimeCacheProvider _runtimeCacheProvider;
+        private readonly IndexRebuilder _indexRebuilder;
+
 
         public ExamineManagementController(IExamineManager examineManager, ILogger logger,
-                                           IRuntimeCacheProvider runtimeCacheProvider)
+            IRuntimeCacheProvider runtimeCacheProvider,
+            IndexRebuilder indexRebuilder)
         {
             _examineManager = examineManager;
             _logger = logger;
             _runtimeCacheProvider = runtimeCacheProvider;
+            _indexRebuilder = indexRebuilder;
         }
 
         /// <summary>
         /// Get the details for indexers
         /// </summary>
         /// <returns></returns>
-        public IEnumerable<ExamineIndexerModel> GetIndexerDetails()
+        public IEnumerable<ExamineIndexModel> GetIndexerDetails()
         {
-            return _examineManager.IndexProviders.Select(CreateModel).OrderBy(x =>
-            {
-                //order by name , but strip the "Indexer" from the end if it exists
-                return x.Name.TrimEnd("Indexer");
-            });
+            return _examineManager.Indexes.Select(CreateModel).OrderBy(x => x.Name.TrimEnd("Indexer"));
         }
 
         /// <summary>
@@ -53,229 +59,189 @@ namespace Umbraco.Web.Editors
         public IEnumerable<ExamineSearcherModel> GetSearcherDetails()
         {
             var model = new List<ExamineSearcherModel>(
-               _examineManager.IndexProviders.Select(indexer =>
-               {
-                   var searcher = indexer.Value.GetSearcher();
-                   var searcherName = (searcher as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher");
-
-                   var indexerModel = new ExamineSearcherModel
-                   {
-                       Name = searcherName
-                   };
-                   var props = TypeHelper.CachedDiscoverableProperties(searcher.GetType(), mustWrite: false)
-                               //ignore these properties
-                               .Where(x => new[] { "Description" }.InvariantContains(x.Name) == false)
-                               .Where(x => x.GetCustomAttribute<EditorBrowsableAttribute>()
-                                           ?.State != EditorBrowsableState.Never)
-                               .OrderBy(x => x.Name);
-                   foreach (var p in props)
-                   {
-                       indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null)?.ToString());
-                   }
-
-                   return indexerModel;
-               }).OrderBy(x =>
-               {
-                   //order by name , but strip the "Searcher" from the end if it exists
-                   return x.Name.TrimEnd("Searcher");
-               }));
+                _examineManager.RegisteredSearchers.Select(searcher => new ExamineSearcherModel { Name = searcher.Name })
+                    .OrderBy(x => x.Name.TrimEnd("Searcher"))); //order by name , but strip the "Searcher" from the end if it exists
             return model;
         }
 
-        public ISearchResults GetSearchResults(string searcherName, string query, string queryType)
+        public SearchResults GetSearchResults(string searcherName, string query, int pageIndex = 0, int pageSize = 20)
         {
-            if (queryType == null)
-            {
-                throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound));
-            }
-
             if (query.IsNullOrWhiteSpace())
+                return SearchResults.Empty();
+
+            var msg = ValidateSearcher(searcherName, out var searcher);
+            if (!msg.IsSuccessStatusCode)
+                throw new HttpResponseException(msg);
+
+            var results = TryParseLuceneQuery(query)
+                ? searcher.Search(searcher.CreateCriteria().RawQuery(query), maxResults: pageSize * (pageIndex + 1))
+                : searcher.Search(query, true, maxResults: pageSize * (pageIndex + 1));
+
+            var pagedResults = results.Skip(pageIndex * pageSize);
+
+            return new SearchResults
             {
-                return LuceneSearchResults.Empty();
-            }
+                TotalRecords = results.TotalItemCount,
+                Results = pagedResults.Select(x => new SearchResult
+                {
+                    Id = x.Id,
+                    Score = x.Score,
+                    //order the values by key
+                    Values = new Dictionary<string, string>(x.Values.OrderBy(y => y.Key).ToDictionary(y => y.Key, y => y.Value))
+                })
+            };
+        }
 
-            var msg = ValidateLuceneSearcher(searcherName, out var searcher);
-            if (msg.IsSuccessStatusCode)
+        private bool TryParseLuceneQuery(string query)
+        {
+            //TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll
+            // also do this rudimentary check
+            if (!query.Contains(":"))
+                return false;
+
+            try
             {
-                if (queryType.InvariantEquals("text"))
-                {
-                    return searcher.Search(query, false);
-                }
-
-                if (queryType.InvariantEquals("lucene"))
-                {
-                    return searcher.Search(searcher.CreateCriteria().RawQuery(query));
-                }
-
-                throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound));
+                //This will pass with a plain old string without any fields, need to figure out a way to have it properly parse
+                var parsed = new QueryParser(Version.LUCENE_30, "nodeName", new KeywordAnalyzer()).Parse(query);
+                return true;
+            }
+            catch (ParseException)
+            {
+                return false;
+            }
+            catch (Exception)
+            {
+                return false;
             }
-
-            throw new HttpResponseException(msg);
         }
 
         /// <summary>
         /// Check if the index has been rebuilt
         /// </summary>
-        /// <param name="indexerName"></param>
+        /// <param name="indexName"></param>
         /// <returns></returns>
         /// <remarks>
         /// This is kind of rudimentary since there's no way we can know that the index has rebuilt, we
         /// have a listener for the index op complete so we'll just check if that key is no longer there in the runtime cache
         /// </remarks>
-        public ExamineIndexerModel PostCheckRebuildIndex(string indexerName)
+        public ExamineIndexModel PostCheckRebuildIndex(string indexName)
         {
-            var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer);
-            if (msg.IsSuccessStatusCode)
-            {
-                var cacheKey = "temp_indexing_op_" + indexerName;
-                var found = ApplicationCache.RuntimeCache.GetCacheItem(cacheKey);
+            var validate = ValidateIndex(indexName, out var index);
+            if (!validate.IsSuccessStatusCode)
+                throw new HttpResponseException(validate);
 
-                //if its still there then it's not done
-                return found != null
-                           ? null
-                           : CreateModel(new KeyValuePair<string, IIndexer>(indexerName, indexer));
-            }
+            validate = ValidatePopulator(indexName);
+            if (!validate.IsSuccessStatusCode)
+                throw new HttpResponseException(validate);
+
+            var cacheKey = "temp_indexing_op_" + indexName;
+            var found = ApplicationCache.RuntimeCache.GetCacheItem(cacheKey);
+
+            //if its still there then it's not done
+            return found != null
+                ? null
+                : CreateModel(index);
 
-            throw new HttpResponseException(msg);
         }
 
         /// <summary>
-        ///     Rebuilds the index
+        /// Rebuilds the index
         /// </summary>
-        /// <param name="indexerName"></param>
+        /// <param name="indexName"></param>
         /// <returns></returns>
-        public HttpResponseMessage PostRebuildIndex(string indexerName)
+        public HttpResponseMessage PostRebuildIndex(string indexName)
         {
-            var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer);
-            if (msg.IsSuccessStatusCode)
+            var validate = ValidateIndex(indexName, out var index);
+            if (!validate.IsSuccessStatusCode)
+                return validate;
+
+            validate = ValidatePopulator(indexName);
+            if (!validate.IsSuccessStatusCode)
+                return validate;
+
+            _logger.Info<ExamineManagementController>("Rebuilding index '{IndexName}'", indexName);
+
+            //remove it in case there's a handler there alraedy
+            index.IndexOperationComplete -= Indexer_IndexOperationComplete;
+
+            //now add a single handler
+            index.IndexOperationComplete += Indexer_IndexOperationComplete;
+
+            try
             {
-                _logger.Info<ExamineManagementController>("Rebuilding index '{IndexerName}'", indexerName);
-
-                //remove it in case there's a handler there alraedy
-                indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
-
-                //now add a single handler
-                indexer.IndexOperationComplete += Indexer_IndexOperationComplete;
-
-                var cacheKey = "temp_indexing_op_" + indexer.Name;
+                //clear and replace
+                index.CreateIndex();
 
+                var cacheKey = "temp_indexing_op_" + index.Name;
                 //put temp val in cache which is used as a rudimentary way to know when the indexing is done
-                ApplicationCache.RuntimeCache.InsertCacheItem(cacheKey, () => "tempValue", TimeSpan.FromMinutes(5),
-                                                              false);
+                ApplicationCache.RuntimeCache.InsertCacheItem(cacheKey, () => "tempValue", TimeSpan.FromMinutes(5));
 
-                try
-                {
-                    indexer.RebuildIndex();
-                }
-                catch (Exception ex)
-                {
-                    //ensure it's not listening
-                    indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
-                    Logger.Error<ExamineManagementController>(ex, "An error occurred rebuilding index");
-                    var response = Request.CreateResponse(HttpStatusCode.Conflict);
-                    response.Content =
-                        new
-                            StringContent($"The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {ex}");
-                    response.ReasonPhrase = "Could Not Rebuild";
-                    return response;
-                }
+                _indexRebuilder.RebuildIndex(indexName);
+
+                ////populate it
+                //foreach (var populator in _populators.Where(x => x.IsRegistered(indexName)))
+                //    populator.Populate(index);
+
+                return Request.CreateResponse(HttpStatusCode.OK);
+            }
+            catch (Exception ex)
+            {
+                //ensure it's not listening
+                index.IndexOperationComplete -= Indexer_IndexOperationComplete;
+                Logger.Error<ExamineManagementController>(ex, "An error occurred rebuilding index");
+                var response = Request.CreateResponse(HttpStatusCode.Conflict);
+                response.Content =
+                    new
+                        StringContent($"The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {ex}");
+                response.ReasonPhrase = "Could Not Rebuild";
+                return response;
             }
-
-            return msg;
         }
 
-        private ExamineIndexerModel CreateModel(KeyValuePair<string, IIndexer> indexerKeyVal)
+
+
+        private ExamineIndexModel CreateModel(IIndex index)
         {
-            var indexer = indexerKeyVal.Value;
-            var indexName = indexerKeyVal.Key;
-            var indexerModel = new ExamineIndexerModel
+            var indexName = index.Name;
+
+            if (!(index is IIndexDiagnostics indexDiag))
+                indexDiag = new GenericIndexDiagnostics(index);
+
+
+            var isHealth = indexDiag.IsHealthy();
+            var properties = new Dictionary<string, object>
             {
-                FieldDefinitions = indexer.FieldDefinitionCollection,
-                Name = indexName
+                [nameof(IIndexDiagnostics.DocumentCount)] = indexDiag.DocumentCount,
+                [nameof(IIndexDiagnostics.FieldCount)] = indexDiag.FieldCount,
+            };
+            foreach (var p in indexDiag.Metadata)
+                properties[p.Key] = p.Value;
+
+            var indexerModel = new ExamineIndexModel
+            {
+                Name = indexName,
+                HealthStatus = isHealth.Success ? (isHealth.Result ?? "Healthy") : (isHealth.Result ?? "Unhealthy"),
+                ProviderProperties = properties,
+                CanRebuild = _indexRebuilder.CanRebuild(indexName)
             };
 
-            var props = TypeHelper.CachedDiscoverableProperties(indexer.GetType(), mustWrite: false)
-                                  //ignore these properties
-                                  .Where(x => new[] { "IndexerData", "Description", "WorkingFolder" }
-                                                  .InvariantContains(x.Name) == false)
-                                  .OrderBy(x => x.Name);
-
-            foreach (var p in props)
-            {
-                var val = p.GetValue(indexer, null);
-                if (val == null)
-                {
-                    // Do not warn for new new attribute that is optional
-                    if (string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false)
-                    {
-                        Logger
-                            .Warn<ExamineManagementController
-                            >("Property value was null when setting up property on indexer: " + indexName +
-                              " property: " + p.Name);
-                    }
-
-                    val = string.Empty;
-                }
-
-                indexerModel.ProviderProperties.Add(p.Name, val.ToString());
-            }
-
-            if (indexer is LuceneIndexer luceneIndexer)
-            {
-                indexerModel.IsLuceneIndex = true;
-
-                if (luceneIndexer.IndexExists())
-                {
-                    indexerModel.IsHealthy = luceneIndexer.IsHealthy(out var indexError);
-
-                    if (indexerModel.IsHealthy == false)
-                    {
-                        //we cannot continue at this point
-                        indexerModel.Error = indexError.ToString();
-                        return indexerModel;
-                    }
-
-                    indexerModel.DocumentCount = luceneIndexer.GetIndexDocumentCount();
-                    indexerModel.FieldCount = luceneIndexer.GetIndexFieldCount();
-                }
-                else
-                {
-                    indexerModel.DocumentCount = 0;
-                    indexerModel.FieldCount = 0;
-                }
-            }
 
             return indexerModel;
         }
 
-        private HttpResponseMessage ValidateLuceneSearcher(string searcherName, out LuceneSearcher searcher)
+        private HttpResponseMessage ValidateSearcher(string searcherName, out ISearcher searcher)
         {
-            foreach (var indexer in _examineManager.IndexProviders)
+            //try to get the searcher from the indexes
+            if (_examineManager.TryGetIndex(searcherName, out var index))
             {
-                var s = indexer.Value.GetSearcher();
-                var sName = (s as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher");
-                if (sName != searcherName)
-                {
-                    continue;
-                }
-
-                searcher = s as LuceneSearcher;
-
-                //Found it, return OK
-                if (searcher != null)
-                {
-                    return Request.CreateResponse(HttpStatusCode.OK);
-                }
-
-                //Return an error since it's not the right type
-                var response = Request.CreateResponse(HttpStatusCode.BadRequest);
-                response.Content =
-                    new StringContent($"The searcher {searcherName} is not of type {typeof(LuceneSearcher)}");
-                response.ReasonPhrase = "Wrong Searcher Type";
-                return response;
+                searcher = index.GetSearcher();
+                return Request.CreateResponse(HttpStatusCode.OK);
             }
 
-            searcher = null;
+
+            //if we didn't find anything try to find it by an explicitly declared searcher
+            if (_examineManager.TryGetSearcher(searcherName, out searcher))
+                return Request.CreateResponse(HttpStatusCode.OK);
 
             var response1 = Request.CreateResponse(HttpStatusCode.BadRequest);
             response1.Content = new StringContent($"No searcher found with name = {searcherName}");
@@ -283,29 +249,38 @@ namespace Umbraco.Web.Editors
             return response1;
         }
 
-        private HttpResponseMessage ValidateLuceneIndexer<T>(string indexerName, out T indexer)
-            where T : class, IIndexer
+        private HttpResponseMessage ValidatePopulator(string indexName)
         {
-            indexer = null;
+            if (_indexRebuilder.CanRebuild(indexName))
+                return Request.CreateResponse(HttpStatusCode.OK);
 
-            if (_examineManager.IndexProviders.ContainsKey(indexerName)
-                && _examineManager.IndexProviders[indexerName] is T casted)
+            var response = Request.CreateResponse(HttpStatusCode.BadRequest);
+            response.Content = new StringContent($"The index {indexName} cannot be rebuilt because it does not have an associated {typeof(IIndexPopulator)}");
+            response.ReasonPhrase = "Index cannot be rebuilt";
+            return response;
+        }
+
+        private HttpResponseMessage ValidateIndex(string indexName, out IIndex index)
+        {
+            index = null;
+
+            if (_examineManager.TryGetIndex(indexName, out index))
             {
                 //return Ok!
-                indexer = casted;
                 return Request.CreateResponse(HttpStatusCode.OK);
             }
 
             var response = Request.CreateResponse(HttpStatusCode.BadRequest);
-            response.Content = new StringContent($"No indexer found with name = {indexerName} of type {typeof(T)}");
-            response.ReasonPhrase = "Indexer Not Found";
+            response.Content = new StringContent($"No index found with name = {indexName}");
+            response.ReasonPhrase = "Index Not Found";
             return response;
         }
 
-        //static listener so it's not GC'd
         private void Indexer_IndexOperationComplete(object sender, EventArgs e)
         {
-            var indexer = (LuceneIndexer) sender;
+            var indexer = (LuceneIndex)sender;
+
+            _logger.Debug<ExamineManagementController>("Logging operation completed for index {IndexName}", indexer.Name);
 
             //ensure it's not listening anymore
             indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;

src/Umbraco.Web/Editors/TemplateQueryController.cs

@@ -59,8 +59,7 @@ namespace Umbraco.Web.Editors
 
         public QueryResultModel PostTemplateQuery(QueryModel model)
         {
-            var umbraco = new UmbracoHelper(UmbracoContext, Services, ApplicationCache);
-
+            
             var queryResult = new QueryResultModel();
 
             var sb = new StringBuilder();
@@ -72,7 +71,7 @@ namespace Umbraco.Web.Editors
 
             timer.Start();
 
-            var currentPage = umbraco.ContentAtRoot().FirstOrDefault();
+            var currentPage = Umbraco.ContentAtRoot().FirstOrDefault();
             timer.Stop();
 
             var pointerNode = currentPage;
@@ -80,7 +79,7 @@ namespace Umbraco.Web.Editors
             // adjust the "FROM"
             if (model != null && model.Source != null && model.Source.Id > 0)
             {
-                var targetNode = umbraco.Content(model.Source.Id);
+                var targetNode = Umbraco.Content(model.Source.Id);
 
                 if (targetNode != null)
                 {

src/Umbraco.Web/ExamineExtensions.cs

@@ -12,7 +12,7 @@ namespace Umbraco.Web
     /// </summary>
     public static class ExamineExtensions
     {
-        public static IEnumerable<PublishedSearchResult> ToPublishedSearchResults(this IEnumerable<SearchResult> results, IPublishedCache cache)
+        public static IEnumerable<PublishedSearchResult> ToPublishedSearchResults(this IEnumerable<ISearchResult> results, IPublishedCache cache)
         {
             var list = new List<PublishedSearchResult>();
 

src/Umbraco.Web/ExamineStartup.cs

@@ -1,212 +0,0 @@
-// fixme - Examine is borked
-// this cannot compile because it seems to require some changes that are not in Examine v2
-// this should *not* exist, see ExamineComponent instead, which handles everything about Examine
-
-//using System;
-//using System.Collections.Generic;
-//using System.Linq;
-//using Examine;
-//using Examine.Config;
-//using Examine.LuceneEngine;
-//using Examine.LuceneEngine.Providers;
-//using Examine.Providers;
-//using Lucene.Net.Index;
-//using Lucene.Net.Store;
-//using Umbraco.Core;
-//using Umbraco.Core.Logging;
-//using UmbracoExamine;
-
-//namespace Umbraco.Web
-//{
-//    /// <summary>
-//    /// Used to configure Examine during startup for the web application
-//    /// </summary>
-//    internal class ExamineStartup
-//    {
-//        private readonly List<BaseIndexProvider> _indexesToRebuild = new List<BaseIndexProvider>();
-//        private readonly ApplicationContext _appCtx;
-//        private readonly ProfilingLogger _profilingLogger;
-//        private static bool _isConfigured = false;
-
-//        //this is used if we are not the MainDom, in which case we need to ensure that if indexes need rebuilding that this
-//        //doesn't occur since that should only occur when we are MainDom
-//        private bool _disableExamineIndexing = false;
-
-//        public ExamineStartup(ApplicationContext appCtx)
-//        {
-//            _appCtx = appCtx;
-//            _profilingLogger = appCtx.ProfilingLogger;
-//        }
-
-//        /// <summary>
-//        /// Called during the initialize operation of the boot manager process
-//        /// </summary>
-//        public void Initialize()
-//        {
-//            //We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior
-//            //and then we'll use MainDom to control Examine's shutdown
-//            ExamineManager.DisableDefaultHostingEnvironmentRegistration();
-
-//            //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
-//            //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
-//            //which simply checks the existence of the lock file
-//            DirectoryTracker.DefaultLockFactory = d =>
-//            {
-//                var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
-//                return simpleFsLockFactory;
-//            };
-
-//            //This is basically a hack for this item: http://issues.umbraco.org/issue/U4-5976
-//            // when Examine initializes it will try to rebuild if the indexes are empty, however in many cases not all of Examine's
-//            // event handlers will be assigned during bootup when the rebuilding starts which is a problem. So with the examine 0.1.58.2941 build
-//            // it has an event we can subscribe to in order to cancel this rebuilding process, but what we'll do is cancel it and postpone the rebuilding until the
-//            // boot process has completed. It's a hack but it works.
-//            ExamineManager.Instance.BuildingEmptyIndexOnStartup += OnInstanceOnBuildingEmptyIndexOnStartup;
-
-//            //let's deal with shutting down Examine with MainDom
-//            var examineShutdownRegistered = _appCtx.MainDom.Register(() =>
-//            {
-//                using (_profilingLogger.TraceDuration<ExamineStartup>("Examine shutting down"))
-//                {
-//                    //Due to the way Examine's own IRegisteredObject works, we'll first run it with immediate=false and then true so that
-//                    //it's correct subroutines are executed (otherwise we'd have to run this logic manually ourselves)
-//                    ExamineManager.Instance.Stop(false);
-//                    ExamineManager.Instance.Stop(true);
-//                }
-//            });
-//            if (examineShutdownRegistered)
-//            {
-//                _profilingLogger.Logger.Debug<ExamineStartup>("Examine shutdown registered with MainDom");
-//            }
-//            else
-//            {
-//                _profilingLogger.Logger.Debug<ExamineStartup>("Examine shutdown not registered, this appdomain is not the MainDom");
-
-//                //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled
-//                //from indexing anything on startup!!
-//                _disableExamineIndexing = true;
-//                Suspendable.ExamineEvents.SuspendIndexers();
-//            }
-//        }
-
-//        /// <summary>
-//        /// Called during the Complete operation of the boot manager process
-//        /// </summary>
-//        public void Complete()
-//        {
-//            EnsureUnlockedAndConfigured();
-
-//            //Ok, now that everything is complete we'll check if we've stored any references to index that need rebuilding and run them
-//            // (see the initialize method for notes) - we'll ensure we remove the event handler too in case examine manager doesn't actually
-//            // initialize during startup, in which case we want it to rebuild the indexes itself.
-//            ExamineManager.Instance.BuildingEmptyIndexOnStartup -= OnInstanceOnBuildingEmptyIndexOnStartup;
-
-//            //don't do anything if we have disabled this
-//            if (_disableExamineIndexing == false)
-//            {
-//                foreach (var indexer in _indexesToRebuild)
-//                {
-//                    indexer.RebuildIndex();
-//                }
-//            }
-//        }
-
-//        /// <summary>
-//        /// Called to perform the rebuilding indexes on startup if the indexes don't exist
-//        /// </summary>
-//        public void RebuildIndexes()
-//        {
-//            //don't do anything if we have disabled this
-//            if (_disableExamineIndexing) return;
-
-//            EnsureUnlockedAndConfigured();
-
-//            //If the developer has explicitly opted out of rebuilding indexes on startup then we
-//            // should adhere to that and not do it, this means that if they are load balancing things will be
-//            // out of sync if they are auto-scaling but there's not much we can do about that.
-//            if (ExamineSettings.Instance.RebuildOnAppStart == false) return;
-
-//            foreach (var indexer in GetIndexesForColdBoot())
-//            {
-//                indexer.RebuildIndex();
-//            }
-//        }
-
-//        /// <summary>
-//        /// The method used to create indexes on a cold boot
-//        /// </summary>
-//        /// <remarks>
-//        /// A cold boot is when the server determines it will not (or cannot) process instructions in the cache table and
-//        /// will rebuild it's own caches itself.
-//        /// </remarks>
-//        public IEnumerable<BaseIndexProvider> GetIndexesForColdBoot()
-//        {
-//            // NOTE: This is IMPORTANT! ... we don't want to rebuild any index that is already flagged to be re-indexed
-//            // on startup based on our _indexesToRebuild variable and how Examine auto-rebuilds when indexes are empty.
-//            // This callback is used above for the DatabaseServerMessenger startup options.
-
-//            // all indexes
-//            IEnumerable<BaseIndexProvider> indexes = ExamineManager.Instance.IndexProviderCollection;
-
-//            // except those that are already flagged
-//            // and are processed in Complete()
-//            if (_indexesToRebuild.Any())
-//                indexes = indexes.Except(_indexesToRebuild);
-
-//            // return
-//            foreach (var index in indexes)
-//                yield return index;
-//        }
-
-//        /// <summary>
-//        /// Must be called to configure each index and ensure it's unlocked before any indexing occurs
-//        /// </summary>
-//        /// <remarks>
-//        /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before
-//        /// either of these happens, we need to configure the indexes.
-//        /// </remarks>
-//        private void EnsureUnlockedAndConfigured()
-//        {
-//            if (_isConfigured) return;
-
-//            _isConfigured = true;
-
-//            foreach (var luceneIndexer in ExamineManager.Instance.IndexProviderCollection.OfType<LuceneIndexer>())
-//            {
-//                //We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending
-//                //indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because
-//                //that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems.
-//                luceneIndexer.WaitForIndexQueueOnShutdown = false;
-
-//                //we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that
-//                //the indexes are not operational unless MainDom is true so if _disableExamineIndexing is false then we should be in charge
-//                if (_disableExamineIndexing == false)
-//                {
-//                    var dir = luceneIndexer.GetLuceneDirectory();
-//                    if (IndexWriter.IsLocked(dir))
-//                    {
-//                        _profilingLogger.Logger.Info<ExamineStartup>("Forcing index " + luceneIndexer.IndexSetName + " to be unlocked since it was left in a locked state");
-//                        IndexWriter.Unlock(dir);
-//                    }
-//                }
-//            }
-//        }
-
-//        private void OnInstanceOnBuildingEmptyIndexOnStartup(object sender, BuildingEmptyIndexOnStartupEventArgs args)
-//        {
-//            //store the indexer that needs rebuilding because it's empty for when the boot process
-//            // is complete and cancel this current event so the rebuild process doesn't start right now.
-//            args.Cancel = true;
-//            _indexesToRebuild.Add((BaseIndexProvider)args.Indexer);
-
-//            //check if the index is rebuilding due to an error and log it
-//            if (args.IsHealthy == false)
-//            {
-//                var baseIndex = args.Indexer as BaseIndexProvider;
-//                var name = baseIndex != null ? baseIndex.Name : "[UKNOWN]";
-
-//                _profilingLogger.Logger.Error<ExamineStartup>(string.Format("The index {0} is rebuilding due to being unreadable/corrupt", name), args.UnhealthyException);
-//            }
-//        }
-//    }
-//}

src/Umbraco.Web/IPublishedContentQuery.cs

@@ -39,7 +39,7 @@ namespace Umbraco.Web
         /// <summary>
         /// Searches content.
         /// </summary>
-        IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null);
+        IEnumerable<PublishedSearchResult> Search(int skip, int take, out long totalRecords, string term, bool useWildCards = true, string indexName = null);
 
         /// <summary>
         /// Searches content.
@@ -49,6 +49,6 @@ namespace Umbraco.Web
         /// <summary>
         /// Searches content.
         /// </summary>
-        IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null);
+        IEnumerable<PublishedSearchResult> Search(int skip, int take, out long totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searcher = null);
     }
 }

src/Umbraco.Web/Models/ContentEditing/SearchResult.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+
+namespace Umbraco.Web.Models.ContentEditing
+{
+    [DataContract(Name = "result", Namespace = "")]
+    public class SearchResult
+    {
+        [DataMember(Name = "id")]
+        public string Id { get; set; }
+
+        [DataMember(Name = "score")]
+        public float Score { get; set; }
+
+        [DataMember(Name = "fieldCount")]
+        public int FieldCount => Values?.Count ?? 0;
+
+        [DataMember(Name = "values")]
+        public IReadOnlyDictionary<string, string> Values { get; set; }
+    }
+}

src/Umbraco.Web/Models/ContentEditing/SearchResultEntity.cs

@@ -3,7 +3,7 @@
 namespace Umbraco.Web.Models.ContentEditing
 {
     [DataContract(Name = "searchResult", Namespace = "")]
-    public class SearchResultItem : EntityBasic
+    public class SearchResultEntity : EntityBasic
     {
         /// <summary>
         /// The score of the search result

src/Umbraco.Web/Models/ContentEditing/SearchResults.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.Serialization;
+
+namespace Umbraco.Web.Models.ContentEditing
+{
+    [DataContract(Name = "results", Namespace = "")]
+    public class SearchResults
+    {
+        public static SearchResults Empty() => new SearchResults
+        {
+            Results = Enumerable.Empty<SearchResult>(),
+            TotalRecords = 0
+        };
+
+        [DataMember(Name = "totalRecords")]
+        public long TotalRecords { get; set; }
+
+        [DataMember(Name = "results")]
+        public IEnumerable<SearchResult> Results { get; set; }
+    }
+}

src/Umbraco.Web/Models/ContentEditing/TreeSearchResult.cs

@@ -30,6 +30,6 @@ namespace Umbraco.Web.Models.ContentEditing
         public string JsFormatterMethod { get; set; }
 
         [DataMember(Name = "results")]
-        public IEnumerable<SearchResultItem> Results { get; set; }
+        public IEnumerable<SearchResultEntity> Results { get; set; }
     }
 }

src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs

@@ -84,7 +84,7 @@ namespace Umbraco.Web.Models.Mapping
                 .ForMember(dest => dest.Trashed, opt => opt.Ignore())
                 .ForMember(dest => dest.AdditionalData, opt => opt.Ignore());
 
-            CreateMap<EntitySlim, SearchResultItem>()
+            CreateMap<EntitySlim, SearchResultEntity>()
                 .ForMember(dest => dest.Udi, opt => opt.MapFrom(src => Udi.Create(ObjectTypes.GetUdiType(src.NodeObjectType), src.Key)))
                 .ForMember(dest => dest.Icon, opt => opt.MapFrom(src=> GetContentTypeIcon(src)))
                 .ForMember(dest => dest.Trashed, opt => opt.Ignore())
@@ -107,7 +107,7 @@ namespace Umbraco.Web.Models.Mapping
                     }
                 });
 
-            CreateMap<SearchResult, SearchResultItem>()
+            CreateMap<ISearchResult, SearchResultEntity>()
                 //default to document icon
                   .ForMember(dest => dest.Score, opt => opt.MapFrom(result => result.Score))
                   .ForMember(dest => dest.Udi, opt => opt.Ignore())
@@ -123,22 +123,22 @@ namespace Umbraco.Web.Models.Mapping
                   .AfterMap((src, dest) =>
                       {
                           //get the icon if there is one
-                          dest.Icon = src.Fields.ContainsKey(UmbracoExamineIndexer.IconFieldName)
-                              ? src.Fields[UmbracoExamineIndexer.IconFieldName]
+                          dest.Icon = src.Values.ContainsKey(UmbracoExamineIndexer.IconFieldName)
+                              ? src.Values[UmbracoExamineIndexer.IconFieldName]
                               : "icon-document";
 
-                          dest.Name = src.Fields.ContainsKey("nodeName") ? src.Fields["nodeName"] : "[no name]";
-                          if (src.Fields.ContainsKey(UmbracoExamineIndexer.NodeKeyFieldName))
+                          dest.Name = src.Values.ContainsKey("nodeName") ? src.Values["nodeName"] : "[no name]";
+                          if (src.Values.ContainsKey(UmbracoExamineIndexer.NodeKeyFieldName))
                           {
                               Guid key;
-                              if (Guid.TryParse(src.Fields[UmbracoExamineIndexer.NodeKeyFieldName], out key))
+                              if (Guid.TryParse(src.Values[UmbracoExamineIndexer.NodeKeyFieldName], out key))
                               {
                                   dest.Key = key;
 
                                   //need to set the UDI
-                                  if (src.Fields.ContainsKey(LuceneIndexer.CategoryFieldName))
+                                  if (src.Values.ContainsKey(LuceneIndex.CategoryFieldName))
                                   {
-                                      switch (src.Fields[LuceneIndexer.CategoryFieldName])
+                                      switch (src.Values[LuceneIndex.CategoryFieldName])
                                       {
                                           case IndexTypes.Member:
                                               dest.Udi = new GuidUdi(Constants.UdiEntityType.Member, dest.Key);
@@ -154,10 +154,10 @@ namespace Umbraco.Web.Models.Mapping
                               }
                           }
 
-                          if (src.Fields.ContainsKey("parentID"))
+                          if (src.Values.ContainsKey("parentID"))
                           {
                               int parentId;
-                              if (int.TryParse(src.Fields["parentID"], out parentId))
+                              if (int.TryParse(src.Values["parentID"], out parentId))
                               {
                                   dest.ParentId = parentId;
                               }
@@ -166,19 +166,19 @@ namespace Umbraco.Web.Models.Mapping
                                   dest.ParentId = -1;
                               }
                           }
-                          dest.Path = src.Fields.ContainsKey(UmbracoExamineIndexer.IndexPathFieldName) ? src.Fields[UmbracoExamineIndexer.IndexPathFieldName] : "";
+                          dest.Path = src.Values.ContainsKey(UmbracoExamineIndexer.IndexPathFieldName) ? src.Values[UmbracoExamineIndexer.IndexPathFieldName] : "";
 
-                          if (src.Fields.ContainsKey(LuceneIndexer.ItemTypeFieldName))
+                          if (src.Values.ContainsKey(LuceneIndex.ItemTypeFieldName))
                           {
-                              dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.ItemTypeFieldName]);
+                              dest.AdditionalData.Add("contentType", src.Values[LuceneIndex.ItemTypeFieldName]);
                           }
                       });
 
-            CreateMap<ISearchResults, IEnumerable<SearchResultItem>>()
-                  .ConvertUsing(results => results.Select(Mapper.Map<SearchResultItem>).ToList());
+            CreateMap<ISearchResults, IEnumerable<SearchResultEntity>>()
+                  .ConvertUsing(results => results.Select(Mapper.Map<SearchResultEntity>).ToList());
 
-            CreateMap<IEnumerable<SearchResult>, IEnumerable<SearchResultItem>>()
-                  .ConvertUsing(results => results.Select(Mapper.Map<SearchResultItem>).ToList());
+            CreateMap<IEnumerable<ISearchResult>, IEnumerable<SearchResultEntity>>()
+                  .ConvertUsing(results => results.Select(Mapper.Map<SearchResultEntity>).ToList());
         }
 
         /// <summary>

src/Umbraco.Web/Mvc/EnsurePublishedContentRequestAttribute.cs

@@ -80,7 +80,7 @@ namespace Umbraco.Web.Mvc
         /// Exposes an UmbracoHelper
         /// </summary>
         protected UmbracoHelper Umbraco => _helper
-            ?? (_helper = new UmbracoHelper(Current.UmbracoContext, Current.Services, Current.ApplicationCache));
+            ?? (_helper = new UmbracoHelper(Current.UmbracoContext, Current.Services));
 
         public override void OnActionExecuted(ActionExecutedContext filterContext)
         {

src/Umbraco.Web/Mvc/PluginController.cs

@@ -82,7 +82,7 @@ namespace Umbraco.Web.Mvc
             get
             {
                 return _umbracoHelper
-                    ?? (_umbracoHelper = new UmbracoHelper(UmbracoContext, Services, ApplicationCache));
+                    ?? (_umbracoHelper = new UmbracoHelper(UmbracoContext, Services));
             }
             internal set // tests
             {