Merge pull request #6038 from umbraco/v8/bugfix/examine-startup

Fixes how examine starts up to make it easier for custom index developers
This commit is contained in:
Bjarke Berg
2019-08-05 10:39:29 +02:00
committed by GitHub
15 changed files with 353 additions and 188 deletions
@@ -54,7 +54,7 @@ namespace Umbraco.Examine
{"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate
{"nodeName", (PublishedValuesOnly //Always add invariant nodeName
? c.PublishName?.Yield()
: c?.Name.Yield()) ?? Enumerable.Empty<string>()},
: c.Name?.Yield()) ?? Enumerable.Empty<string>()},
{"urlName", urlValue?.Yield() ?? Enumerable.Empty<string>()}, //Always add invariant urlName
{"path", c.Path?.Yield() ?? Enumerable.Empty<string>()},
{"nodeType", c.ContentType.Id.ToString().Yield() ?? Enumerable.Empty<string>()},
+25
View File
@@ -28,6 +28,31 @@ namespace Umbraco.Examine
/// </remarks>
internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled);
private static volatile bool _isUnlocked = false;
private static readonly object IsUnlockedLocker = new object();
/// <summary>
/// Unlocks all Lucene based indexes registered with the <see cref="IExamineManager"/>
/// </summary>
/// <remarks>
/// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before
/// either of these happens, we need to configure the indexes.
/// </remarks>
internal static void EnsureUnlocked(this IExamineManager examineManager, IMainDom mainDom, ILogger logger)
{
if (!mainDom.IsMainDom) return;
if (_isUnlocked) return;
lock (IsUnlockedLocker)
{
//double check
if (_isUnlocked) return;
_isUnlocked = true;
examineManager.UnlockLuceneIndexes(logger);
}
}
//TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression
/// <summary>
+2 -1
View File
@@ -5,7 +5,8 @@ using System.Threading.Tasks;
using Examine;
namespace Umbraco.Examine
{
{
/// <summary>
/// Utility to rebuild all indexes ensuring minimal data queries
/// </summary>
@@ -0,0 +1,83 @@
using System.Collections.Generic;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Lucene.Net.Store;
using Umbraco.Core.IO;
using System.Linq;
namespace Umbraco.Examine
{
public class LuceneIndexDiagnostics : IIndexDiagnostics
{
public LuceneIndexDiagnostics(LuceneIndex index, ILogger logger)
{
Index = index;
Logger = logger;
}
public LuceneIndex Index { get; }
public ILogger Logger { get; }
public int DocumentCount
{
get
{
try
{
return Index.GetIndexDocumentCount();
}
catch (AlreadyClosedException)
{
Logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexDocumentCount, the writer is already closed");
return 0;
}
}
}
public int FieldCount
{
get
{
try
{
return Index.GetIndexFieldCount();
}
catch (AlreadyClosedException)
{
Logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexFieldCount, the writer is already closed");
return 0;
}
}
}
public Attempt<string> IsHealthy()
{
var isHealthy = Index.IsHealthy(out var indexError);
return isHealthy ? Attempt<string>.Succeed() : Attempt.Fail(indexError.Message);
}
public virtual IReadOnlyDictionary<string, object> Metadata
{
get
{
var luceneDir = Index.GetLuceneDirectory();
var d = new Dictionary<string, object>
{
[nameof(UmbracoExamineIndex.CommitCount)] = Index.CommitCount,
[nameof(UmbracoExamineIndex.DefaultAnalyzer)] = Index.DefaultAnalyzer.GetType().Name,
["LuceneDirectory"] = luceneDir.GetType().Name
};
if (luceneDir is FSDirectory fsDir)
{
d[nameof(UmbracoExamineIndex.LuceneIndexFolder)] = fsDir.Directory.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/');
}
return d;
}
}
}
}
@@ -72,6 +72,7 @@
<Compile Include="IPublishedContentValueSetBuilder.cs" />
<Compile Include="IUmbracoIndex.cs" />
<Compile Include="IValueSetBuilder.cs" />
<Compile Include="LuceneIndexDiagnostics.cs" />
<Compile Include="MediaIndexPopulator.cs" />
<Compile Include="MediaValueSetBuilder.cs" />
<Compile Include="MemberIndexPopulator.cs" />
@@ -7,73 +7,24 @@ using Umbraco.Core.Logging;
namespace Umbraco.Examine
{
public class UmbracoExamineIndexDiagnostics : IIndexDiagnostics
public class UmbracoExamineIndexDiagnostics : LuceneIndexDiagnostics
{
private readonly UmbracoExamineIndex _index;
private readonly ILogger _logger;
public UmbracoExamineIndexDiagnostics(UmbracoExamineIndex index, ILogger logger)
: base(index, logger)
{
_index = index;
_logger = logger;
}
public int DocumentCount
public override IReadOnlyDictionary<string, object> Metadata
{
get
{
try
{
return _index.GetIndexDocumentCount();
}
catch (AlreadyClosedException)
{
_logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexDocumentCount, the writer is already closed");
return 0;
}
}
}
var d = base.Metadata.ToDictionary(x => x.Key, x => x.Value);
public int FieldCount
{
get
{
try
{
return _index.GetIndexFieldCount();
}
catch (AlreadyClosedException)
{
_logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexFieldCount, the writer is already closed");
return 0;
}
}
}
public Attempt<string> IsHealthy()
{
var isHealthy = _index.IsHealthy(out var indexError);
return isHealthy ? Attempt<string>.Succeed() : Attempt.Fail(indexError.Message);
}
public virtual IReadOnlyDictionary<string, object> Metadata
{
get
{
var d = new Dictionary<string, object>
{
[nameof(UmbracoExamineIndex.CommitCount)] = _index.CommitCount,
[nameof(UmbracoExamineIndex.DefaultAnalyzer)] = _index.DefaultAnalyzer.GetType().Name,
["LuceneDirectory"] = _index.GetLuceneDirectory().GetType().Name,
[nameof(UmbracoExamineIndex.EnableDefaultEventHandler)] = _index.EnableDefaultEventHandler,
[nameof(UmbracoExamineIndex.LuceneIndexFolder)] =
_index.LuceneIndexFolder == null
? string.Empty
: _index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'),
[nameof(UmbracoExamineIndex.PublishedValuesOnly)] = _index.PublishedValuesOnly,
//There's too much info here
//[nameof(UmbracoExamineIndexer.FieldDefinitionCollection)] = _index.FieldDefinitionCollection,
};
d[nameof(UmbracoExamineIndex.EnableDefaultEventHandler)] = _index.EnableDefaultEventHandler;
d[nameof(UmbracoExamineIndex.PublishedValuesOnly)] = _index.PublishedValuesOnly;
if (_index.ValueSetValidator is ValueSetValidator vsv)
{
@@ -0,0 +1,123 @@
using System;
using System.Threading;
using Umbraco.Core.Logging;
using Umbraco.Examine;
using System.Threading.Tasks;
using Umbraco.Core;
using Umbraco.Web.Scheduling;
namespace Umbraco.Web.Search
{
/// <summary>
/// Utility to rebuild all indexes on a background thread
/// </summary>
public sealed class BackgroundIndexRebuilder
{
private static readonly object RebuildLocker = new object();
private readonly IndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
private readonly IProfilingLogger _logger;
private static BackgroundTaskRunner<IBackgroundTask> _rebuildOnStartupRunner;
public BackgroundIndexRebuilder(IMainDom mainDom, IProfilingLogger logger, IndexRebuilder indexRebuilder)
{
_mainDom = mainDom;
_logger = logger;
_indexRebuilder = indexRebuilder;
}
/// <summary>
/// Called to rebuild empty indexes on startup
/// </summary>
/// <param name="indexRebuilder"></param>
/// <param name="logger"></param>
/// <param name="onlyEmptyIndexes"></param>
/// <param name="waitMilliseconds"></param>
public void RebuildIndexes(bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
// TODO: need a way to disable rebuilding on startup
lock (RebuildLocker)
{
if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning)
{
_logger.Warn<BackgroundIndexRebuilder>("Call was made to RebuildIndexes but the task runner for rebuilding is already running");
return;
}
_logger.Info<BackgroundIndexRebuilder>("Starting initialize async background thread.");
//do the rebuild on a managed background thread
var task = new RebuildOnStartupTask(_mainDom, _indexRebuilder, _logger, onlyEmptyIndexes, waitMilliseconds);
_rebuildOnStartupRunner = new BackgroundTaskRunner<IBackgroundTask>(
"RebuildIndexesOnStartup",
_logger);
_rebuildOnStartupRunner.TryAdd(task);
}
}
/// <summary>
/// Background task used to rebuild empty indexes on startup
/// </summary>
private class RebuildOnStartupTask : IBackgroundTask
{
private readonly IMainDom _mainDom;
private readonly IndexRebuilder _indexRebuilder;
private readonly ILogger _logger;
private readonly bool _onlyEmptyIndexes;
private readonly int _waitMilliseconds;
public RebuildOnStartupTask(IMainDom mainDom,
IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
_mainDom = mainDom;
_indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_onlyEmptyIndexes = onlyEmptyIndexes;
_waitMilliseconds = waitMilliseconds;
}
public bool IsAsync => false;
public void Dispose()
{
}
public void Run()
{
try
{
// rebuilds indexes
RebuildIndexes();
}
catch (Exception ex)
{
_logger.Error<RebuildOnStartupTask>(ex, "Failed to rebuild empty indexes.");
}
}
public Task RunAsync(CancellationToken token)
{
throw new NotImplementedException();
}
/// <summary>
/// Used to rebuild indexes on startup or cold boot
/// </summary>
private void RebuildIndexes()
{
//do not attempt to do this if this has been disabled since we are not the main dom.
//this can be called during a cold boot
if (!_mainDom.IsMainDom) return;
if (_waitMilliseconds > 0)
Thread.Sleep(_waitMilliseconds);
_indexRebuilder.ExamineManager.EnsureUnlocked(_mainDom, _logger);
_indexRebuilder.RebuildIndexes(_onlyEmptyIndexes);
}
}
}
}
+11 -131
View File
@@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Cache;
@@ -15,41 +14,35 @@ using Umbraco.Core.Sync;
using Umbraco.Web.Cache;
using Umbraco.Examine;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Web.Scheduling;
using System.Threading.Tasks;
using Examine.LuceneEngine.Directories;
using Umbraco.Core.Composing;
using System.ComponentModel;
namespace Umbraco.Web.Search
{
public sealed class ExamineComponent : IComponent
public sealed class ExamineComponent : Umbraco.Core.Composing.IComponent
{
private readonly IExamineManager _examineManager;
private readonly IContentValueSetBuilder _contentValueSetBuilder;
private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder;
private readonly IValueSetBuilder<IMedia> _mediaValueSetBuilder;
private readonly IValueSetBuilder<IMember> _memberValueSetBuilder;
private static bool _disableExamineIndexing = false;
private static volatile bool _isConfigured = false;
private static readonly object IsConfiguredLocker = new object();
private readonly IScopeProvider _scopeProvider;
private readonly ServiceContext _services;
private static BackgroundTaskRunner<IBackgroundTask> _rebuildOnStartupRunner;
private static readonly object RebuildLocker = new object();
private readonly ServiceContext _services;
private readonly IMainDom _mainDom;
private readonly IProfilingLogger _logger;
private readonly IUmbracoIndexesCreator _indexCreator;
private readonly IndexRebuilder _indexRebuilder;
// the default enlist priority is 100
// enlist with a lower priority to ensure that anything "default" runs after us
// but greater that SafeXmlReaderWriter priority which is 60
private const int EnlistPriority = 80;
public ExamineComponent(IMainDom mainDom,
IExamineManager examineManager, IProfilingLogger profilingLogger,
IScopeProvider scopeProvider, IUmbracoIndexesCreator indexCreator,
IndexRebuilder indexRebuilder, ServiceContext services,
ServiceContext services,
IContentValueSetBuilder contentValueSetBuilder,
IPublishedContentValueSetBuilder publishedContentValueSetBuilder,
IValueSetBuilder<IMedia> mediaValueSetBuilder,
@@ -66,7 +59,6 @@ namespace Umbraco.Web.Search
_mainDom = mainDom;
_logger = profilingLogger;
_indexCreator = indexCreator;
_indexRebuilder = indexRebuilder;
}
public void Initialize()
@@ -95,7 +87,6 @@ namespace Umbraco.Web.Search
//if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled!
Suspendable.ExamineEvents.SuspendIndexers(_logger);
_disableExamineIndexing = true;
return; //exit, do not continue
}
@@ -116,71 +107,17 @@ namespace Umbraco.Web.Search
// bind to distributed cache events - this ensures that this logic occurs on ALL servers
// that are taking part in a load balanced environment.
ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated;
ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated; ;
ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated;
MediaCacheRefresher.CacheUpdated += MediaCacheRefresherUpdated;
MemberCacheRefresher.CacheUpdated += MemberCacheRefresherUpdated;
EnsureUnlocked(_logger, _examineManager);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
RebuildIndexes(_indexRebuilder, _logger, true, 5000);
}
public void Terminate()
{ }
/// <summary>
/// Called to rebuild empty indexes on startup
/// </summary>
/// <param name="indexRebuilder"></param>
/// <param name="logger"></param>
/// <param name="onlyEmptyIndexes"></param>
/// <param name="waitMilliseconds"></param>
public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
// TODO: need a way to disable rebuilding on startup
lock(RebuildLocker)
{
if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning)
{
logger.Warn<ExamineComponent>("Call was made to RebuildIndexes but the task runner for rebuilding is already running");
return;
}
logger.Info<ExamineComponent>("Starting initialize async background thread.");
//do the rebuild on a managed background thread
var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds);
_rebuildOnStartupRunner = new BackgroundTaskRunner<IBackgroundTask>(
"RebuildIndexesOnStartup",
logger);
_rebuildOnStartupRunner.TryAdd(task);
}
}
/// <summary>
/// Must be called to each index is unlocked before any indexing occurs
/// </summary>
/// <remarks>
/// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before
/// either of these happens, we need to configure the indexes.
/// </remarks>
private static void EnsureUnlocked(ILogger logger, IExamineManager examineManager)
{
if (_disableExamineIndexing) return;
if (_isConfigured) return;
lock (IsConfiguredLocker)
{
//double check
if (_isConfigured) return;
_isConfigured = true;
examineManager.UnlockLuceneIndexes(logger);
}
}
[EditorBrowsable(EditorBrowsableState.Never)]
[Obsolete("This method should not be used and will be removed in future versions, rebuilding indexes can be done with the IndexRebuilder or the BackgroundIndexRebuilder")]
public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) => Current.Factory.GetInstance<BackgroundIndexRebuilder>().RebuildIndexes(onlyEmptyIndexes, waitMilliseconds);
#region Cache refresher updated event handlers
@@ -746,63 +683,6 @@ namespace Umbraco.Web.Search
}
#endregion
/// <summary>
/// Background task used to rebuild empty indexes on startup
/// </summary>
private class RebuildOnStartupTask : IBackgroundTask
{
private readonly IndexRebuilder _indexRebuilder;
private readonly ILogger _logger;
private readonly bool _onlyEmptyIndexes;
private readonly int _waitMilliseconds;
public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
_indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_onlyEmptyIndexes = onlyEmptyIndexes;
_waitMilliseconds = waitMilliseconds;
}
public bool IsAsync => false;
public void Dispose()
{
}
public void Run()
{
try
{
// rebuilds indexes
RebuildIndexes();
}
catch (Exception ex)
{
_logger.Error<ExamineComponent>(ex, "Failed to rebuild empty indexes.");
}
}
public Task RunAsync(CancellationToken token)
{
throw new NotImplementedException();
}
/// <summary>
/// Used to rebuild indexes on startup or cold boot
/// </summary>
private void RebuildIndexes()
{
//do not attempt to do this if this has been disabled since we are not the main dom.
//this can be called during a cold boot
if (_disableExamineIndexing) return;
if (_waitMilliseconds > 0)
Thread.Sleep(_waitMilliseconds);
EnsureUnlocked(_logger, _indexRebuilder.ExamineManager);
_indexRebuilder.RebuildIndexes(_onlyEmptyIndexes);
}
}
}
}
@@ -10,6 +10,7 @@ using Umbraco.Examine;
namespace Umbraco.Web.Search
{
/// <summary>
/// Configures and installs Examine.
/// </summary>
@@ -43,6 +44,7 @@ namespace Umbraco.Web.Search
false));
composition.RegisterUnique<IValueSetBuilder<IMedia>, MediaValueSetBuilder>();
composition.RegisterUnique<IValueSetBuilder<IMember>, MemberValueSetBuilder>();
composition.RegisterUnique<BackgroundIndexRebuilder>();
//We want to manage Examine's AppDomain shutdown sequence ourselves so first we'll disable Examine's default behavior
//and then we'll use MainDom to control Examine's shutdown - this MUST be done in Compose ie before ExamineManager
@@ -0,0 +1,42 @@
using Examine;
using Umbraco.Core.Logging;
using Umbraco.Examine;
using Umbraco.Core.Composing;
using Umbraco.Core;
namespace Umbraco.Web.Search
{
/// <summary>
/// Executes after all other examine components have executed
/// </summary>
public sealed class ExamineFinalComponent : IComponent
{
private readonly IProfilingLogger _logger;
private readonly IExamineManager _examineManager;
BackgroundIndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
{
_logger = logger;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_mainDom = mainDom;
}
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
_examineManager.EnsureUnlocked(_mainDom, _logger);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
_indexRebuilder.RebuildIndexes(true, 5000);
}
public void Terminate()
{
}
}
}
@@ -0,0 +1,13 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
namespace Umbraco.Web.Search
{
// examine's final composer composes after all user composers
// and *also* after ICoreComposer (in case IUserComposer is disabled)
[ComposeAfter(typeof(IUserComposer))]
[ComposeAfter(typeof(ICoreComposer))]
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public class ExamineFinalComposer : ComponentComposer<ExamineFinalComponent>
{ }
}
@@ -0,0 +1,37 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
namespace Umbraco.Web.Search
{
/// <summary>
/// An abstract class for custom index authors to inherit from
/// </summary>
public abstract class ExamineUserComponent : IComponent
{
private readonly IMainDom _mainDom;
public ExamineUserComponent(IMainDom mainDom)
{
_mainDom = mainDom;
}
/// <summary>
/// Initialize the component, eagerly exits if ExamineComponent.ExamineEnabled == false
/// </summary>
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
InitializeComponent();
}
/// <summary>
/// Abstract method which executes to initialize this component if ExamineComponent.ExamineEnabled == true
/// </summary>
protected abstract void InitializeComponent();
public virtual void Terminate()
{
}
}
}
@@ -8,6 +8,7 @@ using Umbraco.Examine;
namespace Umbraco.Web.Search
{
/// <summary>
/// Used to return diagnostic data for any index
/// </summary>
+2
View File
@@ -50,6 +50,8 @@ namespace Umbraco.Web
}
}
//This is really needed at all since the only place this is used is in ExamineComponent and that already maintains a flag of whether it suspsended or not
// AHH... but Deploy probably uses this?
public static class ExamineEvents
{
private static bool _tried, _suspended;
+4
View File
@@ -233,6 +233,10 @@
<Compile Include="Routing\IPublishedRouter.cs" />
<Compile Include="Routing\MediaUrlProviderCollection.cs" />
<Compile Include="Routing\MediaUrlProviderCollectionBuilder.cs" />
<Compile Include="Search\BackgroundIndexRebuilder.cs" />
<Compile Include="Search\ExamineFinalComponent.cs" />
<Compile Include="Search\ExamineFinalComposer.cs" />
<Compile Include="Search\ExamineUserComponent.cs" />
<Compile Include="Services\DashboardService.cs" />
<Compile Include="Services\IDashboardService.cs" />
<Compile Include="Models\Link.cs" />