From 9f6a7dec2e474854adc52151078ac4f8de770835 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 2 Aug 2019 00:28:48 +1000 Subject: [PATCH 1/8] Fixes how examine starts up to make it easier for custom index developers --- src/Umbraco.Web/Search/ExamineComponent.cs | 134 ++-------------- src/Umbraco.Web/Search/ExamineComposer.cs | 1 + .../Search/ExamineFinalComponent.cs | 151 ++++++++++++++++++ .../Search/ExamineFinalComposer.cs | 11 ++ .../Search/ExamineUserComponent.cs | 29 ++++ src/Umbraco.Web/Umbraco.Web.csproj | 3 + 6 files changed, 210 insertions(+), 119 deletions(-) create mode 100644 src/Umbraco.Web/Search/ExamineFinalComponent.cs create mode 100644 src/Umbraco.Web/Search/ExamineFinalComposer.cs create mode 100644 src/Umbraco.Web/Search/ExamineUserComponent.cs diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 3583e5b7f9..b468963380 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Threading; using Examine; using Umbraco.Core; using Umbraco.Core.Cache; @@ -15,13 +14,12 @@ using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Examine; using Umbraco.Core.Persistence.DatabaseModelDefinitions; -using Umbraco.Web.Scheduling; -using System.Threading.Tasks; using Examine.LuceneEngine.Directories; using Umbraco.Core.Composing; namespace Umbraco.Web.Search { + public sealed class ExamineComponent : IComponent { private readonly IExamineManager _examineManager; @@ -30,26 +28,31 @@ namespace Umbraco.Web.Search private readonly IValueSetBuilder _mediaValueSetBuilder; private readonly IValueSetBuilder _memberValueSetBuilder; private static bool _disableExamineIndexing = false; - private static volatile bool _isConfigured = false; - private static readonly object IsConfiguredLocker = new object(); private readonly IScopeProvider _scopeProvider; - private readonly ServiceContext _services; - private static BackgroundTaskRunner _rebuildOnStartupRunner; - private static readonly object RebuildLocker = new object(); + private readonly ServiceContext _services; private readonly IMainDom _mainDom; private readonly IProfilingLogger _logger; private readonly IUmbracoIndexesCreator _indexCreator; - private readonly IndexRebuilder _indexRebuilder; + // the default enlist priority is 100 // enlist with a lower priority to ensure that anything "default" runs after us // but greater that SafeXmlReaderWriter priority which is 60 private const int EnlistPriority = 80; + /// + /// Returns true if Examine is enabled for this AppDomain + /// + /// + /// This flag should be used by custom Examine index implementors to determine if they should + /// execute. This value is true if this component successfully registered with + /// + public static bool ExamineEnabled => !_disableExamineIndexing; + public ExamineComponent(IMainDom mainDom, IExamineManager examineManager, IProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesCreator indexCreator, - IndexRebuilder indexRebuilder, ServiceContext services, + ServiceContext services, IContentValueSetBuilder contentValueSetBuilder, IPublishedContentValueSetBuilder publishedContentValueSetBuilder, IValueSetBuilder mediaValueSetBuilder, @@ -66,7 +69,6 @@ namespace Umbraco.Web.Search _mainDom = mainDom; _logger = profilingLogger; _indexCreator = indexCreator; - _indexRebuilder = indexRebuilder; } public void Initialize() @@ -119,11 +121,6 @@ namespace Umbraco.Web.Search ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated; ; MediaCacheRefresher.CacheUpdated += MediaCacheRefresherUpdated; MemberCacheRefresher.CacheUpdated += MemberCacheRefresherUpdated; - - EnsureUnlocked(_logger, _examineManager); - - // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? - RebuildIndexes(_indexRebuilder, _logger, true, 5000); } public void Terminate() @@ -136,51 +133,7 @@ namespace Umbraco.Web.Search /// /// /// - public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) - { - // TODO: need a way to disable rebuilding on startup - - lock(RebuildLocker) - { - if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning) - { - logger.Warn("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); - return; - } - - logger.Info("Starting initialize async background thread."); - //do the rebuild on a managed background thread - var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); - - _rebuildOnStartupRunner = new BackgroundTaskRunner( - "RebuildIndexesOnStartup", - logger); - - _rebuildOnStartupRunner.TryAdd(task); - } - } - - /// - /// Must be called to each index is unlocked before any indexing occurs - /// - /// - /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before - /// either of these happens, we need to configure the indexes. - /// - private static void EnsureUnlocked(ILogger logger, IExamineManager examineManager) - { - if (_disableExamineIndexing) return; - if (_isConfigured) return; - - lock (IsConfiguredLocker) - { - //double check - if (_isConfigured) return; - - _isConfigured = true; - examineManager.UnlockLuceneIndexes(logger); - } - } + public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) => ExamineFinalComponent.RebuildIndexes(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); #region Cache refresher updated event handlers @@ -746,63 +699,6 @@ namespace Umbraco.Web.Search } #endregion - /// - /// Background task used to rebuild empty indexes on startup - /// - private class RebuildOnStartupTask : IBackgroundTask - { - private readonly IndexRebuilder _indexRebuilder; - private readonly ILogger _logger; - private readonly bool _onlyEmptyIndexes; - private readonly int _waitMilliseconds; - - public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) - { - _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _onlyEmptyIndexes = onlyEmptyIndexes; - _waitMilliseconds = waitMilliseconds; - } - - public bool IsAsync => false; - - public void Dispose() - { - } - - public void Run() - { - try - { - // rebuilds indexes - RebuildIndexes(); - } - catch (Exception ex) - { - _logger.Error(ex, "Failed to rebuild empty indexes."); - } - } - - public Task RunAsync(CancellationToken token) - { - throw new NotImplementedException(); - } - - /// - /// Used to rebuild indexes on startup or cold boot - /// - private void RebuildIndexes() - { - //do not attempt to do this if this has been disabled since we are not the main dom. - //this can be called during a cold boot - if (_disableExamineIndexing) return; - - if (_waitMilliseconds > 0) - Thread.Sleep(_waitMilliseconds); - - EnsureUnlocked(_logger, _indexRebuilder.ExamineManager); - _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); - } - } + } } diff --git a/src/Umbraco.Web/Search/ExamineComposer.cs b/src/Umbraco.Web/Search/ExamineComposer.cs index 6e74f0e89d..de33afe1cb 100644 --- a/src/Umbraco.Web/Search/ExamineComposer.cs +++ b/src/Umbraco.Web/Search/ExamineComposer.cs @@ -10,6 +10,7 @@ using Umbraco.Examine; namespace Umbraco.Web.Search { + /// /// Configures and installs Examine. /// diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs new file mode 100644 index 0000000000..5a13185cdc --- /dev/null +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -0,0 +1,151 @@ +using System; +using System.Threading; +using Examine; +using Umbraco.Core.Logging; +using Umbraco.Examine; +using Umbraco.Web.Scheduling; +using System.Threading.Tasks; +using Umbraco.Core.Composing; + +namespace Umbraco.Web.Search +{ + /// + /// Executes after all other examine components have executed + /// + public sealed class ExamineFinalComponent : IComponent + { + private static volatile bool _isConfigured = false; + private static readonly object IsConfiguredLocker = new object(); + private readonly IProfilingLogger _logger; + private readonly IExamineManager _examineManager; + private static readonly object RebuildLocker = new object(); + private readonly IndexRebuilder _indexRebuilder; + private static BackgroundTaskRunner _rebuildOnStartupRunner; + + public void Initialize() + { + if (!ExamineComponent.ExamineEnabled) return; + + EnsureUnlocked(_logger, _examineManager); + + // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? + ExamineComponent.RebuildIndexes(_indexRebuilder, _logger, true, 5000); + } + + public void Terminate() + { + } + + /// + /// Called to rebuild empty indexes on startup + /// + /// + /// + /// + /// + internal static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) + { + // TODO: need a way to disable rebuilding on startup + + lock (RebuildLocker) + { + if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning) + { + logger.Warn("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); + return; + } + + logger.Info("Starting initialize async background thread."); + //do the rebuild on a managed background thread + var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); + + _rebuildOnStartupRunner = new BackgroundTaskRunner( + "RebuildIndexesOnStartup", + logger); + + _rebuildOnStartupRunner.TryAdd(task); + } + } + + /// + /// Must be called to each index is unlocked before any indexing occurs + /// + /// + /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before + /// either of these happens, we need to configure the indexes. + /// + internal static void EnsureUnlocked(ILogger logger, IExamineManager examineManager) + { + if (!ExamineComponent.ExamineEnabled) return; + if (_isConfigured) return; + + lock (IsConfiguredLocker) + { + //double check + if (_isConfigured) return; + + _isConfigured = true; + examineManager.UnlockLuceneIndexes(logger); + } + } + + /// + /// Background task used to rebuild empty indexes on startup + /// + private class RebuildOnStartupTask : IBackgroundTask + { + private readonly IndexRebuilder _indexRebuilder; + private readonly ILogger _logger; + private readonly bool _onlyEmptyIndexes; + private readonly int _waitMilliseconds; + + public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) + { + _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _onlyEmptyIndexes = onlyEmptyIndexes; + _waitMilliseconds = waitMilliseconds; + } + + public bool IsAsync => false; + + public void Dispose() + { + } + + public void Run() + { + try + { + // rebuilds indexes + RebuildIndexes(); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to rebuild empty indexes."); + } + } + + public Task RunAsync(CancellationToken token) + { + throw new NotImplementedException(); + } + + /// + /// Used to rebuild indexes on startup or cold boot + /// + private void RebuildIndexes() + { + //do not attempt to do this if this has been disabled since we are not the main dom. + //this can be called during a cold boot + if (!ExamineComponent.ExamineEnabled) return; + + if (_waitMilliseconds > 0) + Thread.Sleep(_waitMilliseconds); + + EnsureUnlocked(_logger, _indexRebuilder.ExamineManager); + _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); + } + } + } +} diff --git a/src/Umbraco.Web/Search/ExamineFinalComposer.cs b/src/Umbraco.Web/Search/ExamineFinalComposer.cs new file mode 100644 index 0000000000..6b33459159 --- /dev/null +++ b/src/Umbraco.Web/Search/ExamineFinalComposer.cs @@ -0,0 +1,11 @@ +using Umbraco.Core.Composing; + +namespace Umbraco.Web.Search +{ + // examine's final composer composes after all user composers + // and *also* after ICoreComposer (in case IUserComposer is disabled) + [ComposeAfter(typeof(IUserComposer))] + [ComposeAfter(typeof(ICoreComposer))] + public class ExamineFinalComposer : ComponentComposer + { } +} diff --git a/src/Umbraco.Web/Search/ExamineUserComponent.cs b/src/Umbraco.Web/Search/ExamineUserComponent.cs new file mode 100644 index 0000000000..ddee07543c --- /dev/null +++ b/src/Umbraco.Web/Search/ExamineUserComponent.cs @@ -0,0 +1,29 @@ +using Umbraco.Core.Composing; + +namespace Umbraco.Web.Search +{ + /// + /// An abstract class for custom index authors to inherit from + /// + public abstract class ExamineUserComponent : IComponent + { + /// + /// Initialize the component, eagerly exits if ExamineComponent.ExamineEnabled == false + /// + public void Initialize() + { + if (!ExamineComponent.ExamineEnabled) return; + + InitializeComponent(); + } + + /// + /// Abstract method which executes to initialize this component if ExamineComponent.ExamineEnabled == true + /// + protected abstract void InitializeComponent(); + + public virtual void Terminate() + { + } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index dcaa3494fd..c9c9a4412a 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -233,6 +233,9 @@ + + + From 366058581a8fc11697f92636a3961fa7d4cbb2d5 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 2 Aug 2019 01:15:08 +1000 Subject: [PATCH 2/8] Adds LuceneIndexDiagnostics to make it easier for devs and custom indexes --- src/Umbraco.Examine/LuceneIndexDiagnostics.cs | 80 +++++++++++++++++++ src/Umbraco.Examine/Umbraco.Examine.csproj | 1 + .../UmbracoExamineIndexDiagnostics.cs | 62 ++------------ src/Umbraco.Web/Search/ExamineComponent.cs | 2 +- .../Search/GenericIndexDiagnostics.cs | 1 + src/Umbraco.Web/Suspendable.cs | 2 + 6 files changed, 91 insertions(+), 57 deletions(-) create mode 100644 src/Umbraco.Examine/LuceneIndexDiagnostics.cs diff --git a/src/Umbraco.Examine/LuceneIndexDiagnostics.cs b/src/Umbraco.Examine/LuceneIndexDiagnostics.cs new file mode 100644 index 0000000000..03e5018a79 --- /dev/null +++ b/src/Umbraco.Examine/LuceneIndexDiagnostics.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using Examine.LuceneEngine.Providers; +using Umbraco.Core; +using Umbraco.Core.Logging; +using Lucene.Net.Store; +using Umbraco.Core.IO; + +namespace Umbraco.Examine +{ + public class LuceneIndexDiagnostics : IIndexDiagnostics + { + public LuceneIndexDiagnostics(LuceneIndex index, ILogger logger) + { + Index = index; + Logger = logger; + } + + public LuceneIndex Index { get; } + public ILogger Logger { get; } + + public int DocumentCount + { + get + { + try + { + return Index.GetIndexDocumentCount(); + } + catch (AlreadyClosedException) + { + Logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexDocumentCount, the writer is already closed"); + return 0; + } + } + } + + public int FieldCount + { + get + { + try + { + return Index.GetIndexFieldCount(); + } + catch (AlreadyClosedException) + { + Logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexFieldCount, the writer is already closed"); + return 0; + } + } + } + + public Attempt IsHealthy() + { + var isHealthy = Index.IsHealthy(out var indexError); + return isHealthy ? Attempt.Succeed() : Attempt.Fail(indexError.Message); + } + + public virtual IReadOnlyDictionary Metadata + { + get + { + var d = new Dictionary + { + [nameof(UmbracoExamineIndex.CommitCount)] = Index.CommitCount, + [nameof(UmbracoExamineIndex.DefaultAnalyzer)] = Index.DefaultAnalyzer.GetType().Name, + ["LuceneDirectory"] = Index.GetLuceneDirectory().GetType().Name, + [nameof(UmbracoExamineIndex.LuceneIndexFolder)] = + Index.LuceneIndexFolder == null + ? string.Empty + : Index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'), + }; + + return d; + } + } + + + } +} diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 95690c17e4..e28a8e674e 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -72,6 +72,7 @@ + diff --git a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs index fed5b9bae7..8be46869ac 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs @@ -7,73 +7,23 @@ using Umbraco.Core.Logging; namespace Umbraco.Examine { - public class UmbracoExamineIndexDiagnostics : IIndexDiagnostics + public class UmbracoExamineIndexDiagnostics : LuceneIndexDiagnostics { private readonly UmbracoExamineIndex _index; - private readonly ILogger _logger; public UmbracoExamineIndexDiagnostics(UmbracoExamineIndex index, ILogger logger) + : base(index, logger) { - _index = index; - _logger = logger; } - public int DocumentCount + public override IReadOnlyDictionary Metadata { get { - try - { - return _index.GetIndexDocumentCount(); - } - catch (AlreadyClosedException) - { - _logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexDocumentCount, the writer is already closed"); - return 0; - } - } - } + var d = base.Metadata.ToDictionary(x => x.Key, x => x.Value); - public int FieldCount - { - get - { - try - { - return _index.GetIndexFieldCount(); - } - catch (AlreadyClosedException) - { - _logger.Warn(typeof(UmbracoContentIndex), "Cannot get GetIndexFieldCount, the writer is already closed"); - return 0; - } - } - } - - public Attempt IsHealthy() - { - var isHealthy = _index.IsHealthy(out var indexError); - return isHealthy ? Attempt.Succeed() : Attempt.Fail(indexError.Message); - } - - public virtual IReadOnlyDictionary Metadata - { - get - { - var d = new Dictionary - { - [nameof(UmbracoExamineIndex.CommitCount)] = _index.CommitCount, - [nameof(UmbracoExamineIndex.DefaultAnalyzer)] = _index.DefaultAnalyzer.GetType().Name, - ["LuceneDirectory"] = _index.GetLuceneDirectory().GetType().Name, - [nameof(UmbracoExamineIndex.EnableDefaultEventHandler)] = _index.EnableDefaultEventHandler, - [nameof(UmbracoExamineIndex.LuceneIndexFolder)] = - _index.LuceneIndexFolder == null - ? string.Empty - : _index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'), - [nameof(UmbracoExamineIndex.PublishedValuesOnly)] = _index.PublishedValuesOnly, - //There's too much info here - //[nameof(UmbracoExamineIndexer.FieldDefinitionCollection)] = _index.FieldDefinitionCollection, - }; + d[nameof(UmbracoExamineIndex.EnableDefaultEventHandler)] = _index.EnableDefaultEventHandler; + d[nameof(UmbracoExamineIndex.PublishedValuesOnly)] = _index.PublishedValuesOnly; if (_index.ValueSetValidator is ValueSetValidator vsv) { diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index b468963380..5d33bfb169 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -118,7 +118,7 @@ namespace Umbraco.Web.Search // bind to distributed cache events - this ensures that this logic occurs on ALL servers // that are taking part in a load balanced environment. ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated; - ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated; ; + ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated; MediaCacheRefresher.CacheUpdated += MediaCacheRefresherUpdated; MemberCacheRefresher.CacheUpdated += MemberCacheRefresherUpdated; } diff --git a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs b/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs index 560fb19f09..cb25e1242a 100644 --- a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs +++ b/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs @@ -8,6 +8,7 @@ using Umbraco.Examine; namespace Umbraco.Web.Search { + /// /// Used to return diagnostic data for any index /// diff --git a/src/Umbraco.Web/Suspendable.cs b/src/Umbraco.Web/Suspendable.cs index 86c83120ea..cfe4d28e8b 100644 --- a/src/Umbraco.Web/Suspendable.cs +++ b/src/Umbraco.Web/Suspendable.cs @@ -50,6 +50,8 @@ namespace Umbraco.Web } } + //This is really needed at all since the only place this is used is in ExamineComponent and that already maintains a flag of whether it suspsended or not + // AHH... but Deploy probably uses this? public static class ExamineEvents { private static bool _tried, _suspended; From 14915e9da788570a1626a0fe8c1a2de7b29c49de Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 2 Aug 2019 01:16:37 +1000 Subject: [PATCH 3/8] oops --- src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs index 8be46869ac..4a926deebe 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs @@ -14,6 +14,7 @@ namespace Umbraco.Examine public UmbracoExamineIndexDiagnostics(UmbracoExamineIndex index, ILogger logger) : base(index, logger) { + _index = index; } public override IReadOnlyDictionary Metadata From 86a97c30d193d92c67c9fe917c17e2a783449d51 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 2 Aug 2019 12:40:45 +1000 Subject: [PATCH 4/8] fixes up LuceneIndexDiagnostics --- src/Umbraco.Examine/LuceneIndexDiagnostics.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Examine/LuceneIndexDiagnostics.cs b/src/Umbraco.Examine/LuceneIndexDiagnostics.cs index 03e5018a79..96363904b4 100644 --- a/src/Umbraco.Examine/LuceneIndexDiagnostics.cs +++ b/src/Umbraco.Examine/LuceneIndexDiagnostics.cs @@ -4,6 +4,7 @@ using Umbraco.Core; using Umbraco.Core.Logging; using Lucene.Net.Store; using Umbraco.Core.IO; +using System.Linq; namespace Umbraco.Examine { @@ -60,17 +61,19 @@ namespace Umbraco.Examine { get { + var luceneDir = Index.GetLuceneDirectory(); var d = new Dictionary { [nameof(UmbracoExamineIndex.CommitCount)] = Index.CommitCount, [nameof(UmbracoExamineIndex.DefaultAnalyzer)] = Index.DefaultAnalyzer.GetType().Name, - ["LuceneDirectory"] = Index.GetLuceneDirectory().GetType().Name, - [nameof(UmbracoExamineIndex.LuceneIndexFolder)] = - Index.LuceneIndexFolder == null - ? string.Empty - : Index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'), + ["LuceneDirectory"] = luceneDir.GetType().Name }; + if (luceneDir is FSDirectory fsDir) + { + d[nameof(UmbracoExamineIndex.LuceneIndexFolder)] = fsDir.Directory.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'); + } + return d; } } From 345e6091d2d09f3c83d9d26044e0ace9f22cd5d4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 2 Aug 2019 13:01:02 +1000 Subject: [PATCH 5/8] fixes null check, was on the wrong element --- src/Umbraco.Examine/ContentValueSetBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Examine/ContentValueSetBuilder.cs b/src/Umbraco.Examine/ContentValueSetBuilder.cs index 44cef08813..9cbc311639 100644 --- a/src/Umbraco.Examine/ContentValueSetBuilder.cs +++ b/src/Umbraco.Examine/ContentValueSetBuilder.cs @@ -54,7 +54,7 @@ namespace Umbraco.Examine {"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate {"nodeName", (PublishedValuesOnly //Always add invariant nodeName ? c.PublishName?.Yield() - : c?.Name.Yield()) ?? Enumerable.Empty()}, + : c.Name?.Yield()) ?? Enumerable.Empty()}, {"urlName", urlValue?.Yield() ?? Enumerable.Empty()}, //Always add invariant urlName {"path", c.Path?.Yield() ?? Enumerable.Empty()}, {"nodeType", c.ContentType.Id.ToString().Yield() ?? Enumerable.Empty()}, From 89429bd65a3377b8b86ffd284b44ac2483693a64 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 5 Aug 2019 16:28:23 +1000 Subject: [PATCH 6/8] fixes ctor --- src/Umbraco.Web/Search/ExamineFinalComponent.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs index 5a13185cdc..4213efac34 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComponent.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -14,6 +14,7 @@ namespace Umbraco.Web.Search /// public sealed class ExamineFinalComponent : IComponent { + private static volatile bool _isConfigured = false; private static readonly object IsConfiguredLocker = new object(); private readonly IProfilingLogger _logger; @@ -22,6 +23,13 @@ namespace Umbraco.Web.Search private readonly IndexRebuilder _indexRebuilder; private static BackgroundTaskRunner _rebuildOnStartupRunner; + public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, IndexRebuilder indexRebuilder) + { + _logger = logger; + _examineManager = examineManager; + _indexRebuilder = indexRebuilder; + } + public void Initialize() { if (!ExamineComponent.ExamineEnabled) return; From be4536cf164dd5a27c2046603319409b8a7b9eb3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 5 Aug 2019 17:14:29 +1000 Subject: [PATCH 7/8] Refactors ugly statics and nested classes --- src/Umbraco.Examine/ExamineExtensions.cs | 25 ++++ src/Umbraco.Examine/IndexRebuilder.cs | 3 +- .../Search/BackgroundIndexRebuilder.cs | 123 +++++++++++++++ src/Umbraco.Web/Search/ExamineComponent.cs | 28 +--- src/Umbraco.Web/Search/ExamineComposer.cs | 1 + .../Search/ExamineFinalComponent.cs | 141 ++---------------- .../Search/ExamineUserComponent.cs | 12 +- src/Umbraco.Web/Umbraco.Web.csproj | 1 + 8 files changed, 180 insertions(+), 154 deletions(-) create mode 100644 src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs diff --git a/src/Umbraco.Examine/ExamineExtensions.cs b/src/Umbraco.Examine/ExamineExtensions.cs index 376950c95e..ae6913f4ee 100644 --- a/src/Umbraco.Examine/ExamineExtensions.cs +++ b/src/Umbraco.Examine/ExamineExtensions.cs @@ -28,6 +28,31 @@ namespace Umbraco.Examine /// internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled); + private static volatile bool _isUnlocked = false; + private static readonly object IsUnlockedLocker = new object(); + + /// + /// Unlocks all Lucene based indexes registered with the + /// + /// + /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before + /// either of these happens, we need to configure the indexes. + /// + internal static void EnsureUnlocked(this IExamineManager examineManager, IMainDom mainDom, ILogger logger) + { + if (!mainDom.IsMainDom) return; + if (_isUnlocked) return; + + lock (IsUnlockedLocker) + { + //double check + if (_isUnlocked) return; + + _isUnlocked = true; + examineManager.UnlockLuceneIndexes(logger); + } + } + //TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression /// diff --git a/src/Umbraco.Examine/IndexRebuilder.cs b/src/Umbraco.Examine/IndexRebuilder.cs index 43c309b9c5..786aecac71 100644 --- a/src/Umbraco.Examine/IndexRebuilder.cs +++ b/src/Umbraco.Examine/IndexRebuilder.cs @@ -5,7 +5,8 @@ using System.Threading.Tasks; using Examine; namespace Umbraco.Examine -{ +{ + /// /// Utility to rebuild all indexes ensuring minimal data queries /// diff --git a/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs b/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs new file mode 100644 index 0000000000..d747d8d9e4 --- /dev/null +++ b/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs @@ -0,0 +1,123 @@ +using System; +using System.Threading; +using Umbraco.Core.Logging; +using Umbraco.Examine; +using System.Threading.Tasks; +using Umbraco.Core; +using Umbraco.Web.Scheduling; + +namespace Umbraco.Web.Search +{ + /// + /// Utility to rebuild all indexes on a background thread + /// + public sealed class BackgroundIndexRebuilder + { + private static readonly object RebuildLocker = new object(); + private readonly IndexRebuilder _indexRebuilder; + private readonly IMainDom _mainDom; + private readonly IProfilingLogger _logger; + private static BackgroundTaskRunner _rebuildOnStartupRunner; + + public BackgroundIndexRebuilder(IMainDom mainDom, IProfilingLogger logger, IndexRebuilder indexRebuilder) + { + _mainDom = mainDom; + _logger = logger; + _indexRebuilder = indexRebuilder; + } + + /// + /// Called to rebuild empty indexes on startup + /// + /// + /// + /// + /// + public void RebuildIndexes(bool onlyEmptyIndexes, int waitMilliseconds = 0) + { + // TODO: need a way to disable rebuilding on startup + + lock (RebuildLocker) + { + if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning) + { + _logger.Warn("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); + return; + } + + _logger.Info("Starting initialize async background thread."); + //do the rebuild on a managed background thread + var task = new RebuildOnStartupTask(_mainDom, _indexRebuilder, _logger, onlyEmptyIndexes, waitMilliseconds); + + _rebuildOnStartupRunner = new BackgroundTaskRunner( + "RebuildIndexesOnStartup", + _logger); + + _rebuildOnStartupRunner.TryAdd(task); + } + } + + /// + /// Background task used to rebuild empty indexes on startup + /// + private class RebuildOnStartupTask : IBackgroundTask + { + private readonly IMainDom _mainDom; + + private readonly IndexRebuilder _indexRebuilder; + private readonly ILogger _logger; + private readonly bool _onlyEmptyIndexes; + private readonly int _waitMilliseconds; + + public RebuildOnStartupTask(IMainDom mainDom, + IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) + { + _mainDom = mainDom; + _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _onlyEmptyIndexes = onlyEmptyIndexes; + _waitMilliseconds = waitMilliseconds; + } + + public bool IsAsync => false; + + public void Dispose() + { + } + + public void Run() + { + try + { + // rebuilds indexes + RebuildIndexes(); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to rebuild empty indexes."); + } + } + + public Task RunAsync(CancellationToken token) + { + throw new NotImplementedException(); + } + + /// + /// Used to rebuild indexes on startup or cold boot + /// + private void RebuildIndexes() + { + //do not attempt to do this if this has been disabled since we are not the main dom. + //this can be called during a cold boot + if (!_mainDom.IsMainDom) return; + + if (_waitMilliseconds > 0) + Thread.Sleep(_waitMilliseconds); + + _indexRebuilder.ExamineManager.EnsureUnlocked(_mainDom, _logger); + _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); + } + } + } +} diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 5d33bfb169..fd1eee8f96 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -16,18 +16,17 @@ using Umbraco.Examine; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Examine.LuceneEngine.Directories; using Umbraco.Core.Composing; +using System.ComponentModel; namespace Umbraco.Web.Search { - - public sealed class ExamineComponent : IComponent + public sealed class ExamineComponent : Umbraco.Core.Composing.IComponent { private readonly IExamineManager _examineManager; private readonly IContentValueSetBuilder _contentValueSetBuilder; private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; private readonly IValueSetBuilder _mediaValueSetBuilder; private readonly IValueSetBuilder _memberValueSetBuilder; - private static bool _disableExamineIndexing = false; private readonly IScopeProvider _scopeProvider; private readonly ServiceContext _services; private readonly IMainDom _mainDom; @@ -39,16 +38,7 @@ namespace Umbraco.Web.Search // enlist with a lower priority to ensure that anything "default" runs after us // but greater that SafeXmlReaderWriter priority which is 60 private const int EnlistPriority = 80; - - /// - /// Returns true if Examine is enabled for this AppDomain - /// - /// - /// This flag should be used by custom Examine index implementors to determine if they should - /// execute. This value is true if this component successfully registered with - /// - public static bool ExamineEnabled => !_disableExamineIndexing; - + public ExamineComponent(IMainDom mainDom, IExamineManager examineManager, IProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesCreator indexCreator, @@ -97,7 +87,6 @@ namespace Umbraco.Web.Search //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! Suspendable.ExamineEvents.SuspendIndexers(_logger); - _disableExamineIndexing = true; return; //exit, do not continue } @@ -126,14 +115,9 @@ namespace Umbraco.Web.Search public void Terminate() { } - /// - /// Called to rebuild empty indexes on startup - /// - /// - /// - /// - /// - public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) => ExamineFinalComponent.RebuildIndexes(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method should not be used and will be removed in future versions, rebuilding indexes can be done with the IndexRebuilder or the BackgroundIndexRebuilder")] + public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) => Current.Factory.GetInstance().RebuildIndexes(onlyEmptyIndexes, waitMilliseconds); #region Cache refresher updated event handlers diff --git a/src/Umbraco.Web/Search/ExamineComposer.cs b/src/Umbraco.Web/Search/ExamineComposer.cs index de33afe1cb..0ade432d70 100644 --- a/src/Umbraco.Web/Search/ExamineComposer.cs +++ b/src/Umbraco.Web/Search/ExamineComposer.cs @@ -44,6 +44,7 @@ namespace Umbraco.Web.Search false)); composition.RegisterUnique, MediaValueSetBuilder>(); composition.RegisterUnique, MemberValueSetBuilder>(); + composition.RegisterUnique(); //We want to manage Examine's AppDomain shutdown sequence ourselves so first we'll disable Examine's default behavior //and then we'll use MainDom to control Examine's shutdown - this MUST be done in Compose ie before ExamineManager diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs index 4213efac34..4514a78faa 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComponent.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -1,159 +1,42 @@ -using System; -using System.Threading; -using Examine; +using Examine; using Umbraco.Core.Logging; using Umbraco.Examine; -using Umbraco.Web.Scheduling; -using System.Threading.Tasks; using Umbraco.Core.Composing; +using Umbraco.Core; namespace Umbraco.Web.Search { + /// /// Executes after all other examine components have executed /// public sealed class ExamineFinalComponent : IComponent - { - - private static volatile bool _isConfigured = false; - private static readonly object IsConfiguredLocker = new object(); + { private readonly IProfilingLogger _logger; private readonly IExamineManager _examineManager; - private static readonly object RebuildLocker = new object(); - private readonly IndexRebuilder _indexRebuilder; - private static BackgroundTaskRunner _rebuildOnStartupRunner; - - public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, IndexRebuilder indexRebuilder) + BackgroundIndexRebuilder _indexRebuilder; + private readonly IMainDom _mainDom; + + public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom) { _logger = logger; _examineManager = examineManager; _indexRebuilder = indexRebuilder; + _mainDom = mainDom; } public void Initialize() { - if (!ExamineComponent.ExamineEnabled) return; + if (!_mainDom.IsMainDom) return; - EnsureUnlocked(_logger, _examineManager); + _examineManager.EnsureUnlocked(_mainDom, _logger); // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? - ExamineComponent.RebuildIndexes(_indexRebuilder, _logger, true, 5000); + _indexRebuilder.RebuildIndexes(true, 5000); } public void Terminate() { } - - /// - /// Called to rebuild empty indexes on startup - /// - /// - /// - /// - /// - internal static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) - { - // TODO: need a way to disable rebuilding on startup - - lock (RebuildLocker) - { - if (_rebuildOnStartupRunner != null && _rebuildOnStartupRunner.IsRunning) - { - logger.Warn("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); - return; - } - - logger.Info("Starting initialize async background thread."); - //do the rebuild on a managed background thread - var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); - - _rebuildOnStartupRunner = new BackgroundTaskRunner( - "RebuildIndexesOnStartup", - logger); - - _rebuildOnStartupRunner.TryAdd(task); - } - } - - /// - /// Must be called to each index is unlocked before any indexing occurs - /// - /// - /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before - /// either of these happens, we need to configure the indexes. - /// - internal static void EnsureUnlocked(ILogger logger, IExamineManager examineManager) - { - if (!ExamineComponent.ExamineEnabled) return; - if (_isConfigured) return; - - lock (IsConfiguredLocker) - { - //double check - if (_isConfigured) return; - - _isConfigured = true; - examineManager.UnlockLuceneIndexes(logger); - } - } - - /// - /// Background task used to rebuild empty indexes on startup - /// - private class RebuildOnStartupTask : IBackgroundTask - { - private readonly IndexRebuilder _indexRebuilder; - private readonly ILogger _logger; - private readonly bool _onlyEmptyIndexes; - private readonly int _waitMilliseconds; - - public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) - { - _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _onlyEmptyIndexes = onlyEmptyIndexes; - _waitMilliseconds = waitMilliseconds; - } - - public bool IsAsync => false; - - public void Dispose() - { - } - - public void Run() - { - try - { - // rebuilds indexes - RebuildIndexes(); - } - catch (Exception ex) - { - _logger.Error(ex, "Failed to rebuild empty indexes."); - } - } - - public Task RunAsync(CancellationToken token) - { - throw new NotImplementedException(); - } - - /// - /// Used to rebuild indexes on startup or cold boot - /// - private void RebuildIndexes() - { - //do not attempt to do this if this has been disabled since we are not the main dom. - //this can be called during a cold boot - if (!ExamineComponent.ExamineEnabled) return; - - if (_waitMilliseconds > 0) - Thread.Sleep(_waitMilliseconds); - - EnsureUnlocked(_logger, _indexRebuilder.ExamineManager); - _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); - } - } } } diff --git a/src/Umbraco.Web/Search/ExamineUserComponent.cs b/src/Umbraco.Web/Search/ExamineUserComponent.cs index ddee07543c..35bc3f59ea 100644 --- a/src/Umbraco.Web/Search/ExamineUserComponent.cs +++ b/src/Umbraco.Web/Search/ExamineUserComponent.cs @@ -1,4 +1,5 @@ -using Umbraco.Core.Composing; +using Umbraco.Core; +using Umbraco.Core.Composing; namespace Umbraco.Web.Search { @@ -7,12 +8,19 @@ namespace Umbraco.Web.Search /// public abstract class ExamineUserComponent : IComponent { + private readonly IMainDom _mainDom; + + public ExamineUserComponent(IMainDom mainDom) + { + _mainDom = mainDom; + } + /// /// Initialize the component, eagerly exits if ExamineComponent.ExamineEnabled == false /// public void Initialize() { - if (!ExamineComponent.ExamineEnabled) return; + if (!_mainDom.IsMainDom) return; InitializeComponent(); } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index c9c9a4412a..2365017504 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -233,6 +233,7 @@ + From e4f346620ee051a17357da7ece6ba65500620505 Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Mon, 5 Aug 2019 10:38:35 +0200 Subject: [PATCH 8/8] Added runtime level to ExamineFinalComposer --- src/Umbraco.Web/Search/ExamineFinalComposer.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/Search/ExamineFinalComposer.cs b/src/Umbraco.Web/Search/ExamineFinalComposer.cs index 6b33459159..5b6334f1f6 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComposer.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComposer.cs @@ -1,4 +1,5 @@ -using Umbraco.Core.Composing; +using Umbraco.Core; +using Umbraco.Core.Composing; namespace Umbraco.Web.Search { @@ -6,6 +7,7 @@ namespace Umbraco.Web.Search // and *also* after ICoreComposer (in case IUserComposer is disabled) [ComposeAfter(typeof(IUserComposer))] [ComposeAfter(typeof(ICoreComposer))] + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] public class ExamineFinalComposer : ComponentComposer { } }