diff --git a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs b/src/Umbraco.Abstractions/Search/IUmbracoTreeSearcherFields.cs similarity index 99% rename from src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs rename to src/Umbraco.Abstractions/Search/IUmbracoTreeSearcherFields.cs index c5a6c53d19..fdbf5bf60d 100644 --- a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs +++ b/src/Umbraco.Abstractions/Search/IUmbracoTreeSearcherFields.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; namespace Umbraco.Web.Search -{ +{ /// /// Used to propagate hardcoded internal Field lists /// diff --git a/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs new file mode 100644 index 0000000000..71d192146b --- /dev/null +++ b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs @@ -0,0 +1,346 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Examine; +using Umbraco.Core; +using Umbraco.Core.Models; +using Umbraco.Core.Models.Identity; +using Umbraco.Core.Persistence; +using Umbraco.Core.Services; +using Umbraco.Web.Models.ContentEditing; +using Umbraco.Web.Search; + +namespace Umbraco.Examine +{ + public class BackOfficeExamineSearcher : IBackOfficeExamineSearcher + { + private readonly IExamineManager _examineManager; + private readonly ILocalizationService _languageService; + private readonly ICurrentUserAccessor _currentUserAccessor; + private readonly IEntityService _entityService; + private readonly IUmbracoTreeSearcherFields _treeSearcherFields; + + public BackOfficeExamineSearcher(IExamineManager examineManager, + ILocalizationService languageService, + ICurrentUserAccessor currentUserAccessor, + IEntityService entityService, + IUmbracoTreeSearcherFields treeSearcherFields) + { + _examineManager = examineManager; + _languageService = languageService; + _currentUserAccessor = currentUserAccessor; + _entityService = entityService; + _treeSearcherFields = treeSearcherFields; + } + + public IEnumerable Search(string query, UmbracoEntityTypes entityType, int pageSize, long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false) + { + var sb = new StringBuilder(); + + string type; + var indexName = Constants.UmbracoIndexes.InternalIndexName; + var fields = _treeSearcherFields.GetBackOfficeFields().ToList(); + + // TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string + // manipulation for things like start paths, member types, etc... + //if (Examine.ExamineExtensions.TryParseLuceneQuery(query)) + //{ + + //} + + //special GUID check since if a user searches on one specifically we need to escape it + if (Guid.TryParse(query, out var g)) + { + query = "\"" + g.ToString() + "\""; + } + + var currentUser = _currentUserAccessor.TryGetCurrentUser(); + + switch (entityType) + { + case UmbracoEntityTypes.Member: + indexName = Constants.UmbracoIndexes.MembersIndexName; + type = "member"; + fields.AddRange(_treeSearcherFields.GetBackOfficeMembersFields()); + if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1") + { + sb.Append("+__NodeTypeAlias:"); + sb.Append(searchFrom); + sb.Append(" "); + } + break; + case UmbracoEntityTypes.Media: + type = "media"; + fields.AddRange(_treeSearcherFields.GetBackOfficeMediaFields()); + var allMediaStartNodes = currentUser != null + ? currentUser.CalculateMediaStartNodeIds(_entityService) + : Array.Empty(); + AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService); + break; + case UmbracoEntityTypes.Document: + type = "content"; + fields.AddRange(_treeSearcherFields.GetBackOfficeDocumentFields()); + var allContentStartNodes = currentUser != null + ? currentUser.CalculateContentStartNodeIds(_entityService) + : Array.Empty(); + AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService); + break; + default: + throw new NotSupportedException("The " + typeof(BackOfficeExamineSearcher) + " currently does not support searching against object type " + entityType); + } + + if (!_examineManager.TryGetIndex(indexName, out var index)) + throw new InvalidOperationException("No index found by name " + indexName); + + var internalSearcher = index.GetSearcher(); + + if (!BuildQuery(sb, query, searchFrom, fields, type)) + { + totalFound = 0; + return Enumerable.Empty(); + } + + var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()) + //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested + .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); + + totalFound = result.TotalItemCount; + + var pagedResult = result.Skip(Convert.ToInt32(pageIndex)); + + return pagedResult; + } + + private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List fields, string type) + { + //build a lucene query: + // the nodeName will be boosted 10x without wildcards + // then nodeName will be matched normally with wildcards + // the rest will be normal without wildcards + + var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList(); + + //check if text is surrounded by single or double quotes, if so, then exact match + var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$") + || Regex.IsMatch(query, "^\'.*?\'$"); + + if (surroundedByQuotes) + { + //strip quotes, escape string, the replace again + query = query.Trim('\"', '\''); + + query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + + //nothing to search + if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace()) + { + return false; + } + + //update the query with the query term + if (query.IsNullOrWhiteSpace() == false) + { + //add back the surrounding quotes + query = string.Format("{0}{1}{0}", "\"", query); + + sb.Append("+("); + + AppendNodeNamePhraseWithBoost(sb, query, allLangs); + + foreach (var f in fields) + { + //additional fields normally + sb.Append(f); + sb.Append(": ("); + sb.Append(query); + sb.Append(") "); + } + + sb.Append(") "); + } + } + else + { + var trimmed = query.Trim(new[] { '\"', '\'' }); + + //nothing to search + if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace()) + { + return false; + } + + //update the query with the query term + if (trimmed.IsNullOrWhiteSpace() == false) + { + query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + + var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + + sb.Append("+("); + + AppendNodeNameExactWithBoost(sb, query, allLangs); + + AppendNodeNameWithWildcards(sb, querywords, allLangs); + + foreach (var f in fields) + { + var queryWordsReplaced = new string[querywords.Length]; + + // when searching file names containing hyphens we need to replace the hyphens with spaces + if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName)) + { + for (var index = 0; index < querywords.Length; index++) + { + queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" "); + } + } + else + { + queryWordsReplaced = querywords; + } + + //additional fields normally + sb.Append(f); + sb.Append(":"); + sb.Append("("); + foreach (var w in queryWordsReplaced) + { + sb.Append(w.ToLower()); + sb.Append("* "); + } + sb.Append(")"); + sb.Append(" "); + } + + sb.Append(") "); + } + } + + //must match index type + sb.Append("+__IndexType:"); + sb.Append(type); + + return true; + } + + private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable allLangs) + { + //node name exactly boost x 10 + sb.Append("nodeName: ("); + sb.Append(query.ToLower()); + sb.Append(")^10.0 "); + + //also search on all variant node names + foreach (var lang in allLangs) + { + //node name exactly boost x 10 + sb.Append($"nodeName_{lang}: ("); + sb.Append(query.ToLower()); + sb.Append(")^10.0 "); + } + } + + private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable allLangs) + { + //node name exactly boost x 10 + sb.Append("nodeName:"); + sb.Append("\""); + sb.Append(query.ToLower()); + sb.Append("\""); + sb.Append("^10.0 "); + //also search on all variant node names + foreach (var lang in allLangs) + { + //node name exactly boost x 10 + sb.Append($"nodeName_{lang}:"); + sb.Append("\""); + sb.Append(query.ToLower()); + sb.Append("\""); + sb.Append("^10.0 "); + } + } + + private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable allLangs) + { + //node name normally with wildcards + sb.Append("nodeName:"); + sb.Append("("); + foreach (var w in querywords) + { + sb.Append(w.ToLower()); + sb.Append("* "); + } + sb.Append(") "); + //also search on all variant node names + foreach (var lang in allLangs) + { + //node name normally with wildcards + sb.Append($"nodeName_{lang}:"); + sb.Append("("); + foreach (var w in querywords) + { + sb.Append(w.ToLower()); + sb.Append("* "); + } + sb.Append(") "); + } + } + + private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService) + { + if (sb == null) throw new ArgumentNullException(nameof(sb)); + if (entityService == null) throw new ArgumentNullException(nameof(entityService)); + + UdiParser.TryParse(searchFrom, true, out var udi); + searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString(); + + var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0 + ? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault() + : null; + if (entityPath != null) + { + // find... only what's underneath + sb.Append("+__Path:"); + AppendPath(sb, entityPath.Path, false); + sb.Append(" "); + } + else if (startNodeIds.Length == 0) + { + // make sure we don't find anything + sb.Append("+__Path:none "); + } + else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction + { + var entityPaths = entityService.GetAllPaths(objectType, startNodeIds); + + // for each start node, find the start node, and what's underneath + // +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...) + sb.Append("+__Path:("); + var first = true; + foreach (var ep in entityPaths) + { + if (first) + first = false; + else + sb.Append(" "); + AppendPath(sb, ep.Path, true); + } + sb.Append(") "); + } + } + + private void AppendPath(StringBuilder sb, string path, bool includeThisNode) + { + path = path.Replace("-", "\\-").Replace(",", "\\,"); + if (includeThisNode) + { + sb.Append(path); + sb.Append(" "); + } + sb.Append(path); + sb.Append("\\,*"); + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineExtensions.cs b/src/Umbraco.Examine.Lucene/ExamineExtensions.cs index 2b763305c2..d697bf6f0d 100644 --- a/src/Umbraco.Examine.Lucene/ExamineExtensions.cs +++ b/src/Umbraco.Examine.Lucene/ExamineExtensions.cs @@ -1,7 +1,5 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Text.RegularExpressions; using Examine; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; @@ -15,13 +13,12 @@ using System.Threading; namespace Umbraco.Examine { + /// /// Extension methods for the LuceneIndex /// public static class ExamineExtensions { - - private static bool _isConfigured = false; private static object _configuredInit = null; private static object _isConfiguredLocker = new object(); @@ -74,7 +71,7 @@ namespace Umbraco.Examine /// /// This is not thread safe, use with care /// - internal static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing) + private static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing) { foreach (var luceneIndexer in examineManager.Indexes.OfType()) { diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs new file mode 100644 index 0000000000..0d701d388d --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs @@ -0,0 +1,50 @@ +using Examine; +using Examine.LuceneEngine.Directories; +using Umbraco.Core; +using Umbraco.Core.Composing; +using Umbraco.Core.Logging; + +namespace Umbraco.Examine +{ + + public sealed class ExamineLuceneComponent : IComponent + { + private readonly IndexRebuilder _indexRebuilder; + private readonly IExamineManager _examineManager; + private readonly IMainDom _mainDom; + private readonly ILogger _logger; + + public ExamineLuceneComponent(IndexRebuilder indexRebuilder, IExamineManager examineManager, IMainDom mainDom, ILogger logger) + { + _indexRebuilder = indexRebuilder; + _examineManager = examineManager; + _mainDom = mainDom; + _logger = logger; + } + + public void Initialize() + { + //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain + //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock + //which simply checks the existence of the lock file + DirectoryFactory.DefaultLockFactory = d => + { + var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); + return simpleFsLockFactory; + }; + + _indexRebuilder.RebuildingIndexes += IndexRebuilder_RebuildingIndexes; + } + + /// + /// Handles event to ensure that all lucene based indexes are properly configured before rebuilding + /// + /// + /// + private void IndexRebuilder_RebuildingIndexes(object sender, IndexRebuildingEventArgs e) => _examineManager.ConfigureIndexes(_mainDom, _logger); + + public void Terminate() + { + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs new file mode 100644 index 0000000000..0334531691 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs @@ -0,0 +1,21 @@ +using Umbraco.Core; +using Umbraco.Core.Composing; +using Umbraco.Web.Search; + +namespace Umbraco.Examine +{ + // We want to run after core composers since we are replacing some items + [ComposeAfter(typeof(ICoreComposer))] + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] + public sealed class ExamineLuceneComposer : ComponentComposer + { + public override void Compose(Composition composition) + { + base.Compose(composition); + + composition.RegisterUnique(); + composition.RegisterUnique(); + composition.RegisterUnique(); + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs new file mode 100644 index 0000000000..e1e80ead2f --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs @@ -0,0 +1,33 @@ +using Examine; +using Umbraco.Core; +using Umbraco.Core.Composing; +using Umbraco.Core.Logging; + +namespace Umbraco.Examine +{ + public class ExamineLuceneFinalComponent : IComponent + { + private readonly IProfilingLogger _logger; + private readonly IExamineManager _examineManager; + private readonly IMainDom _mainDom; + + public ExamineLuceneFinalComponent(IProfilingLogger logger, IExamineManager examineManager, IMainDom mainDom) + { + _logger = logger; + _examineManager = examineManager; + _mainDom = mainDom; + } + + public void Initialize() + { + if (!_mainDom.IsMainDom) return; + + // Ensures all lucene based indexes are unlocked and ready to go + _examineManager.ConfigureIndexes(_mainDom, _logger); + } + + public void Terminate() + { + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs new file mode 100644 index 0000000000..1a73426568 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs @@ -0,0 +1,13 @@ +using Umbraco.Core; +using Umbraco.Core.Composing; + +namespace Umbraco.Examine +{ + // examine's Lucene final composer composes after all user composers + // and *also* after ICoreComposer (in case IUserComposer is disabled) + [ComposeAfter(typeof(IUserComposer))] + [ComposeAfter(typeof(ICoreComposer))] + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] + public class ExamineLuceneFinalComposer : ComponentComposer + { } +} diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs new file mode 100644 index 0000000000..cc1fc115ca --- /dev/null +++ b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs @@ -0,0 +1,35 @@ +using Examine; +using Examine.LuceneEngine.Providers; +using Umbraco.Core.Logging; +using Umbraco.Core.IO; + +namespace Umbraco.Examine +{ + /// + /// Implementation of which returns + /// for lucene based indexes that don't have an implementation else fallsback to the default implementation. + /// + public class LuceneIndexDiagnosticsFactory : IndexDiagnosticsFactory + { + private readonly ILogger _logger; + private readonly IIOHelper _ioHelper; + + public LuceneIndexDiagnosticsFactory(ILogger logger, IIOHelper ioHelper) + { + _logger = logger; + _ioHelper = ioHelper; + } + + public override IIndexDiagnostics Create(IIndex index) + { + if (!(index is IIndexDiagnostics indexDiag)) + { + if (index is LuceneIndex luceneIndex) + indexDiag = new LuceneIndexDiagnostics(luceneIndex, _logger, _ioHelper); + else + indexDiag = base.Create(index); + } + return indexDiag; + } + } +} diff --git a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj index 8df6aafa2d..255df2d2d6 100644 --- a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj +++ b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj @@ -66,8 +66,14 @@ + + + + + + @@ -75,6 +81,7 @@ + Properties\SolutionInfo.cs diff --git a/src/Umbraco.Web/Search/UmbracoIndexesCreator.cs b/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs similarity index 97% rename from src/Umbraco.Web/Search/UmbracoIndexesCreator.cs rename to src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs index 71b5bee108..8e50c6ebc4 100644 --- a/src/Umbraco.Web/Search/UmbracoIndexesCreator.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using Umbraco.Core.Logging; using Umbraco.Core.Services; using Umbraco.Examine; @@ -9,7 +8,6 @@ using Examine; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.IO; -using Umbraco.Web.PublishedCache.NuCache; namespace Umbraco.Web.Search { diff --git a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs index 494c661062..0e9128d31a 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs @@ -1,6 +1,4 @@ -using System.Collections.Generic; -using Examine; -using Examine.LuceneEngine; +using Examine; using Lucene.Net.Analysis; using Umbraco.Core; using Umbraco.Core.IO; diff --git a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs b/src/Umbraco.Examine/GenericIndexDiagnostics.cs similarity index 97% rename from src/Umbraco.Web/Search/GenericIndexDiagnostics.cs rename to src/Umbraco.Examine/GenericIndexDiagnostics.cs index cb25e1242a..c384392710 100644 --- a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs +++ b/src/Umbraco.Examine/GenericIndexDiagnostics.cs @@ -4,9 +4,8 @@ using System.Linq; using Examine; using Umbraco.Core; using Umbraco.Core.Composing; -using Umbraco.Examine; -namespace Umbraco.Web.Search +namespace Umbraco.Examine { /// diff --git a/src/Umbraco.Examine/IBackOfficeExamineSearcher.cs b/src/Umbraco.Examine/IBackOfficeExamineSearcher.cs new file mode 100644 index 0000000000..719d5a33f2 --- /dev/null +++ b/src/Umbraco.Examine/IBackOfficeExamineSearcher.cs @@ -0,0 +1,17 @@ +using Examine; +using System.Collections.Generic; +using Umbraco.Web.Models.ContentEditing; + +namespace Umbraco.Examine +{ + /// + /// Used to search the back office for Examine indexed entities (Documents, Media and Members) + /// + public interface IBackOfficeExamineSearcher + { + IEnumerable Search(string query, + UmbracoEntityTypes entityType, + int pageSize, + long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false); + } +} diff --git a/src/Umbraco.Examine/IIndexDiagnostics.cs b/src/Umbraco.Examine/IIndexDiagnostics.cs index 29d530c2d0..b51f6f256f 100644 --- a/src/Umbraco.Examine/IIndexDiagnostics.cs +++ b/src/Umbraco.Examine/IIndexDiagnostics.cs @@ -4,6 +4,7 @@ using Umbraco.Core; namespace Umbraco.Examine { + /// /// Exposes diagnostic information about an index /// diff --git a/src/Umbraco.Examine/IIndexDiagnosticsFactory.cs b/src/Umbraco.Examine/IIndexDiagnosticsFactory.cs new file mode 100644 index 0000000000..f7922e64cb --- /dev/null +++ b/src/Umbraco.Examine/IIndexDiagnosticsFactory.cs @@ -0,0 +1,13 @@ +using Examine; + +namespace Umbraco.Examine +{ + + /// + /// Creates for an index if it doesn't implement + /// + public interface IIndexDiagnosticsFactory + { + IIndexDiagnostics Create(IIndex index); + } +} diff --git a/src/Umbraco.Web/Search/IUmbracoIndexesCreator.cs b/src/Umbraco.Examine/IUmbracoIndexesCreator.cs similarity index 63% rename from src/Umbraco.Web/Search/IUmbracoIndexesCreator.cs rename to src/Umbraco.Examine/IUmbracoIndexesCreator.cs index d654e4effd..d64046a940 100644 --- a/src/Umbraco.Web/Search/IUmbracoIndexesCreator.cs +++ b/src/Umbraco.Examine/IUmbracoIndexesCreator.cs @@ -1,8 +1,4 @@ -using System.Collections.Generic; -using Examine; -using Umbraco.Examine; - -namespace Umbraco.Web.Search +namespace Umbraco.Examine { /// /// diff --git a/src/Umbraco.Examine/IndexDiagnosticsFactory.cs b/src/Umbraco.Examine/IndexDiagnosticsFactory.cs new file mode 100644 index 0000000000..9daa1705a9 --- /dev/null +++ b/src/Umbraco.Examine/IndexDiagnosticsFactory.cs @@ -0,0 +1,17 @@ +using Examine; + +namespace Umbraco.Examine +{ + /// + /// Default implementation of which returns for indexes that don't have an implementation + /// + public class IndexDiagnosticsFactory : IIndexDiagnosticsFactory + { + public virtual IIndexDiagnostics Create(IIndex index) + { + if (!(index is IIndexDiagnostics indexDiag)) + indexDiag = new GenericIndexDiagnostics(index); + return indexDiag; + } + } +} diff --git a/src/Umbraco.Examine/IndexRebuilder.cs b/src/Umbraco.Examine/IndexRebuilder.cs index 786aecac71..d87241e6f6 100644 --- a/src/Umbraco.Examine/IndexRebuilder.cs +++ b/src/Umbraco.Examine/IndexRebuilder.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Examine; namespace Umbraco.Examine -{ +{ /// /// Utility to rebuild all indexes ensuring minimal data queries @@ -45,6 +45,8 @@ namespace Umbraco.Examine if (indexes.Length == 0) return; + OnRebuildingIndexes(new IndexRebuildingEventArgs(indexes)); + foreach (var index in indexes) { index.CreateIndex(); // clear the index @@ -54,5 +56,11 @@ namespace Umbraco.Examine Parallel.ForEach(_populators, populator => populator.Populate(indexes)); } + /// + /// Event raised when indexes are being rebuilt + /// + public event EventHandler RebuildingIndexes; + + private void OnRebuildingIndexes(IndexRebuildingEventArgs args) => RebuildingIndexes?.Invoke(this, args); } } diff --git a/src/Umbraco.Examine/IndexRebuildingEventArgs.cs b/src/Umbraco.Examine/IndexRebuildingEventArgs.cs new file mode 100644 index 0000000000..20141a7194 --- /dev/null +++ b/src/Umbraco.Examine/IndexRebuildingEventArgs.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using Examine; + +namespace Umbraco.Examine +{ + public class IndexRebuildingEventArgs : EventArgs + { + public IndexRebuildingEventArgs(IEnumerable indexes) + { + Indexes = indexes; + } + + /// + /// The indexes being rebuilt + /// + public IEnumerable Indexes { get; } + } +} diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index b66dc0ed6a..2e1656b45d 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -5,7 +5,6 @@ using System.Net; using System.Net.Http; using System.Web.Http; using Examine; -using Examine.LuceneEngine.Providers; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.IO; @@ -24,17 +23,19 @@ namespace Umbraco.Web.Editors private readonly IExamineManager _examineManager; private readonly ILogger _logger; private readonly IIOHelper _ioHelper; + private readonly IIndexDiagnosticsFactory _indexDiagnosticsFactory; private readonly IAppPolicyCache _runtimeCache; private readonly IndexRebuilder _indexRebuilder; - public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper, + public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper, IIndexDiagnosticsFactory indexDiagnosticsFactory, AppCaches appCaches, IndexRebuilder indexRebuilder) { _examineManager = examineManager; _logger = logger; _ioHelper = ioHelper; + _indexDiagnosticsFactory = indexDiagnosticsFactory; _runtimeCache = appCaches.RuntimeCache; _indexRebuilder = indexRebuilder; } @@ -69,9 +70,8 @@ namespace Umbraco.Web.Editors if (!msg.IsSuccessStatusCode) throw new HttpResponseException(msg); - var results = global::Umbraco.Examine.ExamineExtensions.TryParseLuceneQuery(query) - ? searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1)) - : searcher.Search(query, maxResults: pageSize * (pageIndex + 1)); + // NativeQuery will work for a single word/phrase too (but depends on the implementation) the lucene one will work. + var results = searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1)); var pagedResults = results.Skip(pageIndex * pageSize); @@ -173,19 +173,11 @@ namespace Umbraco.Web.Editors } } - - private ExamineIndexModel CreateModel(IIndex index) { var indexName = index.Name; - if (!(index is IIndexDiagnostics indexDiag)) - { - if (index is LuceneIndex luceneIndex) - indexDiag = new LuceneIndexDiagnostics(luceneIndex, Logger, _ioHelper); - else - indexDiag = new GenericIndexDiagnostics(index); - } + var indexDiag = _indexDiagnosticsFactory.Create(index); var isHealth = indexDiag.IsHealthy(); var properties = new Dictionary diff --git a/src/Umbraco.Web/ExamineExtensions.cs b/src/Umbraco.Web/ExamineExtensions.cs index dd1277d3e2..5da21de6df 100644 --- a/src/Umbraco.Web/ExamineExtensions.cs +++ b/src/Umbraco.Web/ExamineExtensions.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; -using System.Linq; using Examine; -using Examine.LuceneEngine.Providers; using Umbraco.Core.Models.PublishedContent; using Umbraco.Examine; using Umbraco.Web.PublishedCache; diff --git a/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs index 308605a197..c454f19ce7 100644 --- a/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; -using System.Linq; using Examine; -using Examine.LuceneEngine.Providers; using Umbraco.Core; using Umbraco.Core.Mapping; using Umbraco.Core.Models; diff --git a/src/Umbraco.Web/Runtime/WebInitialComposer.cs b/src/Umbraco.Web/Runtime/WebInitialComposer.cs index d34398e066..1f4678e2e1 100644 --- a/src/Umbraco.Web/Runtime/WebInitialComposer.cs +++ b/src/Umbraco.Web/Runtime/WebInitialComposer.cs @@ -44,6 +44,7 @@ using Umbraco.Web.Trees; using Umbraco.Web.WebApi; using Current = Umbraco.Web.Composing.Current; using Umbraco.Web.PropertyEditors; +using Umbraco.Examine; namespace Umbraco.Web.Runtime { diff --git a/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs b/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs index 9f83b97393..2c964a2723 100644 --- a/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs +++ b/src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs @@ -118,7 +118,6 @@ namespace Umbraco.Web.Search if (_waitMilliseconds > 0) Thread.Sleep(_waitMilliseconds); - _indexRebuilder.ExamineManager.ConfigureIndexes(_mainDom, _logger); _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); } } diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 416513d512..037981d8b4 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -13,9 +13,6 @@ using Umbraco.Core.Services.Changes; using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Examine; -using Examine.LuceneEngine.Directories; -using Umbraco.Web.Composing; -using System.ComponentModel; namespace Umbraco.Web.Search { @@ -65,15 +62,6 @@ namespace Umbraco.Web.Search public void Initialize() { - //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain - //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock - //which simply checks the existence of the lock file - DirectoryFactory.DefaultLockFactory = d => - { - var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); - return simpleFsLockFactory; - }; - //let's deal with shutting down Examine with MainDom var examineShutdownRegistered = _mainDom.Register(() => { diff --git a/src/Umbraco.Web/Search/ExamineComposer.cs b/src/Umbraco.Web/Search/ExamineComposer.cs index 9d8af5f3a0..fe3c604f8b 100644 --- a/src/Umbraco.Web/Search/ExamineComposer.cs +++ b/src/Umbraco.Web/Search/ExamineComposer.cs @@ -30,7 +30,7 @@ namespace Umbraco.Web.Search composition.Register(Lifetime.Singleton); composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); composition.RegisterUnique(factory => new ContentValueSetBuilder( factory.GetInstance(), diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs index 95000b2b46..ffcd96af7e 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComponent.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -12,15 +12,11 @@ namespace Umbraco.Web.Search /// public sealed class ExamineFinalComponent : IComponent { - private readonly IProfilingLogger _logger; - private readonly IExamineManager _examineManager; BackgroundIndexRebuilder _indexRebuilder; private readonly IMainDom _mainDom; - public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom) + public ExamineFinalComponent(BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom) { - _logger = logger; - _examineManager = examineManager; _indexRebuilder = indexRebuilder; _mainDom = mainDom; } @@ -29,8 +25,6 @@ namespace Umbraco.Web.Search { if (!_mainDom.IsMainDom) return; - _examineManager.ConfigureIndexes(_mainDom, _logger); - // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? _indexRebuilder.RebuildIndexes(true, 5000); } diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index 306d4c0593..0f52384409 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Text.RegularExpressions; using Examine; using Umbraco.Core; using Umbraco.Core.Mapping; @@ -22,29 +20,26 @@ namespace Umbraco.Web.Search /// public class UmbracoTreeSearcher { - private readonly IExamineManager _examineManager; private readonly UmbracoContext _umbracoContext; private readonly ILocalizationService _languageService; private readonly IEntityService _entityService; private readonly UmbracoMapper _mapper; private readonly ISqlContext _sqlContext; - private readonly IUmbracoTreeSearcherFields _umbracoTreeSearcherFields; + private readonly IBackOfficeExamineSearcher _backOfficeExamineSearcher; - public UmbracoTreeSearcher(IExamineManager examineManager, - UmbracoContext umbracoContext, + public UmbracoTreeSearcher(UmbracoContext umbracoContext, ILocalizationService languageService, IEntityService entityService, UmbracoMapper mapper, - ISqlContext sqlContext,IUmbracoTreeSearcherFields umbracoTreeSearcherFields) + ISqlContext sqlContext, IBackOfficeExamineSearcher backOfficeExamineSearcher) { - _examineManager = examineManager ?? throw new ArgumentNullException(nameof(examineManager)); _umbracoContext = umbracoContext; _languageService = languageService; _entityService = entityService; _mapper = mapper; _sqlContext = sqlContext; - _umbracoTreeSearcherFields = umbracoTreeSearcherFields; + _backOfficeExamineSearcher = backOfficeExamineSearcher; } /// @@ -66,72 +61,7 @@ namespace Umbraco.Web.Search int pageSize, long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false) { - var sb = new StringBuilder(); - - string type; - var indexName = Constants.UmbracoIndexes.InternalIndexName; - var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList(); - - // TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string - // manipulation for things like start paths, member types, etc... - //if (Examine.ExamineExtensions.TryParseLuceneQuery(query)) - //{ - - //} - - //special GUID check since if a user searches on one specifically we need to escape it - if (Guid.TryParse(query, out var g)) - { - query = "\"" + g.ToString() + "\""; - } - - switch (entityType) - { - case UmbracoEntityTypes.Member: - indexName = Constants.UmbracoIndexes.MembersIndexName; - type = "member"; - fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMembersFields()); - if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1") - { - sb.Append("+__NodeTypeAlias:"); - sb.Append(searchFrom); - sb.Append(" "); - } - break; - case UmbracoEntityTypes.Media: - type = "media"; - fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMediaFields()); - var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService); - AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService); - break; - case UmbracoEntityTypes.Document: - type = "content"; - fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeDocumentFields()); - var allContentStartNodes = _umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService); - AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService); - break; - default: - throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType); - } - - if (!_examineManager.TryGetIndex(indexName, out var index)) - throw new InvalidOperationException("No index found by name " + indexName); - - var internalSearcher = index.GetSearcher(); - - if (!BuildQuery(sb, query, searchFrom, fields, type)) - { - totalFound = 0; - return Enumerable.Empty(); - } - - var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()) - //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested - .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); - - totalFound = result.TotalItemCount; - - var pagedResult = result.Skip(Convert.ToInt32(pageIndex)); + var pagedResult = _backOfficeExamineSearcher.Search(query, entityType, pageSize, pageIndex, out totalFound, searchFrom, ignoreUserStartNodes); switch (entityType) { @@ -166,236 +96,6 @@ namespace Umbraco.Web.Search return _mapper.MapEnumerable(results); } - private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List fields, string type) - { - //build a lucene query: - // the nodeName will be boosted 10x without wildcards - // then nodeName will be matched normally with wildcards - // the rest will be normal without wildcards - - var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList(); - - //check if text is surrounded by single or double quotes, if so, then exact match - var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$") - || Regex.IsMatch(query, "^\'.*?\'$"); - - if (surroundedByQuotes) - { - //strip quotes, escape string, the replace again - query = query.Trim('\"', '\''); - - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); - - //nothing to search - if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace()) - { - return false; - } - - //update the query with the query term - if (query.IsNullOrWhiteSpace() == false) - { - //add back the surrounding quotes - query = string.Format("{0}{1}{0}", "\"", query); - - sb.Append("+("); - - AppendNodeNamePhraseWithBoost(sb, query, allLangs); - - foreach (var f in fields) - { - //additional fields normally - sb.Append(f); - sb.Append(": ("); - sb.Append(query); - sb.Append(") "); - } - - sb.Append(") "); - } - } - else - { - var trimmed = query.Trim(new[] { '\"', '\'' }); - - //nothing to search - if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace()) - { - return false; - } - - //update the query with the query term - if (trimmed.IsNullOrWhiteSpace() == false) - { - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); - - var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - - sb.Append("+("); - - AppendNodeNameExactWithBoost(sb, query, allLangs); - - AppendNodeNameWithWildcards(sb, querywords, allLangs); - - foreach (var f in fields) - { - var queryWordsReplaced = new string[querywords.Length]; - - // when searching file names containing hyphens we need to replace the hyphens with spaces - if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName)) - { - for (var index = 0; index < querywords.Length; index++) - { - queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" "); - } - } - else - { - queryWordsReplaced = querywords; - } - - //additional fields normally - sb.Append(f); - sb.Append(":"); - sb.Append("("); - foreach (var w in queryWordsReplaced) - { - sb.Append(w.ToLower()); - sb.Append("* "); - } - sb.Append(")"); - sb.Append(" "); - } - - sb.Append(") "); - } - } - - //must match index type - sb.Append("+__IndexType:"); - sb.Append(type); - - return true; - } - - private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable allLangs) - { - //node name exactly boost x 10 - sb.Append("nodeName: ("); - sb.Append(query.ToLower()); - sb.Append(")^10.0 "); - - //also search on all variant node names - foreach (var lang in allLangs) - { - //node name exactly boost x 10 - sb.Append($"nodeName_{lang}: ("); - sb.Append(query.ToLower()); - sb.Append(")^10.0 "); - } - } - - private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable allLangs) - { - //node name exactly boost x 10 - sb.Append("nodeName:"); - sb.Append("\""); - sb.Append(query.ToLower()); - sb.Append("\""); - sb.Append("^10.0 "); - //also search on all variant node names - foreach (var lang in allLangs) - { - //node name exactly boost x 10 - sb.Append($"nodeName_{lang}:"); - sb.Append("\""); - sb.Append(query.ToLower()); - sb.Append("\""); - sb.Append("^10.0 "); - } - } - - private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable allLangs) - { - //node name normally with wildcards - sb.Append("nodeName:"); - sb.Append("("); - foreach (var w in querywords) - { - sb.Append(w.ToLower()); - sb.Append("* "); - } - sb.Append(") "); - //also search on all variant node names - foreach (var lang in allLangs) - { - //node name normally with wildcards - sb.Append($"nodeName_{lang}:"); - sb.Append("("); - foreach (var w in querywords) - { - sb.Append(w.ToLower()); - sb.Append("* "); - } - sb.Append(") "); - } - } - - private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService) - { - if (sb == null) throw new ArgumentNullException(nameof(sb)); - if (entityService == null) throw new ArgumentNullException(nameof(entityService)); - - UdiParser.TryParse(searchFrom, true, out var udi); - searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString(); - - var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0 - ? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault() - : null; - if (entityPath != null) - { - // find... only what's underneath - sb.Append("+__Path:"); - AppendPath(sb, entityPath.Path, false); - sb.Append(" "); - } - else if (startNodeIds.Length == 0) - { - // make sure we don't find anything - sb.Append("+__Path:none "); - } - else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction - { - var entityPaths = entityService.GetAllPaths(objectType, startNodeIds); - - // for each start node, find the start node, and what's underneath - // +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...) - sb.Append("+__Path:("); - var first = true; - foreach (var ep in entityPaths) - { - if (first) - first = false; - else - sb.Append(" "); - AppendPath(sb, ep.Path, true); - } - sb.Append(") "); - } - } - - private void AppendPath(StringBuilder sb, string path, bool includeThisNode) - { - path = path.Replace("-", "\\-").Replace(",", "\\,"); - if (includeThisNode) - { - sb.Append(path); - sb.Append(" "); - } - sb.Append(path); - sb.Append("\\,*"); - } - /// /// Returns a collection of entities for media based on search results /// diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 9d226cda3d..d285e7e587 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -65,9 +65,6 @@ 2.0.0-alpha.20200128.15 - - 2.0.0-alpha.20200128.15 - 2.7.0.100 @@ -116,10 +113,6 @@ {f9b7fe05-0f93-4d0d-9c10-690b33ecbbd8} Umbraco.Examine - - {07fbc26b-2927-4a22-8d96-d644c667fecc} - Umbraco.Examine.Lucene - {3ae7bf57-966b-45a5-910a-954d7c554441} Umbraco.Infrastructure @@ -246,7 +239,6 @@ - @@ -271,9 +263,6 @@ - - -