Merge pull request #7529 from umbraco/netcore/feature/3679-examine-netstandard

Updating solution to use netstandard abstractions of Examine
This commit is contained in:
Shannon Deminick
2020-02-04 12:35:12 +11:00
committed by GitHub
71 changed files with 1066 additions and 698 deletions
+1 -1
View File
@@ -7,6 +7,6 @@
-->
<packageSources>
<add key="UmbracoCoreMyGet" value="https://www.myget.org/F/umbracocore/api/v3/index.json" />
<add key="ExamineAppVeyor" value="https://ci.appveyor.com/nuget/examine-f73l6qv0oqfh/" />
<add key="ExamineAzurePipelines" value="https://shazwazza.pkgs.visualstudio.com/Examine/_packaging/Examine-Beta/nuget/v3/index.json" />
</packageSources>
</configuration>
@@ -0,0 +1,345 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Identity;
using Umbraco.Core.Persistence;
using Umbraco.Core.Services;
using Umbraco.Web.Models.ContentEditing;
namespace Umbraco.Examine
{
public class BackOfficeExamineSearcher : IBackOfficeExamineSearcher
{
private readonly IExamineManager _examineManager;
private readonly ILocalizationService _languageService;
private readonly ICurrentUserAccessor _currentUserAccessor;
private readonly IEntityService _entityService;
private readonly IUmbracoTreeSearcherFields _treeSearcherFields;
public BackOfficeExamineSearcher(IExamineManager examineManager,
ILocalizationService languageService,
ICurrentUserAccessor currentUserAccessor,
IEntityService entityService,
IUmbracoTreeSearcherFields treeSearcherFields)
{
_examineManager = examineManager;
_languageService = languageService;
_currentUserAccessor = currentUserAccessor;
_entityService = entityService;
_treeSearcherFields = treeSearcherFields;
}
public IEnumerable<ISearchResult> Search(string query, UmbracoEntityTypes entityType, int pageSize, long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false)
{
var sb = new StringBuilder();
string type;
var indexName = Constants.UmbracoIndexes.InternalIndexName;
var fields = _treeSearcherFields.GetBackOfficeFields().ToList();
// TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
//special GUID check since if a user searches on one specifically we need to escape it
if (Guid.TryParse(query, out var g))
{
query = "\"" + g.ToString() + "\"";
}
var currentUser = _currentUserAccessor.TryGetCurrentUser();
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields.AddRange(_treeSearcherFields.GetBackOfficeMembersFields());
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
{
sb.Append("+__NodeTypeAlias:");
sb.Append(searchFrom);
sb.Append(" ");
}
break;
case UmbracoEntityTypes.Media:
type = "media";
fields.AddRange(_treeSearcherFields.GetBackOfficeMediaFields());
var allMediaStartNodes = currentUser != null
? currentUser.CalculateMediaStartNodeIds(_entityService)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
fields.AddRange(_treeSearcherFields.GetBackOfficeDocumentFields());
var allContentStartNodes = currentUser != null
? currentUser.CalculateContentStartNodeIds(_entityService)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(BackOfficeExamineSearcher) + " currently does not support searching against object type " + entityType);
}
if (!_examineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found by name " + indexName);
var internalSearcher = index.GetSearcher();
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<ISearchResult>();
}
var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString())
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Execute(Convert.ToInt32(pageSize * (pageIndex + 1)));
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
return pagedResult;
}
private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List<string> fields, string type)
{
//build a lucene query:
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
|| Regex.IsMatch(query, "^\'.*?\'$");
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim('\"', '\'');
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (query.IsNullOrWhiteSpace() == false)
{
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
//additional fields normally
sb.Append(f);
sb.Append(": (");
sb.Append(query);
sb.Append(") ");
}
sb.Append(") ");
}
}
else
{
var trimmed = query.Trim(new[] { '\"', '\'' });
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (trimmed.IsNullOrWhiteSpace() == false)
{
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
var queryWordsReplaced = new string[querywords.Length];
// when searching file names containing hyphens we need to replace the hyphens with spaces
if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName))
{
for (var index = 0; index < querywords.Length; index++)
{
queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" ");
}
}
else
{
queryWordsReplaced = querywords;
}
//additional fields normally
sb.Append(f);
sb.Append(":");
sb.Append("(");
foreach (var w in queryWordsReplaced)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(")");
sb.Append(" ");
}
sb.Append(") ");
}
}
//must match index type
sb.Append("+__IndexType:");
sb.Append(type);
return true;
}
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService)
{
if (sb == null) throw new ArgumentNullException(nameof(sb));
if (entityService == null) throw new ArgumentNullException(nameof(entityService));
UdiParser.TryParse(searchFrom, true, out var udi);
searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString();
var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0
? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault()
: null;
if (entityPath != null)
{
// find... only what's underneath
sb.Append("+__Path:");
AppendPath(sb, entityPath.Path, false);
sb.Append(" ");
}
else if (startNodeIds.Length == 0)
{
// make sure we don't find anything
sb.Append("+__Path:none ");
}
else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction
{
var entityPaths = entityService.GetAllPaths(objectType, startNodeIds);
// for each start node, find the start node, and what's underneath
// +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...)
sb.Append("+__Path:(");
var first = true;
foreach (var ep in entityPaths)
{
if (first)
first = false;
else
sb.Append(" ");
AppendPath(sb, ep.Path, true);
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, string path, bool includeThisNode)
{
path = path.Replace("-", "\\-").Replace(",", "\\,");
if (includeThisNode)
{
sb.Append(path);
sb.Append(" ");
}
sb.Append(path);
sb.Append("\\,*");
}
}
}
@@ -1,14 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Examine;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Umbraco.Core;
using Version = Lucene.Net.Util.Version;
using Umbraco.Core.Logging;
@@ -16,19 +13,12 @@ using System.Threading;
namespace Umbraco.Examine
{
/// <summary>
/// Extension methods for the LuceneIndex
/// </summary>
public static class ExamineExtensions
{
/// <summary>
/// Matches a culture iso name suffix
/// </summary>
/// <remarks>
/// myFieldName_en-us will match the "en-us"
/// </remarks>
internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled);
private static bool _isConfigured = false;
private static object _configuredInit = null;
private static object _isConfiguredLocker = new object();
@@ -52,51 +42,6 @@ namespace Umbraco.Examine
});
}
//TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression
/// <summary>
/// Returns all index fields that are culture specific (suffixed)
/// </summary>
/// <param name="index"></param>
/// <param name="culture"></param>
/// <returns></returns>
public static IEnumerable<string> GetCultureFields(this IUmbracoIndex index, string culture)
{
var allFields = index.GetFields();
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var field in allFields)
{
var match = CultureIsoCodeFieldNameMatchExpression.Match(field);
if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value))
yield return field;
}
}
/// <summary>
/// Returns all index fields that are culture specific (suffixed) or invariant
/// </summary>
/// <param name="index"></param>
/// <param name="culture"></param>
/// <returns></returns>
public static IEnumerable<string> GetCultureAndInvariantFields(this IUmbracoIndex index, string culture)
{
var allFields = index.GetFields();
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var field in allFields)
{
var match = CultureIsoCodeFieldNameMatchExpression.Match(field);
if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value))
{
yield return field; //matches this culture field
}
else if (!match.Success)
{
yield return field; //matches no culture field (invariant)
}
}
}
internal static bool TryParseLuceneQuery(string query)
{
// TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll
@@ -107,7 +52,7 @@ namespace Umbraco.Examine
try
{
//This will pass with a plain old string without any fields, need to figure out a way to have it properly parse
var parsed = new QueryParser(Version.LUCENE_30, "nodeName", new KeywordAnalyzer()).Parse(query);
var parsed = new QueryParser(Version.LUCENE_30, UmbracoExamineFieldNames.NodeNameFieldName, new KeywordAnalyzer()).Parse(query);
return true;
}
catch (ParseException)
@@ -126,7 +71,7 @@ namespace Umbraco.Examine
/// <remarks>
/// This is not thread safe, use with care
/// </remarks>
internal static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing)
private static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing)
{
foreach (var luceneIndexer in examineManager.Indexes.OfType<LuceneIndex>())
{
@@ -0,0 +1,50 @@
using Examine;
using Examine.LuceneEngine.Directories;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
namespace Umbraco.Examine
{
public sealed class ExamineLuceneComponent : IComponent
{
private readonly IndexRebuilder _indexRebuilder;
private readonly IExamineManager _examineManager;
private readonly IMainDom _mainDom;
private readonly ILogger _logger;
public ExamineLuceneComponent(IndexRebuilder indexRebuilder, IExamineManager examineManager, IMainDom mainDom, ILogger logger)
{
_indexRebuilder = indexRebuilder;
_examineManager = examineManager;
_mainDom = mainDom;
_logger = logger;
}
public void Initialize()
{
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryFactory.DefaultLockFactory = d =>
{
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
_indexRebuilder.RebuildingIndexes += IndexRebuilder_RebuildingIndexes;
}
/// <summary>
/// Handles event to ensure that all lucene based indexes are properly configured before rebuilding
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void IndexRebuilder_RebuildingIndexes(object sender, IndexRebuildingEventArgs e) => _examineManager.ConfigureIndexes(_mainDom, _logger);
public void Terminate()
{
}
}
}
@@ -0,0 +1,20 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
namespace Umbraco.Examine
{
// We want to run after core composers since we are replacing some items
[ComposeAfter(typeof(ICoreComposer))]
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public sealed class ExamineLuceneComposer : ComponentComposer<ExamineLuceneComponent>
{
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.RegisterUnique<IBackOfficeExamineSearcher, BackOfficeExamineSearcher>();
composition.RegisterUnique<IUmbracoIndexesCreator, UmbracoIndexesCreator>();
composition.RegisterUnique<IIndexDiagnosticsFactory, LuceneIndexDiagnosticsFactory>();
}
}
}
@@ -0,0 +1,33 @@
using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
namespace Umbraco.Examine
{
public class ExamineLuceneFinalComponent : IComponent
{
private readonly IProfilingLogger _logger;
private readonly IExamineManager _examineManager;
private readonly IMainDom _mainDom;
public ExamineLuceneFinalComponent(IProfilingLogger logger, IExamineManager examineManager, IMainDom mainDom)
{
_logger = logger;
_examineManager = examineManager;
_mainDom = mainDom;
}
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
// Ensures all lucene based indexes are unlocked and ready to go
_examineManager.ConfigureIndexes(_mainDom, _logger);
}
public void Terminate()
{
}
}
}
@@ -0,0 +1,13 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
namespace Umbraco.Examine
{
// examine's Lucene final composer composes after all user composers
// and *also* after ICoreComposer (in case IUserComposer is disabled)
[ComposeAfter(typeof(IUserComposer))]
[ComposeAfter(typeof(ICoreComposer))]
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public class ExamineLuceneFinalComposer : ComponentComposer<ExamineLuceneFinalComponent>
{ }
}
@@ -0,0 +1,35 @@
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core.Logging;
using Umbraco.Core.IO;
namespace Umbraco.Examine
{
/// <summary>
/// Implementation of <see cref="IIndexDiagnosticsFactory"/> which returns <see cref="LuceneIndexDiagnostics"/>
/// for lucene based indexes that don't have an implementation else fallsback to the default <see cref="IndexDiagnosticsFactory"/> implementation.
/// </summary>
public class LuceneIndexDiagnosticsFactory : IndexDiagnosticsFactory
{
private readonly ILogger _logger;
private readonly IIOHelper _ioHelper;
public LuceneIndexDiagnosticsFactory(ILogger logger, IIOHelper ioHelper)
{
_logger = logger;
_ioHelper = ioHelper;
}
public override IIndexDiagnostics Create(IIndex index)
{
if (!(index is IIndexDiagnostics indexDiag))
{
if (index is LuceneIndex luceneIndex)
indexDiag = new LuceneIndexDiagnostics(luceneIndex, _logger, _ioHelper);
else
indexDiag = base.Create(index);
}
return indexDiag;
}
}
}
@@ -0,0 +1,104 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<ProjectGuid>{07FBC26B-2927-4A22-8D96-D644C667FECC}</ProjectGuid>
<OutputType>Library</OutputType>
<AssemblyName>Umbraco.Examine.Lucene</AssemblyName>
<RootNamespace>Umbraco.Examine</RootNamespace>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<TargetFrameworkProfile />
<AdditionalFileItemNames>$(AdditionalFileItemNames);Content</AdditionalFileItemNames>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>portable</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>portable</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Release\Umbraco.Examine.Lucene.xml</DocumentationFile>
<Prefer32Bit>false</Prefer32Bit>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Runtime.Caching" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Web" />
<Reference Include="System.Web.Abstractions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Xml" />
<Reference Include="Microsoft.CSharp" />
</ItemGroup>
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine.Lucene">
<Version>2.0.0-alpha.20200128.15</Version>
</PackageReference>
<PackageReference Include="Microsoft.SourceLink.GitHub">
<Version>1.0.0-beta2-19554-01</Version>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="NPoco" Version="4.0.2" />
<PackageReference Include="SecurityCodeScan">
<Version>3.3.0</Version>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<Compile Include="BackOfficeExamineSearcher.cs" />
<Compile Include="ExamineExtensions.cs" />
<Compile Include="ExamineLuceneComponent.cs" />
<Compile Include="ExamineLuceneComposer.cs" />
<Compile Include="ExamineLuceneFinalComponent.cs" />
<Compile Include="ExamineLuceneFinalComposer.cs" />
<Compile Include="LuceneIndexDiagnostics.cs" />
<Compile Include="LuceneIndexDiagnosticsFactory.cs" />
<Compile Include="NoPrefixSimpleFsLockFactory.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="UmbracoContentIndex.cs" />
<Compile Include="UmbracoExamineIndexDiagnostics.cs" />
<Compile Include="UmbracoExamineIndex.cs" />
<Compile Include="LuceneIndexCreator.cs" />
<Compile Include="UmbracoIndexesCreator.cs" />
<Compile Include="UmbracoMemberIndex.cs" />
<Compile Include="..\SolutionInfo.cs">
<Link>Properties\SolutionInfo.cs</Link>
</Compile>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Umbraco.Abstractions\Umbraco.Abstractions.csproj">
<Project>{29aa69d9-b597-4395-8d42-43b1263c240a}</Project>
<Name>Umbraco.Abstractions</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<Project>{f9b7fe05-0f93-4d0d-9c10-690b33ecbbd8}</Project>
<Name>Umbraco.Examine</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
<Name>Umbraco.Infrastructure</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>
@@ -20,7 +20,7 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoContentIndex : UmbracoExamineIndex, IUmbracoContentIndex
{
public const string VariesByCultureFieldName = SpecialFieldPrefix + "VariesByCulture";
protected ILocalizationService LanguageService { get; }
#region Constructors
@@ -132,7 +132,7 @@ namespace Umbraco.Examine
{
//find all descendants based on path
var descendantPath = $@"\-1\,*{nodeId}\,*";
var rawQuery = $"{IndexPathFieldName}:{descendantPath}";
var rawQuery = $"{UmbracoExamineFieldNames.IndexPathFieldName}:{descendantPath}";
var searcher = GetSearcher();
var c = searcher.CreateQuery();
var filtered = c.NativeQuery(rawQuery);
@@ -28,19 +28,7 @@ namespace Umbraco.Examine
// call context (and the database it can contain)! ideally we should be able to override
// SafelyProcessQueueItems but that's not possible in the current version of Examine.
/// <summary>
/// Used to store the path of a content object
/// </summary>
public const string IndexPathFieldName = SpecialFieldPrefix + "Path";
public const string NodeKeyFieldName = SpecialFieldPrefix + "Key";
public const string UmbracoFileFieldName = "umbracoFileSrc";
public const string IconFieldName = SpecialFieldPrefix + "Icon";
public const string PublishedFieldName = SpecialFieldPrefix + "Published";
/// <summary>
/// The prefix added to a field when it is duplicated in order to store the original raw value.
/// </summary>
public const string RawFieldPrefix = SpecialFieldPrefix + "Raw_";
/// <summary>
/// Create a new <see cref="UmbracoExamineIndex"/>
@@ -141,7 +129,7 @@ namespace Umbraco.Examine
{
var d = docArgs.Document;
foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix)).ToList())
foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(UmbracoExamineFieldNames.RawFieldPrefix)).ToList())
{
if (f.Value.Count > 0)
{
@@ -182,13 +170,13 @@ namespace Umbraco.Examine
var path = e.ValueSet.GetValue("path");
if (path != null)
{
e.ValueSet.Set(IndexPathFieldName, path);
e.ValueSet.Set(UmbracoExamineFieldNames.IndexPathFieldName, path);
}
//icon
if (e.ValueSet.Values.TryGetValue("icon", out var icon) && e.ValueSet.Values.ContainsKey(IconFieldName) == false)
if (e.ValueSet.Values.TryGetValue("icon", out var icon) && e.ValueSet.Values.ContainsKey(UmbracoExamineFieldNames.IconFieldName) == false)
{
e.ValueSet.Values[IconFieldName] = icon;
e.ValueSet.Values[UmbracoExamineFieldNames.IconFieldName] = icon;
}
}
@@ -1,17 +1,14 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using Umbraco.Core.Logging;
using Umbraco.Core.Services;
using Umbraco.Examine;
using Lucene.Net.Analysis.Standard;
using Examine.LuceneEngine;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.IO;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <summary>
/// Creates the indexes used by Umbraco
@@ -53,7 +50,7 @@ namespace Umbraco.Web.Search
/// <returns></returns>
public override IEnumerable<IIndex> Create()
{
return new []
return new[]
{
CreateInternalIndex(),
CreateExternalIndex(),
@@ -1,6 +1,4 @@
using System.Collections.Generic;
using Examine;
using Examine.LuceneEngine;
using Examine;
using Lucene.Net.Analysis;
using Umbraco.Core;
using Umbraco.Core.IO;
@@ -1,5 +1,4 @@
using System.Collections.Generic;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
@@ -6,7 +6,6 @@ using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
namespace Umbraco.Examine
@@ -46,16 +46,16 @@ namespace Umbraco.Examine
var values = new Dictionary<string, IEnumerable<object>>
{
{"icon", c.ContentType.Icon?.Yield() ?? Enumerable.Empty<string>()},
{UmbracoExamineIndex.PublishedFieldName, new object[] {c.Published ? "y" : "n"}}, //Always add invariant published value
{UmbracoExamineFieldNames.PublishedFieldName, new object[] {c.Published ? "y" : "n"}}, //Always add invariant published value
{"id", new object[] {c.Id}},
{UmbracoExamineIndex.NodeKeyFieldName, new object[] {c.Key}},
{UmbracoExamineFieldNames.NodeKeyFieldName, new object[] {c.Key}},
{"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}},
{"level", new object[] {c.Level}},
{"creatorID", new object[] {c.CreatorId}},
{"sortOrder", new object[] {c.SortOrder}},
{"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate
{"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate
{"nodeName", (PublishedValuesOnly //Always add invariant nodeName
{UmbracoExamineFieldNames.NodeNameFieldName, (PublishedValuesOnly //Always add invariant nodeName
? c.PublishName?.Yield()
: c.Name?.Yield()) ?? Enumerable.Empty<string>()},
{"urlName", urlValue?.Yield() ?? Enumerable.Empty<string>()}, //Always add invariant urlName
@@ -65,12 +65,12 @@ namespace Umbraco.Examine
{"writerName",(c.GetWriterProfile(_userService)?.Name ?? "??").Yield() },
{"writerID", new object[] {c.WriterId}},
{"templateID", new object[] {c.TemplateId ?? 0}},
{UmbracoContentIndex.VariesByCultureFieldName, new object[] {"n"}},
{UmbracoExamineFieldNames.VariesByCultureFieldName, new object[] {"n"}},
};
if (isVariant)
{
values[UmbracoContentIndex.VariesByCultureFieldName] = new object[] { "y" };
values[UmbracoExamineFieldNames.VariesByCultureFieldName] = new object[] { "y" };
foreach (var culture in c.AvailableCultures)
{
@@ -80,7 +80,7 @@ namespace Umbraco.Examine
values[$"nodeName_{lowerCulture}"] = (PublishedValuesOnly
? c.GetPublishName(culture)?.Yield()
: c.GetCultureName(culture)?.Yield()) ?? Enumerable.Empty<string>();
values[$"{UmbracoExamineIndex.PublishedFieldName}_{lowerCulture}"] = (c.IsCulturePublished(culture) ? "y" : "n").Yield<object>();
values[$"{UmbracoExamineFieldNames.PublishedFieldName}_{lowerCulture}"] = (c.IsCulturePublished(culture) ? "y" : "n").Yield<object>();
values[$"updateDate_{lowerCulture}"] = (PublishedValuesOnly
? c.GetPublishDate(culture)
: c.GetUpdateDate(culture))?.Yield<object>() ?? Enumerable.Empty<object>();
@@ -1,10 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Examine
@@ -92,18 +89,18 @@ namespace Umbraco.Examine
//check for published content
if (valueSet.Category == IndexTypes.Content && PublishedValuesOnly)
{
if (!valueSet.Values.TryGetValue(UmbracoExamineIndex.PublishedFieldName, out var published))
if (!valueSet.Values.TryGetValue(UmbracoExamineFieldNames.PublishedFieldName, out var published))
return ValueSetValidationResult.Failed;
if (!published[0].Equals("y"))
return ValueSetValidationResult.Failed;
//deal with variants, if there are unpublished variants than we need to remove them from the value set
if (valueSet.Values.TryGetValue(UmbracoContentIndex.VariesByCultureFieldName, out var variesByCulture)
if (valueSet.Values.TryGetValue(UmbracoExamineFieldNames.VariesByCultureFieldName, out var variesByCulture)
&& variesByCulture.Count > 0 && variesByCulture[0].Equals("y"))
{
//so this valueset is for a content that varies by culture, now check for non-published cultures and remove those values
foreach(var publishField in valueSet.Values.Where(x => x.Key.StartsWith($"{UmbracoExamineIndex.PublishedFieldName}_")).ToList())
foreach(var publishField in valueSet.Values.Where(x => x.Key.StartsWith($"{UmbracoExamineFieldNames.PublishedFieldName}_")).ToList())
{
if (publishField.Value.Count <= 0 || !publishField.Value[0].Equals("y"))
{
@@ -4,9 +4,8 @@ using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Examine;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <summary>
@@ -0,0 +1,17 @@
using Examine;
using System.Collections.Generic;
using Umbraco.Web.Models.ContentEditing;
namespace Umbraco.Examine
{
/// <summary>
/// Used to search the back office for Examine indexed entities (Documents, Media and Members)
/// </summary>
public interface IBackOfficeExamineSearcher
{
IEnumerable<ISearchResult> Search(string query,
UmbracoEntityTypes entityType,
int pageSize,
long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false);
}
}
+2 -2
View File
@@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using Umbraco.Core;
namespace Umbraco.Examine
{
/// <summary>
/// Exposes diagnostic information about an index
/// </summary>
@@ -0,0 +1,13 @@
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Creates <see cref="IIndexDiagnostics"/> for an index if it doesn't implement <see cref="IIndexDiagnostics"/>
/// </summary>
public interface IIndexDiagnosticsFactory
{
IIndexDiagnostics Create(IIndex index);
}
}
@@ -1,8 +1,4 @@
using System.Collections.Generic;
using Examine;
using Umbraco.Examine;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <inheritdoc />
/// <summary>
@@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <summary>
/// Used to propagate hardcoded internal Field lists
@@ -0,0 +1,17 @@
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Default implementation of <see cref="IIndexDiagnosticsFactory"/> which returns <see cref="GenericIndexDiagnostics"/> for indexes that don't have an implementation
/// </summary>
public class IndexDiagnosticsFactory : IIndexDiagnosticsFactory
{
public virtual IIndexDiagnostics Create(IIndex index)
{
if (!(index is IIndexDiagnostics indexDiag))
indexDiag = new GenericIndexDiagnostics(index);
return indexDiag;
}
}
}
+9 -1
View File
@@ -5,7 +5,7 @@ using System.Threading.Tasks;
using Examine;
namespace Umbraco.Examine
{
{
/// <summary>
/// Utility to rebuild all indexes ensuring minimal data queries
@@ -45,6 +45,8 @@ namespace Umbraco.Examine
if (indexes.Length == 0) return;
OnRebuildingIndexes(new IndexRebuildingEventArgs(indexes));
foreach (var index in indexes)
{
index.CreateIndex(); // clear the index
@@ -54,5 +56,11 @@ namespace Umbraco.Examine
Parallel.ForEach(_populators, populator => populator.Populate(indexes));
}
/// <summary>
/// Event raised when indexes are being rebuilt
/// </summary>
public event EventHandler<IndexRebuildingEventArgs> RebuildingIndexes;
private void OnRebuildingIndexes(IndexRebuildingEventArgs args) => RebuildingIndexes?.Invoke(this, args);
}
}
@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using Examine;
namespace Umbraco.Examine
{
public class IndexRebuildingEventArgs : EventArgs
{
public IndexRebuildingEventArgs(IEnumerable<IIndex> indexes)
{
Indexes = indexes;
}
/// <summary>
/// The indexes being rebuilt
/// </summary>
public IEnumerable<IIndex> Indexes { get; }
}
}
@@ -1,7 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
+8 -6
View File
@@ -2,7 +2,6 @@
using Examine;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Umbraco.Core.Models;
@@ -10,6 +9,7 @@ using Umbraco.Core.PropertyEditors;
using Umbraco.Core.PropertyEditors.ValueConverters;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Umbraco.Core.Serialization;
namespace Umbraco.Examine
{
@@ -19,16 +19,18 @@ namespace Umbraco.Examine
private readonly IUserService _userService;
private readonly ILogger _logger;
private readonly IShortStringHelper _shortStringHelper;
private readonly IJsonSerializer _serializer;
public MediaValueSetBuilder(PropertyEditorCollection propertyEditors,
UrlSegmentProviderCollection urlSegmentProviders,
IUserService userService, ILogger logger, IShortStringHelper shortStringHelper)
IUserService userService, ILogger logger, IShortStringHelper shortStringHelper, IJsonSerializer serializer)
: base(propertyEditors, false)
{
_urlSegmentProviders = urlSegmentProviders;
_userService = userService;
_logger = logger;
_shortStringHelper = shortStringHelper;
_serializer = serializer;
}
/// <inheritdoc />
@@ -48,7 +50,7 @@ namespace Umbraco.Examine
ImageCropperValue cropper = null;
try
{
cropper = JsonConvert.DeserializeObject<ImageCropperValue>(
cropper = _serializer.Deserialize<ImageCropperValue>(
m.GetValue<string>(Constants.Conventions.Media.File));
}
catch (Exception ex)
@@ -77,19 +79,19 @@ namespace Umbraco.Examine
{
{"icon", m.ContentType.Icon?.Yield() ?? Enumerable.Empty<string>()},
{"id", new object[] {m.Id}},
{UmbracoExamineIndex.NodeKeyFieldName, new object[] {m.Key}},
{UmbracoExamineFieldNames.NodeKeyFieldName, new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", m.Name?.Yield() ?? Enumerable.Empty<string>()},
{UmbracoExamineFieldNames.NodeNameFieldName, m.Name?.Yield() ?? Enumerable.Empty<string>()},
{"urlName", urlValue?.Yield() ?? Enumerable.Empty<string>()},
{"path", m.Path?.Yield() ?? Enumerable.Empty<string>()},
{"nodeType", m.ContentType.Id.ToString().Yield() },
{"creatorName", (m.GetCreatorProfile(_userService)?.Name ?? "??").Yield()},
{UmbracoExamineIndex.UmbracoFileFieldName, umbracoFile.Yield()}
{UmbracoExamineFieldNames.UmbracoFileFieldName, umbracoFile.Yield()}
};
foreach (var property in m.Properties)
@@ -1,7 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using Examine;
using Lucene.Net.Util;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
+2 -2
View File
@@ -24,14 +24,14 @@ namespace Umbraco.Examine
{
{"icon", m.ContentType.Icon?.Yield() ?? Enumerable.Empty<string>()},
{"id", new object[] {m.Id}},
{UmbracoExamineIndex.NodeKeyFieldName, new object[] {m.Key}},
{UmbracoExamineFieldNames.NodeKeyFieldName, new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", m.Name?.Yield() ?? Enumerable.Empty<string>()},
{UmbracoExamineFieldNames.NodeNameFieldName, m.Name?.Yield() ?? Enumerable.Empty<string>()},
{"path", m.Path?.Yield() ?? Enumerable.Empty<string>()},
{"nodeType", m.ContentType.Id.ToString().Yield() },
{"loginName", m.Username?.Yield() ?? Enumerable.Empty<string>()},
@@ -23,7 +23,7 @@ namespace Umbraco.Examine
/// <summary>
/// By default these are the member fields we index
/// </summary>
public static readonly string[] DefaultMemberIndexFields = { "id", "nodeName", "updateDate", "loginName", "email", UmbracoExamineIndex.NodeKeyFieldName };
public static readonly string[] DefaultMemberIndexFields = { "id", UmbracoExamineFieldNames.NodeNameFieldName, "updateDate", "loginName", "email", UmbracoExamineFieldNames.NodeKeyFieldName };
private static readonly IEnumerable<string> ValidCategories = new[] { IndexTypes.Member };
protected override IEnumerable<string> ValidIndexCategories => ValidCategories;
+10 -111
View File
@@ -1,119 +1,18 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<ProjectGuid>{07FBC26B-2927-4A22-8D96-D644C667FECC}</ProjectGuid>
<OutputType>Library</OutputType>
<TargetFramework>netstandard2.0</TargetFramework>
<AssemblyName>Umbraco.Examine</AssemblyName>
<RootNamespace>Umbraco.Examine</RootNamespace>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<TargetFrameworkProfile />
<AdditionalFileItemNames>$(AdditionalFileItemNames);Content</AdditionalFileItemNames>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>portable</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>portable</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Release\Umbraco.Examine.xml</DocumentationFile>
<Prefer32Bit>false</Prefer32Bit>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Runtime.Caching" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Web" />
<Reference Include="System.Web.Abstractions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Xml" />
<Reference Include="Microsoft.CSharp" />
<PackageReference Include="Examine.Core" Version="2.0.0-alpha.20200128.15" />
</ItemGroup>
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine" Version="1.0.2" />
<PackageReference Include="Microsoft.SourceLink.GitHub">
<Version>1.0.0-beta2-19554-01</Version>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="NPoco" Version="4.0.2" />
<PackageReference Include="SecurityCodeScan">
<Version>3.3.0</Version>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<ProjectReference Include="..\Umbraco.Abstractions\Umbraco.Abstractions.csproj" />
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj" />
</ItemGroup>
<ItemGroup>
<Compile Include="BaseValueSetBuilder.cs" />
<Compile Include="ContentIndexPopulator.cs" />
<Compile Include="ContentValueSetBuilder.cs" />
<Compile Include="ExamineExtensions.cs" />
<Compile Include="IContentValueSetBuilder.cs" />
<Compile Include="IContentValueSetValidator.cs" />
<Compile Include="IUmbracoContentIndex.cs" />
<Compile Include="IUmbracoIndexConfig.cs" />
<Compile Include="IIndexCreator.cs" />
<Compile Include="IIndexDiagnostics.cs" />
<Compile Include="IIndexPopulator.cs" />
<Compile Include="IUmbracoMemberIndex.cs" />
<Compile Include="UmbracoIndexConfig.cs" />
<Compile Include="IndexPopulator.cs" />
<Compile Include="IndexRebuilder.cs" />
<Compile Include="IPublishedContentValueSetBuilder.cs" />
<Compile Include="IUmbracoIndex.cs" />
<Compile Include="IValueSetBuilder.cs" />
<Compile Include="LuceneIndexDiagnostics.cs" />
<Compile Include="MediaIndexPopulator.cs" />
<Compile Include="MediaValueSetBuilder.cs" />
<Compile Include="MemberIndexPopulator.cs" />
<Compile Include="MemberValueSetBuilder.cs" />
<Compile Include="MemberValueSetValidator.cs" />
<Compile Include="PublishedContentIndexPopulator.cs" />
<Compile Include="UmbracoExamineExtensions.cs" />
<Compile Include="IndexTypes.cs" />
<Compile Include="NoPrefixSimpleFsLockFactory.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="UmbracoContentIndex.cs" />
<Compile Include="ContentValueSetValidator.cs" />
<Compile Include="UmbracoExamineIndexDiagnostics.cs" />
<Compile Include="UmbracoExamineIndex.cs" />
<Compile Include="LuceneIndexCreator.cs" />
<Compile Include="UmbracoFieldDefinitionCollection.cs" />
<Compile Include="UmbracoMemberIndex.cs" />
<Compile Include="..\SolutionInfo.cs">
<Link>Properties\SolutionInfo.cs</Link>
</Compile>
<Compile Include="ValueSetValidator.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Umbraco.Abstractions\Umbraco.Abstractions.csproj">
<Project>{29aa69d9-b597-4395-8d42-43b1263c240a}</Project>
<Name>Umbraco.Abstractions</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
<Name>Umbraco.Infrastructure</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>
</Project>
@@ -1,11 +1,68 @@
using Examine.LuceneEngine.Search;
using Examine;
using Examine.Search;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using Umbraco.Core;
namespace Umbraco.Examine
{
public static class UmbracoExamineExtensions
{
/// <summary>
/// Matches a culture iso name suffix
/// </summary>
/// <remarks>
/// myFieldName_en-us will match the "en-us"
/// </remarks>
internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled);
//TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression
/// <summary>
/// Returns all index fields that are culture specific (suffixed)
/// </summary>
/// <param name="index"></param>
/// <param name="culture"></param>
/// <returns></returns>
public static IEnumerable<string> GetCultureFields(this IUmbracoIndex index, string culture)
{
var allFields = index.GetFields();
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var field in allFields)
{
var match = CultureIsoCodeFieldNameMatchExpression.Match(field);
if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value))
yield return field;
}
}
/// <summary>
/// Returns all index fields that are culture specific (suffixed) or invariant
/// </summary>
/// <param name="index"></param>
/// <param name="culture"></param>
/// <returns></returns>
public static IEnumerable<string> GetCultureAndInvariantFields(this IUmbracoIndex index, string culture)
{
var allFields = index.GetFields();
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var field in allFields)
{
var match = CultureIsoCodeFieldNameMatchExpression.Match(field);
if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value))
{
yield return field; //matches this culture field
}
else if (!match.Success)
{
yield return field; //matches no culture field (invariant)
}
}
}
public static IBooleanOperation Id(this IQuery query, int id)
{
var fieldQuery = query.Id(id.ToInvariantString());
@@ -32,7 +89,7 @@ namespace Umbraco.Examine
/// <returns></returns>
public static IBooleanOperation NodeName(this IQuery query, string nodeName)
{
var fieldQuery = query.Field("nodeName", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeName));
var fieldQuery = query.Field(UmbracoExamineFieldNames.NodeNameFieldName, (IExamineValue)new ExamineValue(Examineness.Explicit, nodeName));
return fieldQuery;
}
@@ -44,7 +101,7 @@ namespace Umbraco.Examine
/// <returns></returns>
public static IBooleanOperation NodeName(this IQuery query, IExamineValue nodeName)
{
var fieldQuery = query.Field("nodeName", nodeName);
var fieldQuery = query.Field(UmbracoExamineFieldNames.NodeNameFieldName, nodeName);
return fieldQuery;
}
@@ -56,7 +113,7 @@ namespace Umbraco.Examine
/// <returns></returns>
public static IBooleanOperation NodeTypeAlias(this IQuery query, string nodeTypeAlias)
{
var fieldQuery = query.Field("__NodeTypeAlias", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeTypeAlias));
var fieldQuery = query.Field(ExamineFieldNames.ItemTypeFieldName, (IExamineValue)new ExamineValue(Examineness.Explicit, nodeTypeAlias));
return fieldQuery;
}
@@ -68,10 +125,9 @@ namespace Umbraco.Examine
/// <returns></returns>
public static IBooleanOperation NodeTypeAlias(this IQuery query, IExamineValue nodeTypeAlias)
{
var fieldQuery = query.Field("__NodeTypeAlias", nodeTypeAlias);
var fieldQuery = query.Field(ExamineFieldNames.ItemTypeFieldName, nodeTypeAlias);
return fieldQuery;
}
}
}
@@ -0,0 +1,25 @@
using Examine;
namespace Umbraco.Examine
{
public static class UmbracoExamineFieldNames
{
/// <summary>
/// Used to store the path of a content object
/// </summary>
public const string IndexPathFieldName = ExamineFieldNames.SpecialFieldPrefix + "Path";
public const string NodeKeyFieldName = ExamineFieldNames.SpecialFieldPrefix + "Key";
public const string UmbracoFileFieldName = "umbracoFileSrc";
public const string IconFieldName = ExamineFieldNames.SpecialFieldPrefix + "Icon";
public const string PublishedFieldName = ExamineFieldNames.SpecialFieldPrefix + "Published";
/// <summary>
/// The prefix added to a field when it is duplicated in order to store the original raw value.
/// </summary>
public const string RawFieldPrefix = ExamineFieldNames.SpecialFieldPrefix + "Raw_";
public const string VariesByCultureFieldName = ExamineFieldNames.SpecialFieldPrefix + "VariesByCulture";
public const string NodeNameFieldName = "nodeName";
}
}
@@ -33,7 +33,7 @@ namespace Umbraco.Examine
new FieldDefinition("createDate", FieldDefinitionTypes.DateTime),
new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime),
new FieldDefinition(UmbracoExamineIndex.NodeKeyFieldName, FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition(UmbracoExamineFieldNames.NodeKeyFieldName, FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition("version", FieldDefinitionTypes.Raw),
new FieldDefinition("nodeType", FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition("template", FieldDefinitionTypes.Raw),
@@ -42,10 +42,10 @@ namespace Umbraco.Examine
new FieldDefinition("email", FieldDefinitionTypes.EmailAddress),
new FieldDefinition(UmbracoExamineIndex.PublishedFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineIndex.IndexPathFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineIndex.IconFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoContentIndex.VariesByCultureFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineFieldNames.PublishedFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineFieldNames.IconFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(UmbracoExamineFieldNames.VariesByCultureFieldName, FieldDefinitionTypes.Raw),
};
@@ -76,7 +76,7 @@ namespace Umbraco.Examine
if (!fieldName.Contains("_") || !fieldName.Contains("-"))
return false;
var match = ExamineExtensions.CultureIsoCodeFieldNameMatchExpression.Match(fieldName);
var match = UmbracoExamineExtensions.CultureIsoCodeFieldNameMatchExpression.Match(fieldName);
if (match.Success && match.Groups.Count == 3)
{
var nonCultureFieldName = match.Groups[1].Value;
+1 -3
View File
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
namespace Umbraco.Examine
@@ -70,9 +70,9 @@
<Project>{29aa69d9-b597-4395-8d42-43b1263c240a}</Project>
<Name>Umbraco.Abstractions</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<ProjectReference Include="..\Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj">
<Project>{07fbc26b-2927-4a22-8d96-d644c667fecc}</Project>
<Name>Umbraco.Examine</Name>
<Name>Umbraco.Examine.Lucene</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
@@ -1,9 +1,9 @@
using System;
using Examine;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Xml.XPath;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Logging;
@@ -59,7 +59,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
ValidateAndSetProperty(valueDictionary, val => _sortOrder = Int32.Parse(val), "sortOrder");
ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName");
ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName");
ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndex.ItemTypeFieldName);
ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", ExamineFieldNames.ItemTypeFieldName);
ValidateAndSetProperty(valueDictionary, val => _documentTypeId = Int32.Parse(val), "nodeType");
//ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName");
ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132
@@ -43,6 +43,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
private readonly IEntityXmlSerializer _entitySerializer;
private readonly IUmbracoContextAccessor _umbracoContextAccessor;
private readonly IVariationContextAccessor _variationContextAccessor;
private readonly IExamineManager _examineManager = new ExamineManager();
// must be specified by the ctor
private readonly IAppCache _appCache;
@@ -119,7 +120,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
// +(+parentID:-1) +__IndexType:media
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineIndex.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = filter.Execute();
if (result != null)
@@ -229,29 +230,14 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
public override bool HasContent(bool preview) { throw new NotImplementedException(); }
private static IExamineManager GetExamineManagerSafe()
{
try
{
return ExamineManager.Instance;
}
catch (TypeInitializationException)
{
return null;
}
}
private ISearcher GetSearchProviderSafe()
{
if (_searchProvider != null)
return _searchProvider;
var eMgr = GetExamineManagerSafe();
if (eMgr == null) return null;
try
{
return eMgr.TryGetIndex(Constants.UmbracoIndexes.InternalIndexName, out var index) ? index.GetSearcher() : null;
return _examineManager.TryGetIndex(Constants.UmbracoIndexes.InternalIndexName, out var index) ? index.GetSearcher() : null;
}
catch (FileNotFoundException)
{
@@ -303,7 +289,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
// note that since the use of the wildcard, it automatically escapes it in Lucene.
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineIndex.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = filter.Execute().FirstOrDefault();
if (result != null) return ConvertFromSearchResult(result);
@@ -485,7 +471,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
{
//We are going to check for a special field however, that is because in some cases we store a 'Raw'
//value in the index such as for xml/html.
var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineIndex.RawFieldPrefix + alias));
var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineFieldNames.RawFieldPrefix + alias));
return rawValue
?? dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias));
}
@@ -518,7 +504,7 @@ namespace Umbraco.Tests.LegacyXmlPublishedCache
//first check in Examine as this is WAY faster
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineIndex.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard())
var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard())
.OrderBy(new SortableField("sortOrder", SortType.Int));
//the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene.
//+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media
@@ -108,7 +108,7 @@ namespace Umbraco.Tests.Runtimes
composition.RegisterUnique(f => new DistributedCache(f.GetInstance<IServerMessenger>(), f.GetInstance<CacheRefresherCollection>()));
composition.WithCollectionBuilder<UrlProviderCollectionBuilder>().Append<DefaultUrlProvider>();
composition.RegisterUnique<IDistributedCacheBinder, DistributedCacheBinder>();
composition.RegisterUnique<IExamineManager>(f => ExamineManager.Instance);
composition.RegisterUnique<IExamineManager, ExamineManager>();
composition.RegisterUnique<IUmbracoContextFactory, UmbracoContextFactory>();
composition.RegisterUnique<IMacroRenderer, MacroRenderer>();
composition.RegisterUnique<MediaUrlProviderCollection>(_ => new MediaUrlProviderCollection(Enumerable.Empty<IMediaUrlProvider>()));
@@ -23,6 +23,7 @@ using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Persistence;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Runtime;
using Umbraco.Core.Serialization;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
@@ -89,6 +90,7 @@ namespace Umbraco.Tests.TestHelpers
}
public static IShortStringHelper ShortStringHelper { get; } = new DefaultShortStringHelper(new DefaultShortStringHelperConfig());
public static IJsonSerializer JsonSerializer { get; } = new JsonNetSerializer();
public static IVariationContextAccessor VariationContextAccessor { get; } = new TestVariationContextAccessor();
public static IDbProviderFactoryCreator DbProviderFactoryCreator { get; } = new UmbracoDbProviderFactoryCreator(Constants.DbProviderNames.SqlCe);
public static IBulkSqlInsertProvider BulkSqlInsertProvider { get; } = new SqlCeBulkSqlInsertProvider();
+1 -1
View File
@@ -393,7 +393,7 @@ namespace Umbraco.Tests.Testing
Composition.RegisterUnique(factory => factory.GetInstance<IUmbracoSettingsSection>().Content);
Composition.RegisterUnique(factory => factory.GetInstance<IUmbracoSettingsSection>().WebRouting);
Composition.RegisterUnique<IExamineManager>(factory => ExamineManager.Instance);
Composition.RegisterUnique<IExamineManager, ExamineManager>();
Composition.RegisterUnique<IJsonSerializer, JsonNetSerializer>();
+10 -4
View File
@@ -78,13 +78,14 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Castle.Core" Version="4.3.1" />
<PackageReference Include="Examine" Version="1.0.2" />
<PackageReference Include="Examine.Core">
<Version>2.0.0-alpha.20200128.15</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack">
<Version>1.8.14</Version>
</PackageReference>
<PackageReference Include="LightInject" Version="5.4.0" />
<PackageReference Include="LightInject.Annotation" Version="1.1.0" />
<PackageReference Include="Lucene.Net" Version="3.0.3" />
<PackageReference Include="Lucene.Net.Contrib" Version="3.0.3" />
<PackageReference Include="Microsoft.AspNet.Identity.Core" Version="2.2.2" />
<PackageReference Include="Microsoft.AspNet.Mvc" Version="5.2.7" />
@@ -146,6 +147,7 @@
<Compile Include="Persistence\Mappers\MapperTestBase.cs" />
<Compile Include="Persistence\Repositories\DocumentRepositoryTest.cs" />
<Compile Include="Persistence\Repositories\EntityRepositoryTest.cs" />
<Compile Include="UmbracoExamine\ExamineExtensions.cs" />
<Compile Include="PublishedContent\NuCacheChildrenTests.cs" />
<Compile Include="PublishedContent\PublishedContentLanguageVariantTests.cs" />
<Compile Include="PublishedContent\PublishedContentSnapshotTestBase.cs" />
@@ -562,6 +564,10 @@
<Project>{fbe7c065-dac0-4025-a78b-63b24d3ab00b}</Project>
<Name>Umbraco.Configuration</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<Project>{f9b7fe05-0f93-4d0d-9c10-690b33ecbbd8}</Project>
<Name>Umbraco.Examine</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
<Name>Umbraco.Infrastructure</Name>
@@ -578,9 +584,9 @@
<Project>{651E1350-91B6-44B7-BD60-7207006D7003}</Project>
<Name>Umbraco.Web</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<ProjectReference Include="..\Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj">
<Project>{07fbc26b-2927-4a22-8d96-d644c667fecc}</Project>
<Name>Umbraco.Examine</Name>
<Name>Umbraco.Examine.Lucene</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
@@ -0,0 +1,134 @@
using Examine;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Xml.Linq;
namespace Umbraco.Tests.UmbracoExamine
{
/// <summary>
/// LEGACY!! Static methods to help query umbraco xml
/// </summary>
/// <remarks>
/// This should be deleted when we remove the old xml published content with tests which should be replaced with nucache tests
/// </remarks>
internal static class ExamineExtensions
{
/// <summary>
/// Returns true if the XElement is recognized as an umbraco xml NODE (doc type)
/// </summary>
/// <param name="x"></param>
/// <returns></returns>
internal static bool IsExamineElement(this XElement x)
{
var id = (string)x.Attribute("id");
if (string.IsNullOrEmpty(id))
return false;
int parsedId;
if (int.TryParse(id, out parsedId))
if (parsedId > 0)
return true;
return false;
}
/// <summary>
/// This takes into account both schemas and returns the node type alias.
/// If this isn't recognized as an element node, this returns an empty string
/// </summary>
/// <param name="x"></param>
/// <returns></returns>
internal static string ExamineNodeTypeAlias(this XElement x)
{
return string.IsNullOrEmpty((string)x.Attribute("nodeTypeAlias"))
? x.Name.LocalName
: (string)x.Attribute("nodeTypeAlias");
}
/// <summary>
/// Returns umbraco value for a data element with the specified alias.
/// </summary>
/// <param name="xml"></param>
/// <param name="alias"></param>
/// <returns></returns>
internal static string SelectExamineDataValue(this XElement xml, string alias)
{
XElement nodeData = null;
//if there is data children with attributes, we're on the old
if (xml.Elements("data").Any(x => x.HasAttributes))
nodeData = xml.Elements("data").SingleOrDefault(x => string.Equals((string)x.Attribute("alias"), alias, StringComparison.InvariantCultureIgnoreCase));
else
nodeData = xml.Elements().FirstOrDefault(x => string.Equals(x.Name.ToString(), alias, StringComparison.InvariantCultureIgnoreCase));
if (nodeData == null)
return string.Empty;
if (!nodeData.HasElements)
return nodeData.Value;
//it has sub elements so serialize them
var reader = nodeData.CreateReader();
reader.MoveToContent();
return reader.ReadInnerXml();
}
[EditorBrowsable(EditorBrowsableState.Never)]
public static ValueSet ConvertToValueSet(this XElement xml, string indexCategory)
{
if (!xml.IsExamineElement())
throw new InvalidOperationException("Not a supported Examine XML structure");
var allVals = xml.SelectExamineAllValues();
var id = (string)xml.Attribute("id");
//we will use this as the item type, but we also need to add this as the 'nodeTypeAlias' as part of the properties
//since this is what Umbraco expects
var nodeTypeAlias = xml.ExamineNodeTypeAlias();
var set = new ValueSet(id, indexCategory, nodeTypeAlias, allVals);
set.Set("nodeTypeAlias", nodeTypeAlias);
return set;
}
internal static Dictionary<string, object> SelectExamineAllValues(this XElement xml)
{
var attributeValues = xml.Attributes().ToDictionary(x => x.Name.LocalName, x => x.Value);
var dataValues = xml.SelectExamineDataValues();
foreach (var v in attributeValues)
//override the data values with attribute values if they do match, otherwise add
dataValues[v.Key] = v.Value;
return dataValues;
}
internal static Dictionary<string, object> SelectExamineDataValues(this XElement xml)
{
//resolve all element data at once since it is much faster to do this than to relookup all of the XML data
//using Linq and the node.Elements() methods re-gets all of them.
var elementValues = new Dictionary<string, object>();
foreach (var x in xml.Elements())
{
if (x.Attribute("id") != null)
continue;
string key;
if (x.Name.LocalName == "data")
//it's the legacy schema
key = (string)x.Attribute("alias");
else
key = x.Name.LocalName;
if (string.IsNullOrEmpty(key))
continue;
if (!x.HasElements)
elementValues[key] = x.Value;
else
//it has sub elements so serialize them
using (var reader = x.CreateReader())
{
reader.MoveToContent();
elementValues[key] = reader.ReadInnerXml();
}
}
return elementValues;
}
}
}
@@ -47,7 +47,7 @@ namespace Umbraco.Tests.UmbracoExamine
public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService)
{
var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), GetMockUserService(), GetMockLogger(), TestHelper.ShortStringHelper);
var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), GetMockUserService(), GetMockLogger(), TestHelper.ShortStringHelper, TestHelper.JsonSerializer);
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
return mediaIndexDataSource;
}
@@ -68,8 +68,8 @@ namespace Umbraco.Tests.UmbracoExamine
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<ISimpleContentType>(mt =>
@@ -108,8 +108,8 @@ namespace Umbraco.Tests.UmbracoExamine
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<ISimpleContentType>(mt =>
@@ -114,8 +114,8 @@ namespace Umbraco.Tests.UmbracoExamine
Assert.AreEqual("value2", result.AllValues["grid.row1"][1]);
Assert.IsTrue(result.Values.ContainsKey("grid"));
Assert.AreEqual("value1 value2 ", result["grid"]);
Assert.IsTrue(result.Values.ContainsKey($"{UmbracoExamineIndex.RawFieldPrefix}grid"));
Assert.AreEqual(json, result[$"{UmbracoExamineIndex.RawFieldPrefix}grid"]);
Assert.IsTrue(result.Values.ContainsKey($"{UmbracoExamineFieldNames.RawFieldPrefix}grid"));
Assert.AreEqual(json, result[$"{UmbracoExamineFieldNames.RawFieldPrefix}grid"]);
}
}
@@ -165,12 +165,12 @@ namespace Umbraco.Tests.UmbracoExamine
var protectedQuery = new BooleanQuery();
protectedQuery.Add(
new BooleanClause(
new TermQuery(new Term(LuceneIndex.CategoryFieldName, IndexTypes.Content)),
new TermQuery(new Term(ExamineFieldNames.CategoryFieldName, IndexTypes.Content)),
Occur.MUST));
protectedQuery.Add(
new BooleanClause(
new TermQuery(new Term(LuceneIndex.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
new TermQuery(new Term(ExamineFieldNames.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
Occur.MUST));
var collector = TopScoreDocCollector.Create(100, true);
@@ -287,7 +287,7 @@ namespace Umbraco.Tests.UmbracoExamine
//create the whole thing
rebuilder.Populate(indexer);
var result = searcher.CreateQuery().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Execute();
var result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute();
Assert.AreEqual(21, result.TotalItemCount);
//delete all content
@@ -298,13 +298,13 @@ namespace Umbraco.Tests.UmbracoExamine
//ensure it's all gone
result = searcher.CreateQuery().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Execute();
result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute();
Assert.AreEqual(0, result.TotalItemCount);
//call our indexing methods
rebuilder.Populate(indexer);
result = searcher.CreateQuery().Field(LuceneIndex.CategoryFieldName, IndexTypes.Content).Execute();
result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute();
Assert.AreEqual(21, result.TotalItemCount);
}
}
@@ -38,8 +38,8 @@ namespace Umbraco.Tests.UmbracoExamine
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.Published == true &&
@@ -179,7 +179,7 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndex.PublishedFieldName] = "y"
[UmbracoExamineFieldNames.PublishedFieldName] = "y"
}));
Assert.AreEqual(ValueSetValidationResult.Valid, result);
}
@@ -213,7 +213,7 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndex.PublishedFieldName] = "n"
[UmbracoExamineFieldNames.PublishedFieldName] = "n"
}));
Assert.AreEqual(ValueSetValidationResult.Failed, result);
@@ -222,7 +222,7 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndex.PublishedFieldName] = "y"
[UmbracoExamineFieldNames.PublishedFieldName] = "y"
}));
Assert.AreEqual(ValueSetValidationResult.Valid, result);
}
@@ -237,8 +237,8 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndex.VariesByCultureFieldName] = "y",
[UmbracoExamineIndex.PublishedFieldName] = "n"
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "y",
[UmbracoExamineFieldNames.PublishedFieldName] = "n"
}));
Assert.AreEqual(ValueSetValidationResult.Failed, result);
@@ -247,8 +247,8 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndex.VariesByCultureFieldName] = "y",
[UmbracoExamineIndex.PublishedFieldName] = "y"
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "y",
[UmbracoExamineFieldNames.PublishedFieldName] = "y"
}));
Assert.AreEqual(ValueSetValidationResult.Valid, result);
@@ -257,17 +257,17 @@ namespace Umbraco.Tests.UmbracoExamine
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndex.VariesByCultureFieldName] = "y",
[$"{UmbracoExamineIndex.PublishedFieldName}_en-us"] = "y",
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "y",
[$"{UmbracoExamineFieldNames.PublishedFieldName}_en-us"] = "y",
["hello_en-us"] = "world",
["title_en-us"] = "my title",
[$"{UmbracoExamineIndex.PublishedFieldName}_es-es"] = "n",
[$"{UmbracoExamineFieldNames.PublishedFieldName}_es-es"] = "n",
["hello_es-ES"] = "world",
["title_es-ES"] = "my title",
[UmbracoExamineIndex.PublishedFieldName] = "y"
[UmbracoExamineFieldNames.PublishedFieldName] = "y"
});
Assert.AreEqual(10, valueSet.Values.Count());
Assert.IsTrue(valueSet.Values.ContainsKey($"{UmbracoExamineIndex.PublishedFieldName}_es-es"));
Assert.IsTrue(valueSet.Values.ContainsKey($"{UmbracoExamineFieldNames.PublishedFieldName}_es-es"));
Assert.IsTrue(valueSet.Values.ContainsKey("hello_es-ES"));
Assert.IsTrue(valueSet.Values.ContainsKey("title_es-ES"));
@@ -275,7 +275,7 @@ namespace Umbraco.Tests.UmbracoExamine
Assert.AreEqual(ValueSetValidationResult.Filtered, result);
Assert.AreEqual(7, valueSet.Values.Count()); //filtered to 7 values (removes es-es values)
Assert.IsFalse(valueSet.Values.ContainsKey($"{UmbracoExamineIndex.PublishedFieldName}_es-es"));
Assert.IsFalse(valueSet.Values.ContainsKey($"{UmbracoExamineFieldNames.PublishedFieldName}_es-es"));
Assert.IsFalse(valueSet.Values.ContainsKey("hello_es-ES"));
Assert.IsFalse(valueSet.Values.ContainsKey("title_es-ES"));
}
@@ -42,17 +42,17 @@ namespace Umbraco.Tests.Web
indexer.IndexItem(new ValueSet("1", "content", new Dictionary<string, object>
{
[fieldNames[0]] = "Hello world, there are products here",
[UmbracoContentIndex.VariesByCultureFieldName] = "n"
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "n"
}));
indexer.IndexItem(new ValueSet("2", "content", new Dictionary<string, object>
{
[fieldNames[1]] = "Hello world, there are products here",
[UmbracoContentIndex.VariesByCultureFieldName] = "y"
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "y"
}));
indexer.IndexItem(new ValueSet("3", "content", new Dictionary<string, object>
{
[fieldNames[2]] = "Hello world, there are products here",
[UmbracoContentIndex.VariesByCultureFieldName] = "y"
[UmbracoExamineFieldNames.VariesByCultureFieldName] = "y"
}));
}
+2 -3
View File
@@ -86,7 +86,6 @@
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="ClientDependency" Version="1.9.8" />
<PackageReference Include="ClientDependency-Mvc5" Version="1.9.3" />
<PackageReference Include="Examine" Version="1.0.2" />
<PackageReference Include="ImageProcessor.Web" Version="4.10.0.100" />
<PackageReference Include="ImageProcessor.Web.Config" Version="2.5.0.100" />
<PackageReference Include="Microsoft.AspNet.Identity.Owin" Version="2.2.2" />
@@ -117,8 +116,8 @@
<Project>{29aa69d9-b597-4395-8d42-43b1263c240a}</Project>
<Name>Umbraco.Abstractions</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<Name>Umbraco.Examine</Name>
<ProjectReference Include="..\Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj">
<Name>Umbraco.Examine.Lucene</Name>
<Project>{07FBC26B-2927-4A22-8D96-D644C667FECC}</Project>
</ProjectReference>
<ProjectReference Include="..\Umbraco.ModelsBuilder.Embedded\Umbraco.ModelsBuilder.Embedded.csproj">
@@ -5,7 +5,6 @@ using System.Net;
using System.Net.Http;
using System.Web.Http;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.IO;
@@ -24,17 +23,19 @@ namespace Umbraco.Web.Editors
private readonly IExamineManager _examineManager;
private readonly ILogger _logger;
private readonly IIOHelper _ioHelper;
private readonly IIndexDiagnosticsFactory _indexDiagnosticsFactory;
private readonly IAppPolicyCache _runtimeCache;
private readonly IndexRebuilder _indexRebuilder;
public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper,
public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper, IIndexDiagnosticsFactory indexDiagnosticsFactory,
AppCaches appCaches,
IndexRebuilder indexRebuilder)
{
_examineManager = examineManager;
_logger = logger;
_ioHelper = ioHelper;
_indexDiagnosticsFactory = indexDiagnosticsFactory;
_runtimeCache = appCaches.RuntimeCache;
_indexRebuilder = indexRebuilder;
}
@@ -69,9 +70,8 @@ namespace Umbraco.Web.Editors
if (!msg.IsSuccessStatusCode)
throw new HttpResponseException(msg);
var results = Examine.ExamineExtensions.TryParseLuceneQuery(query)
? searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1))
: searcher.Search(query, maxResults: pageSize * (pageIndex + 1));
// NativeQuery will work for a single word/phrase too (but depends on the implementation) the lucene one will work.
var results = searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1));
var pagedResults = results.Skip(pageIndex * pageSize);
@@ -173,19 +173,11 @@ namespace Umbraco.Web.Editors
}
}
private ExamineIndexModel CreateModel(IIndex index)
{
var indexName = index.Name;
if (!(index is IIndexDiagnostics indexDiag))
{
if (index is LuceneIndex luceneIndex)
indexDiag = new LuceneIndexDiagnostics(luceneIndex, Logger, _ioHelper);
else
indexDiag = new GenericIndexDiagnostics(index);
}
var indexDiag = _indexDiagnosticsFactory.Create(index);
var isHealth = indexDiag.IsHealthy();
var properties = new Dictionary<string, object>
+1 -3
View File
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Examine;
using Umbraco.Web.PublishedCache;
@@ -65,7 +63,7 @@ namespace Umbraco.Web
foreach (var result in results)
{
if (int.TryParse(result.Id, out var contentId) &&
result.Values.TryGetValue(LuceneIndex.CategoryFieldName, out var indexType))
result.Values.TryGetValue(ExamineFieldNames.CategoryFieldName, out var indexType))
{
IPublishedContent content;
switch (indexType)
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Mapping;
using Umbraco.Core.Models;
@@ -171,13 +169,13 @@ namespace Umbraco.Web.Models.Mapping
// TODO: Properly map this (not aftermap)
//get the icon if there is one
target.Icon = source.Values.ContainsKey(UmbracoExamineIndex.IconFieldName)
? source.Values[UmbracoExamineIndex.IconFieldName]
target.Icon = source.Values.ContainsKey(UmbracoExamineFieldNames.IconFieldName)
? source.Values[UmbracoExamineFieldNames.IconFieldName]
: Constants.Icons.DefaultIcon;
target.Name = source.Values.ContainsKey("nodeName") ? source.Values["nodeName"] : "[no name]";
target.Name = source.Values.ContainsKey(UmbracoExamineFieldNames.NodeNameFieldName) ? source.Values[UmbracoExamineFieldNames.NodeNameFieldName] : "[no name]";
if (source.Values.TryGetValue(UmbracoExamineIndex.UmbracoFileFieldName, out var umbracoFile))
if (source.Values.TryGetValue(UmbracoExamineFieldNames.UmbracoFileFieldName, out var umbracoFile))
{
if (umbracoFile != null)
{
@@ -185,16 +183,16 @@ namespace Umbraco.Web.Models.Mapping
}
}
if (source.Values.ContainsKey(UmbracoExamineIndex.NodeKeyFieldName))
if (source.Values.ContainsKey(UmbracoExamineFieldNames.NodeKeyFieldName))
{
if (Guid.TryParse(source.Values[UmbracoExamineIndex.NodeKeyFieldName], out var key))
if (Guid.TryParse(source.Values[UmbracoExamineFieldNames.NodeKeyFieldName], out var key))
{
target.Key = key;
//need to set the UDI
if (source.Values.ContainsKey(LuceneIndex.CategoryFieldName))
if (source.Values.ContainsKey(ExamineFieldNames.CategoryFieldName))
{
switch (source.Values[LuceneIndex.CategoryFieldName])
switch (source.Values[ExamineFieldNames.CategoryFieldName])
{
case IndexTypes.Member:
target.Udi = new GuidUdi(Constants.UdiEntityType.Member, target.Key);
@@ -222,11 +220,11 @@ namespace Umbraco.Web.Models.Mapping
}
}
target.Path = source.Values.ContainsKey(UmbracoExamineIndex.IndexPathFieldName) ? source.Values[UmbracoExamineIndex.IndexPathFieldName] : "";
target.Path = source.Values.ContainsKey(UmbracoExamineFieldNames.IndexPathFieldName) ? source.Values[UmbracoExamineFieldNames.IndexPathFieldName] : "";
if (source.Values.ContainsKey(LuceneIndex.ItemTypeFieldName))
if (source.Values.ContainsKey(ExamineFieldNames.ItemTypeFieldName))
{
target.AdditionalData.Add("contentType", source.Values[LuceneIndex.ItemTypeFieldName]);
target.AdditionalData.Add("contentType", source.Values[ExamineFieldNames.ItemTypeFieldName]);
}
}
@@ -63,7 +63,7 @@ namespace Umbraco.Web.PropertyEditors
}
//First save the raw value to a raw field
result.Add(new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineIndex.RawFieldPrefix}{property.Alias}", new[] { rawVal }));
result.Add(new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineFieldNames.RawFieldPrefix}{property.Alias}", new[] { rawVal }));
if (sb.Length > 0)
{
@@ -185,7 +185,7 @@ namespace Umbraco.Web.PropertyEditors
//index the stripped HTML values
yield return new KeyValuePair<string, IEnumerable<object>>(property.Alias, new object[] { strVal.StripHtml() });
//store the raw value
yield return new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineIndex.RawFieldPrefix}{property.Alias}", new object[] { strVal });
yield return new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineFieldNames.RawFieldPrefix}{property.Alias}", new object[] { strVal });
}
}
}
@@ -28,6 +28,7 @@ namespace Umbraco.Web
private static UmbracoContext UmbracoContext => Current.UmbracoContext;
private static ISiteDomainHelper SiteDomainHelper => Current.Factory.GetInstance<ISiteDomainHelper>();
private static IVariationContextAccessor VariationContextAccessor => Current.VariationContextAccessor;
private static IExamineManager ExamineManager => Current.Factory.GetInstance<IExamineManager>();
#region IsComposedOf
@@ -198,7 +199,7 @@ namespace Umbraco.Web
// TODO: inject examine manager
indexName = string.IsNullOrEmpty(indexName) ? Constants.UmbracoIndexes.ExternalIndexName : indexName;
if (!ExamineManager.Instance.TryGetIndex(indexName, out var index))
if (!ExamineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found with name " + indexName);
var searcher = index.GetSearcher();
@@ -207,7 +208,7 @@ namespace Umbraco.Web
//var luceneQuery = "+__Path:(" + content.Path.Replace("-", "\\-") + "*) +" + t;
var query = searcher.CreateQuery()
.Field(UmbracoExamineIndex.IndexPathFieldName, (content.Path + ",").MultipleCharacterWildcard())
.Field(UmbracoExamineFieldNames.IndexPathFieldName, (content.Path + ",").MultipleCharacterWildcard())
.And()
.ManagedQuery(term);
@@ -219,7 +220,7 @@ namespace Umbraco.Web
// TODO: inject examine manager
indexName = string.IsNullOrEmpty(indexName) ? Constants.UmbracoIndexes.ExternalIndexName : indexName;
if (!ExamineManager.Instance.TryGetIndex(indexName, out var index))
if (!ExamineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found with name " + indexName);
var searcher = index.GetSearcher();
+1 -1
View File
@@ -216,7 +216,7 @@ namespace Umbraco.Web
else if (string.IsNullOrWhiteSpace(culture))
{
// Only search invariant
queryExecutor = query.Field(UmbracoContentIndex.VariesByCultureFieldName, "n") // Must not vary by culture
queryExecutor = query.Field(UmbracoExamineFieldNames.VariesByCultureFieldName, "n") // Must not vary by culture
.And().ManagedQuery(term);
}
else
@@ -46,6 +46,7 @@ using Umbraco.Web.Trees;
using Umbraco.Web.WebApi;
using Current = Umbraco.Web.Composing.Current;
using Umbraco.Web.PropertyEditors;
using Umbraco.Examine;
namespace Umbraco.Web.Runtime
{
@@ -152,7 +153,7 @@ namespace Umbraco.Web.Runtime
composition.RegisterUnique<IDashboardService, DashboardService>();
composition.RegisterUnique<IExamineManager>(factory => ExamineManager.Instance);
composition.RegisterUnique<IExamineManager, ExamineManager>();
// configure the container for web
composition.ConfigureForWeb();
@@ -118,7 +118,6 @@ namespace Umbraco.Web.Search
if (_waitMilliseconds > 0)
Thread.Sleep(_waitMilliseconds);
_indexRebuilder.ExamineManager.ConfigureIndexes(_mainDom, _logger);
_indexRebuilder.RebuildIndexes(_onlyEmptyIndexes);
}
}
@@ -13,9 +13,6 @@ using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;
using Umbraco.Web.Cache;
using Umbraco.Examine;
using Examine.LuceneEngine.Directories;
using Umbraco.Web.Composing;
using System.ComponentModel;
namespace Umbraco.Web.Search
{
@@ -65,15 +62,6 @@ namespace Umbraco.Web.Search
public void Initialize()
{
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryFactory.DefaultLockFactory = d =>
{
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
//let's deal with shutting down Examine with MainDom
var examineShutdownRegistered = _mainDom.Register(() =>
{
+1 -6
View File
@@ -30,7 +30,7 @@ namespace Umbraco.Web.Search
composition.Register<IndexRebuilder>(Lifetime.Singleton);
composition.RegisterUnique<IUmbracoIndexConfig, UmbracoIndexConfig>();
composition.RegisterUnique<IUmbracoIndexesCreator, UmbracoIndexesCreator>();
composition.RegisterUnique<IIndexDiagnosticsFactory, IndexDiagnosticsFactory>();
composition.RegisterUnique<IPublishedContentValueSetBuilder>(factory =>
new ContentValueSetBuilder(
factory.GetInstance<PropertyEditorCollection>(),
@@ -48,11 +48,6 @@ namespace Umbraco.Web.Search
composition.RegisterUnique<IValueSetBuilder<IMedia>, MediaValueSetBuilder>();
composition.RegisterUnique<IValueSetBuilder<IMember>, MemberValueSetBuilder>();
composition.RegisterUnique<BackgroundIndexRebuilder>();
//We want to manage Examine's AppDomain shutdown sequence ourselves so first we'll disable Examine's default behavior
//and then we'll use MainDom to control Examine's shutdown - this MUST be done in Compose ie before ExamineManager
//is instantiated, as the value is used during instantiation
ExamineManager.DisableDefaultHostingEnvironmentRegistration();
}
}
}
@@ -12,15 +12,11 @@ namespace Umbraco.Web.Search
/// </summary>
public sealed class ExamineFinalComponent : IComponent
{
private readonly IProfilingLogger _logger;
private readonly IExamineManager _examineManager;
BackgroundIndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
public ExamineFinalComponent(BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
{
_logger = logger;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_mainDom = mainDom;
}
@@ -29,8 +25,6 @@ namespace Umbraco.Web.Search
{
if (!_mainDom.IsMainDom) return;
_examineManager.ConfigureIndexes(_mainDom, _logger);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
_indexRebuilder.RebuildIndexes(true, 5000);
}
+8 -308
View File
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Mapping;
@@ -22,29 +20,26 @@ namespace Umbraco.Web.Search
/// </summary>
public class UmbracoTreeSearcher
{
private readonly IExamineManager _examineManager;
private readonly UmbracoContext _umbracoContext;
private readonly ILocalizationService _languageService;
private readonly IEntityService _entityService;
private readonly UmbracoMapper _mapper;
private readonly ISqlContext _sqlContext;
private readonly IUmbracoTreeSearcherFields _umbracoTreeSearcherFields;
private readonly IBackOfficeExamineSearcher _backOfficeExamineSearcher;
public UmbracoTreeSearcher(IExamineManager examineManager,
UmbracoContext umbracoContext,
public UmbracoTreeSearcher(UmbracoContext umbracoContext,
ILocalizationService languageService,
IEntityService entityService,
UmbracoMapper mapper,
ISqlContext sqlContext,IUmbracoTreeSearcherFields umbracoTreeSearcherFields)
ISqlContext sqlContext, IBackOfficeExamineSearcher backOfficeExamineSearcher)
{
_examineManager = examineManager ?? throw new ArgumentNullException(nameof(examineManager));
_umbracoContext = umbracoContext;
_languageService = languageService;
_entityService = entityService;
_mapper = mapper;
_sqlContext = sqlContext;
_umbracoTreeSearcherFields = umbracoTreeSearcherFields;
_backOfficeExamineSearcher = backOfficeExamineSearcher;
}
/// <summary>
@@ -66,72 +61,7 @@ namespace Umbraco.Web.Search
int pageSize,
long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false)
{
var sb = new StringBuilder();
string type;
var indexName = Constants.UmbracoIndexes.InternalIndexName;
var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList();
// TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
//special GUID check since if a user searches on one specifically we need to escape it
if (Guid.TryParse(query, out var g))
{
query = "\"" + g.ToString() + "\"";
}
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMembersFields());
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
{
sb.Append("+__NodeTypeAlias:");
sb.Append(searchFrom);
sb.Append(" ");
}
break;
case UmbracoEntityTypes.Media:
type = "media";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMediaFields());
var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeDocumentFields());
var allContentStartNodes = _umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
}
if (!_examineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found by name " + indexName);
var internalSearcher = index.GetSearcher();
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<SearchResultEntity>();
}
var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString())
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Execute(Convert.ToInt32(pageSize * (pageIndex + 1)));
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
var pagedResult = _backOfficeExamineSearcher.Search(query, entityType, pageSize, pageIndex, out totalFound, searchFrom, ignoreUserStartNodes);
switch (entityType)
{
@@ -166,236 +96,6 @@ namespace Umbraco.Web.Search
return _mapper.MapEnumerable<IEntitySlim, SearchResultEntity>(results);
}
private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List<string> fields, string type)
{
//build a lucene query:
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
|| Regex.IsMatch(query, "^\'.*?\'$");
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim('\"', '\'');
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (query.IsNullOrWhiteSpace() == false)
{
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
//additional fields normally
sb.Append(f);
sb.Append(": (");
sb.Append(query);
sb.Append(") ");
}
sb.Append(") ");
}
}
else
{
var trimmed = query.Trim(new[] { '\"', '\'' });
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (trimmed.IsNullOrWhiteSpace() == false)
{
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
var queryWordsReplaced = new string[querywords.Length];
// when searching file names containing hyphens we need to replace the hyphens with spaces
if (f.Equals(UmbracoExamineIndex.UmbracoFileFieldName))
{
for (var index = 0; index < querywords.Length; index++)
{
queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" ");
}
}
else
{
queryWordsReplaced = querywords;
}
//additional fields normally
sb.Append(f);
sb.Append(":");
sb.Append("(");
foreach (var w in queryWordsReplaced)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(")");
sb.Append(" ");
}
sb.Append(") ");
}
}
//must match index type
sb.Append("+__IndexType:");
sb.Append(type);
return true;
}
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService)
{
if (sb == null) throw new ArgumentNullException(nameof(sb));
if (entityService == null) throw new ArgumentNullException(nameof(entityService));
UdiParser.TryParse(searchFrom, true, out var udi);
searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString();
var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0
? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault()
: null;
if (entityPath != null)
{
// find... only what's underneath
sb.Append("+__Path:");
AppendPath(sb, entityPath.Path, false);
sb.Append(" ");
}
else if (startNodeIds.Length == 0)
{
// make sure we don't find anything
sb.Append("+__Path:none ");
}
else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction
{
var entityPaths = entityService.GetAllPaths(objectType, startNodeIds);
// for each start node, find the start node, and what's underneath
// +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...)
sb.Append("+__Path:(");
var first = true;
foreach (var ep in entityPaths)
{
if (first)
first = false;
else
sb.Append(" ");
AppendPath(sb, ep.Path, true);
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, string path, bool includeThisNode)
{
path = path.Replace("-", "\\-").Replace(",", "\\,");
if (includeThisNode)
{
sb.Append(path);
sb.Append(" ");
}
sb.Append(path);
sb.Append("\\,*");
}
/// <summary>
/// Returns a collection of entities for media based on search results
/// </summary>
@@ -418,9 +118,9 @@ namespace Umbraco.Web.Search
{
m.AdditionalData["Email"] = result.Values["email"];
}
if (result.Values.ContainsKey(UmbracoExamineIndex.NodeKeyFieldName) && result.Values[UmbracoExamineIndex.NodeKeyFieldName] != null)
if (result.Values.ContainsKey(UmbracoExamineFieldNames.NodeKeyFieldName) && result.Values[UmbracoExamineFieldNames.NodeKeyFieldName] != null)
{
if (Guid.TryParse(result.Values[UmbracoExamineIndex.NodeKeyFieldName], out var key))
if (Guid.TryParse(result.Values[UmbracoExamineFieldNames.NodeKeyFieldName], out var key))
{
m.Key = key;
}
@@ -455,7 +155,7 @@ namespace Umbraco.Web.Search
if (intId.Success)
{
//if it varies by culture, return the default language URL
if (result.Values.TryGetValue(UmbracoContentIndex.VariesByCultureFieldName, out var varies) && varies == "y")
if (result.Values.TryGetValue(UmbracoExamineFieldNames.VariesByCultureFieldName, out var varies) && varies == "y")
{
entity.AdditionalData["Url"] = _umbracoContext.Url(intId.Result, defaultLang);
}
@@ -18,7 +18,7 @@ namespace Umbraco.Web.Search
{
return _backOfficeMembersFields;
}
private IReadOnlyList<string> _backOfficeMediaFields = new List<string> {UmbracoExamineIndex.UmbracoFileFieldName };
private IReadOnlyList<string> _backOfficeMediaFields = new List<string> { UmbracoExamineFieldNames.UmbracoFileFieldName };
public IEnumerable<string> GetBackOfficeMediaFields()
{
return _backOfficeMediaFields;
+4 -6
View File
@@ -62,7 +62,9 @@
<ItemGroup>
<PackageReference Include="ClientDependency" Version="1.9.8" />
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="Examine" Version="1.0.2" />
<PackageReference Include="Examine.Core">
<Version>2.0.0-alpha.20200128.15</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack" Version="1.8.14" />
<PackageReference Include="ImageProcessor">
<Version>2.7.0.100</Version>
@@ -111,8 +113,8 @@
<Name>Umbraco.Configuration</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine\Umbraco.Examine.csproj">
<Project>{f9b7fe05-0f93-4d0d-9c10-690b33ecbbd8}</Project>
<Name>Umbraco.Examine</Name>
<Project>{07FBC26B-2927-4A22-8D96-D644C667FECC}</Project>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
@@ -241,7 +243,6 @@
<Compile Include="Security\UmbracoEmailMessage.cs" />
<Compile Include="Security\UmbracoMembershipProviderBase.cs" />
<Compile Include="Security\UserAwarePasswordHasher.cs" />
<Compile Include="Search\IUmbracoTreeSearcherFields.cs" />
<Compile Include="Search\UmbracoTreeSearcherFields.cs" />
<Compile Include="StringExtensions.cs" />
<Compile Include="Templates\HtmlLocalLinkParser.cs" />
@@ -266,9 +267,6 @@
<Compile Include="Runtime\WebInitialComposer.cs" />
<Compile Include="Scheduling\SchedulerComposer.cs" />
<Compile Include="Search\ExamineComposer.cs" />
<Compile Include="Search\GenericIndexDiagnostics.cs" />
<Compile Include="Search\IUmbracoIndexesCreator.cs" />
<Compile Include="Search\UmbracoIndexesCreator.cs" />
<Compile Include="Security\ActiveDirectoryBackOfficeUserPasswordChecker.cs" />
<Compile Include="Security\BackOfficeClaimsIdentityFactory.cs" />
<Compile Include="Security\BackOfficeUserManagerMarker.cs" />
+13 -6
View File
@@ -1,4 +1,5 @@
Microsoft Visual Studio Solution File, Format Version 12.00
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29209.152
MinimumVisualStudioVersion = 10.0.40219.1
@@ -62,7 +63,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Web", "Umbraco.Web\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Tests", "Umbraco.Tests\Umbraco.Tests.csproj", "{5D3B8245-ADA6-453F-A008-50ED04BFE770}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Examine", "Umbraco.Examine\Umbraco.Examine.csproj", "{07FBC26B-2927-4A22-8D96-D644C667FECC}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Examine.Lucene", "Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj", "{07FBC26B-2927-4A22-8D96-D644C667FECC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{E3F9F378-AFE1-40A5-90BD-82833375DBFE}"
ProjectSection(SolutionItems) = preProject
@@ -100,18 +101,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "IssueTemplates", "IssueTemp
..\.github\ISSUE_TEMPLATE\5_Security_issue.md = ..\.github\ISSUE_TEMPLATE\5_Security_issue.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Abstractions", "Umbraco.Abstractions\Umbraco.Abstractions.csproj", "{29AA69D9-B597-4395-8D42-43B1263C240A}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Abstractions", "Umbraco.Abstractions\Umbraco.Abstractions.csproj", "{29AA69D9-B597-4395-8D42-43B1263C240A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.ModelsBuilder.Embedded", "Umbraco.ModelsBuilder.Embedded\Umbraco.ModelsBuilder.Embedded.csproj", "{52AC0BA8-A60E-4E36-897B-E8B97A54ED1C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Infrastructure", "Umbraco.Infrastructure\Umbraco.Infrastructure.csproj", "{3AE7BF57-966B-45A5-910A-954D7C554441}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Infrastructure", "Umbraco.Infrastructure\Umbraco.Infrastructure.csproj", "{3AE7BF57-966B-45A5-910A-954D7C554441}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Configuration", "Umbraco.Configuration\Umbraco.Configuration.csproj", "{FBE7C065-DAC0-4025-A78B-63B24D3AB00B}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Configuration", "Umbraco.Configuration\Umbraco.Configuration.csproj", "{FBE7C065-DAC0-4025-A78B-63B24D3AB00B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Infrastrucure.Persistance.SqlCe", "Umbraco.Infrastrucure.Persistance.SqlCe\Umbraco.Infrastrucure.Persistance.SqlCe.csproj", "{33085570-9BF2-4065-A9B0-A29D920D13BA}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Infrastrucure.Persistance.SqlCe", "Umbraco.Infrastrucure.Persistance.SqlCe\Umbraco.Infrastrucure.Persistance.SqlCe.csproj", "{33085570-9BF2-4065-A9B0-A29D920D13BA}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.TestData", "Umbraco.TestData\Umbraco.TestData.csproj", "{FB5676ED-7A69-492C-B802-E7B24144C0FC}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Examine", "Umbraco.Examine\Umbraco.Examine.csproj", "{F9B7FE05-0F93-4D0D-9C10-690B33ECBBD8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -164,6 +167,10 @@ Global
{FB5676ED-7A69-492C-B802-E7B24144C0FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FB5676ED-7A69-492C-B802-E7B24144C0FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FB5676ED-7A69-492C-B802-E7B24144C0FC}.Release|Any CPU.Build.0 = Release|Any CPU
{F9B7FE05-0F93-4D0D-9C10-690B33ECBBD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F9B7FE05-0F93-4D0D-9C10-690B33ECBBD8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F9B7FE05-0F93-4D0D-9C10-690B33ECBBD8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F9B7FE05-0F93-4D0D-9C10-690B33ECBBD8}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE