swcs/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v8/dev' into v8/bugfix/models-builder-enable-flag

Browse Source
This commit is contained in:
Shannon
2020-04-20 22:45:50 +10:00
parent 44df0dda2b c3136943e3
commit 8927472457
3 changed files with 114 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs
+42
View File
@@ -18,6 +18,48 @@ namespace Umbraco.Core.Persistence
/// </summary>
public static partial class NPocoDatabaseExtensions
{
/// <summary>
/// Iterates over the result of a paged data set with a db reader
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="database"></param>
/// <param name="pageSize">
/// The number of rows to load per page
/// </param>
/// <param name="sql"></param>
/// <returns></returns>
/// <remarks>
/// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to
/// iterate over each row with a reader using Query vs Fetch.
/// </remarks>
internal static IEnumerable<T> QueryPaged<T>(this IDatabase database, long pageSize, Sql sql)
{
var sqlString = sql.SQL;
var sqlArgs = sql.Arguments;
int? itemCount = null;
long pageIndex = 0;
do
{
// Get the paged queries
database.BuildPageQueries<T>(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var sqlCount, out var sqlPage);
// get the item count once
if (itemCount == null)
{
itemCount = database.ExecuteScalar<int>(sqlCount, sqlArgs);
}
pageIndex++;
// iterate over rows without allocating all items to memory (Query vs Fetch)
foreach (var row in database.Query<T>(sqlPage, sqlArgs))
{
yield return row;
}
} while ((pageIndex * pageSize) < itemCount);
}
// NOTE
//
// proper way to do it with TSQL and SQLCE
src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
+40 -23
View File
@@ -20,6 +20,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
// provides efficient database access for NuCache
internal class DatabaseDataSource : IDataSource
{
private const int PageSize = 500;
// we want arrays, we want them all loaded, not an enumerable
private Sql<ISqlContext> ContentSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
@@ -79,33 +81,43 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
var sql = ContentSourcesSelect(scope, s => s
var sql = ContentSourcesSelect(scope,
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids)
{
if (!ids.Any()) return Enumerable.Empty<ContentNodeKit>();
if (!ids.Any()) yield break;
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
private Sql<ISqlContext> MediaSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
@@ -116,11 +128,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
.AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
.From<NodeDto>();
if (joins != null)
@@ -128,9 +137,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
sql = sql
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit");
return sql;
@@ -152,33 +159,43 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
var sql = MediaSourcesSelect(scope, s => s
.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
var sql = MediaSourcesSelect(scope,
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
{
if (!ids.Any()) return Enumerable.Empty<ContentNodeKit>();
if (!ids.Any()) yield break;
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
private static ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
+32 -6
View File
@@ -5,6 +5,7 @@ using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CSharpTest.Net.Collections;
using Newtonsoft.Json;
using Umbraco.Core;
@@ -868,12 +869,37 @@ namespace Umbraco.Web.PublishedCache.NuCache
//into a new DLL for the application which includes both content types and media types.
//Since the models in the cache are based on these actual classes, all of the objects in the cache need to be updated
//to use the newest version of the class.
using (_contentStore.GetScopedWriteLock(_scopeProvider))
using (_mediaStore.GetScopedWriteLock(_scopeProvider))
{
NotifyLocked(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out var draftChanged, out var publishedChanged);
NotifyLocked(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out var anythingChanged);
}
// NOTE: Ideally this can be run on background threads here which would prevent blocking the UI
// as is the case when saving a content type. Intially one would think that it won't be any different
// between running this here or in another background thread immediately after with regards to how the
// UI will respond because we already know between calling `WithSafeLiveFactoryReset` to reset the PureLive models
// and this code here, that many front-end requests could be attempted to be processed. If that is the case, those pages are going to get a
// model binding error and our ModelBindingExceptionFilter is going to to its magic to reload those pages so the end user is none the wiser.
// So whether or not this executes 'here' or on a background thread immediately wouldn't seem to make any difference except that we can return
// execution to the UI sooner.
// BUT!... there is a difference IIRC. There is still execution logic that continues after this call on this thread with the cache refreshers
// and those cache refreshers need to have the up-to-date data since other user cache refreshers will be expecting the data to be 'live'. If
// we ran this on a background thread then those cache refreshers are going to not get 'live' data when they query the content cache which
// they require.
// These can be run side by side in parallel.
Parallel.Invoke(
() =>
{
using (_contentStore.GetScopedWriteLock(_scopeProvider))
{
NotifyLocked(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _);
}
},
() =>
{
using (_mediaStore.GetScopedWriteLock(_scopeProvider))
{
NotifyLocked(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _);
}
});
}
((PublishedSnapshot)CurrentPublishedSnapshot)?.Resync();