Read sitemap products in projected batches instead of whole documents

src/Business/Grand.Business.Catalog/Queries/Handlers/GetSitemapProductsQueryHandler.cs

@@ -0,0 +1,77 @@
+using Grand.Business.Core.Queries.Catalog;
+using Grand.Data;
+using Grand.Data.LiteDb;
+using Grand.Domain.Catalog;
+using Grand.Infrastructure.Configuration;
+using Grand.Mediator;
+
+namespace Grand.Business.Catalog.Queries.Handlers;
+
+public class GetSitemapProductsQueryHandler : IRequestHandler<GetSitemapProductsQuery, IList<SitemapProduct>>
+{
+    private readonly AccessControlConfig _accessControlConfig;
+    private readonly CatalogSettings _catalogSettings;
+    private readonly IRepository<Product> _productRepository;
+
+    public GetSitemapProductsQueryHandler(
+        IRepository<Product> productRepository,
+        CatalogSettings catalogSettings,
+        AccessControlConfig accessControlConfig)
+    {
+        _productRepository = productRepository;
+        _catalogSettings = catalogSettings;
+        _accessControlConfig = accessControlConfig;
+    }
+
+    public async Task<IList<SitemapProduct>> Handle(GetSitemapProductsQuery request,
+        CancellationToken cancellationToken)
+    {
+        //the same products GetSearchProductsQuery returns for a storefront search without keywords
+        var query = _productRepository.Table.Where(p => p.Published && p.VisibleIndividually);
+
+        if (!_catalogSettings.IgnoreFilterableAvailableStartEndDateTime)
+        {
+            var nowUtc = DateTime.UtcNow;
+            query = query.Where(p =>
+                (p.AvailableStartDateTimeUtc == null || p.AvailableStartDateTimeUtc < nowUtc) &&
+                (p.AvailableEndDateTimeUtc == null || p.AvailableEndDateTimeUtc > nowUtc));
+        }
+
+        if (!_accessControlConfig.IgnoreAcl)
+        {
+            var groupIds = request.CustomerGroupIds?.ToArray() ?? [];
+            query = query.Where(p => !p.LimitedToGroups || groupIds.Any(x => p.CustomerGroups.Contains(x)));
+        }
+
+        if (!string.IsNullOrEmpty(request.StoreId) && !_accessControlConfig.IgnoreStoreLimitations)
+            query = query.Where(p => p.Stores.Any(y => y == request.StoreId) || !p.LimitedToStores);
+
+        //keyset paging: a skip would rescan every earlier batch
+        if (!string.IsNullOrEmpty(request.AfterId))
+            query = query.Where(p => string.Compare(p.Id, request.AfterId) > 0);
+
+        //LiteDB materialises the whole collection on every query, so batching there would read the catalog
+        //once per batch: the rest of the catalog comes back in one go instead
+        IQueryable<Product> ordered = query.OrderBy(p => p.Id);
+        if (_productRepository is not LiteDBRepository<Product>)
+            ordered = ordered.Take(request.PageSize);
+
+        //only the fields of a sitemap entry leave the database, not the whole product
+        var languageId = request.LanguageId;
+        var batch = ordered.Select(p => new {
+            p.Id,
+            p.SeName,
+            TranslatedSeName = p.Locales.Where(l => l.LanguageId == languageId && l.LocaleKey == "SeName")
+                .Select(l => l.LocaleValue).FirstOrDefault(),
+            PictureId = p.ProductPictures.Select(x => x.PictureId).FirstOrDefault(),
+            p.UpdatedOnUtc,
+            p.CreatedOnUtc
+        });
+
+        return (await _productRepository.ToListAsync(batch, cancellationToken))
+            .Select(p => new SitemapProduct(p.Id,
+                string.IsNullOrEmpty(p.TranslatedSeName) ? p.SeName : p.TranslatedSeName,
+                p.PictureId, p.UpdatedOnUtc ?? p.CreatedOnUtc))
+            .ToList();
+    }
+}

src/Business/Grand.Business.Core/Queries/Catalog/GetSitemapProductsQuery.cs

@@ -0,0 +1,28 @@
+using Grand.Mediator;
+
+namespace Grand.Business.Core.Queries.Catalog;
+
+/// <summary>
+///     One batch of the products a sitemap lists: published, visible individually, available now, in the store and
+///     visible to the given customer groups. Batches follow product id, so the next one starts after the last id
+/// </summary>
+public class GetSitemapProductsQuery : IRequest<IList<SitemapProduct>>
+{
+    public string StoreId { get; set; }
+    public IList<string> CustomerGroupIds { get; set; } = [];
+    public string LanguageId { get; set; }
+
+    /// <summary>
+    ///     Id of the last product of the previous batch; null for the first batch
+    /// </summary>
+    public string AfterId { get; set; }
+
+    public int PageSize { get; set; } = 1000;
+}
+
+/// <summary>
+///     The fields of a product a sitemap entry needs
+/// </summary>
+/// <param name="SeName">Search engine name in the requested language, or the default one</param>
+/// <param name="PictureId">The first product picture; null when the product has none</param>
+public record SitemapProduct(string Id, string SeName, string PictureId, DateTime UpdatedOnUtc);

src/Business/Grand.Business.Messages/Commands/Handlers/Common/GetSitemapXMLCommandHandler.cs

@@ -28,6 +28,7 @@ namespace Grand.Business.Messages.Commands.Handlers.Common;
 public class GetSitemapXmlCommandHandler : IRequestHandler<GetSitemapXmlCommand, string>
 {
     private const string DateFormat = "yyyy-MM-dd";
+    private const int ProductBatchSize = 1000;
     private readonly AppConfig _appConfig;
     private readonly IBlogService _blogService;
     private readonly BlogSettings _blogSettings;
@@ -49,6 +50,7 @@ public class GetSitemapXmlCommandHandler : IRequestHandler<GetSitemapXmlCommand,
     //so the audience is passed explicitly instead of coming from the (absent) work context
     private string[] _guestGroupIds;
     private GetSitemapXmlCommand _request;
+    private CancellationToken _cancellationToken;
 
     public GetSitemapXmlCommandHandler(
         ICategoryService categoryService,
@@ -87,6 +89,7 @@ public GetSitemapXmlCommandHandler(
     public async Task<string> Handle(GetSitemapXmlCommand request, CancellationToken cancellationToken)
     {
         _request = request;
+        _cancellationToken = cancellationToken;
         var guests = await _groupService.GetCustomerGroupBySystemName(SystemCustomerGroupNames.Guests);
         _guestGroupIds = guests != null ? [guests.Id] : [];
         return await Generate(request.Language, request.Store);
@@ -292,35 +295,43 @@ private async Task<IEnumerable<SitemapUrl>> GetBrandUrls(Language language, Stor
 
     private async Task<IEnumerable<SitemapUrl>> GetProductUrls(Language language, Store store)
     {
-        var search = await _mediator.Send(new GetSearchProductsQuery {
-            CustomerGroupIds = _guestGroupIds,
-            StoreId = store.Id,
-            VisibleIndividuallyOnly = true,
-            OrderBy = ProductSortingEnum.CreatedOn
-        });
         var storeLocation = GetStoreLocation();
         var products = new List<SitemapUrl>();
-        foreach (var product in search.products)
+        //a large catalog is read in batches of the few fields an entry needs, never as whole documents at once
+        IList<SitemapProduct> batch;
+        string afterId = null;
+        do
         {
-            var url =
-                _appConfig.SeoFriendlyUrlsForLanguagesEnabled
-                    ? _linkGenerator.GetUriByRouteValues("Product",
-                        new { SeName = product.GetSeName(language.Id), language = language.UniqueSeoCode },
-                        GetHttpProtocol(), GetHost())
-                    : _linkGenerator.GetUriByRouteValues("Product", new { SeName = product.GetSeName(language.Id) },
-                        GetHttpProtocol(), GetHost());
+            _cancellationToken.ThrowIfCancellationRequested();
+            batch = await _mediator.Send(new GetSitemapProductsQuery {
+                StoreId = store.Id,
+                CustomerGroupIds = _guestGroupIds,
+                LanguageId = language.Id,
+                AfterId = afterId,
+                PageSize = ProductBatchSize
+            }, _cancellationToken);
+
+            foreach (var product in batch)
+            {
+                var url =
+                    _appConfig.SeoFriendlyUrlsForLanguagesEnabled
+                        ? _linkGenerator.GetUriByRouteValues("Product",
+                            new { product.SeName, language = language.UniqueSeoCode },
+                            GetHttpProtocol(), GetHost())
+                        : _linkGenerator.GetUriByRouteValues("Product", new { product.SeName },
+                            GetHttpProtocol(), GetHost());
 
-            var imageUrl = string.Empty;
-            if (_commonSettings.SitemapIncludeImage)
-                if (!string.IsNullOrEmpty(product.ProductPictures.FirstOrDefault()?.PictureId))
-                {
-                    imageUrl = await _pictureService.GetPictureUrl(product.ProductPictures.FirstOrDefault()?.PictureId,
-                        showDefaultPicture: false, storeLocation: storeLocation);
-                }
+                var imageUrl = string.Empty;
+                if (_commonSettings.SitemapIncludeImage && !string.IsNullOrEmpty(product.PictureId))
+                    imageUrl = await _pictureService.GetPictureUrl(product.PictureId, showDefaultPicture: false,
+                        storeLocation: storeLocation);
 
-            products.Add(new SitemapUrl(url, imageUrl, UpdateFrequency.Weekly,
-                product.UpdatedOnUtc ?? product.CreatedOnUtc));
-        }
+                products.Add(new SitemapUrl(url, imageUrl, UpdateFrequency.Weekly, product.UpdatedOnUtc));
+            }
+
+            if (batch.Count > 0)
+                afterId = batch[^1].Id;
+        } while (batch.Count == ProductBatchSize);
 
         return products;
     }

src/Tests/Grand.Business.Catalog.Tests/Queries/Handlers/GetSitemapProductsQueryHandlerTests.cs

@@ -0,0 +1,163 @@
+using Grand.Business.Catalog.Queries.Handlers;
+using Grand.Business.Core.Queries.Catalog;
+using Grand.Data;
+using Grand.Data.Tests.LiteDb;
+using Grand.Data.Tests.MongoDb;
+using Grand.Domain.Catalog;
+using Grand.Domain.Localization;
+using Grand.Infrastructure.Configuration;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Grand.Business.Catalog.Tests.Queries.Handlers;
+
+[TestClass]
+public class GetSitemapProductsQueryHandlerTests
+{
+    private const string StoreId = "store1";
+    private const string GuestsGroupId = "guests-group";
+    private GetSitemapProductsQueryHandler _handler;
+    private IRepository<Product> _repository;
+
+    [TestInitialize]
+    public void Init()
+    {
+        _repository = new MongoDBRepositoryTest<Product>();
+        _handler = new GetSitemapProductsQueryHandler(_repository, new CatalogSettings(), new AccessControlConfig());
+    }
+
+    private Task<IList<SitemapProduct>> Handle(string afterId = null, int pageSize = 1000)
+    {
+        return _handler.Handle(new GetSitemapProductsQuery {
+            StoreId = StoreId,
+            CustomerGroupIds = [GuestsGroupId],
+            LanguageId = "lang1",
+            AfterId = afterId,
+            PageSize = pageSize
+        }, CancellationToken.None);
+    }
+
+    [TestMethod]
+    public async Task Handle_ReturnsOnlyProductsAGuestCanSeeInTheStore()
+    {
+        //Arrange
+        await _repository.InsertAsync(new Product { Id = "open", Published = true, VisibleIndividually = true });
+        await _repository.InsertAsync(new Product {
+            Id = "guests", Published = true, VisibleIndividually = true, LimitedToGroups = true,
+            CustomerGroups = [GuestsGroupId]
+        });
+        await _repository.InsertAsync(new Product {
+            Id = "admins", Published = true, VisibleIndividually = true, LimitedToGroups = true,
+            CustomerGroups = ["admins-group"]
+        });
+        await _repository.InsertAsync(new Product {
+            Id = "this-store", Published = true, VisibleIndividually = true, LimitedToStores = true,
+            Stores = [StoreId]
+        });
+        await _repository.InsertAsync(new Product {
+            Id = "other-store", Published = true, VisibleIndividually = true, LimitedToStores = true,
+            Stores = ["store2"]
+        });
+        await _repository.InsertAsync(new Product { Id = "draft", Published = false, VisibleIndividually = true });
+        await _repository.InsertAsync(new Product { Id = "variant", Published = true, VisibleIndividually = false });
+        await _repository.InsertAsync(new Product {
+            Id = "expired", Published = true, VisibleIndividually = true,
+            AvailableEndDateTimeUtc = DateTime.UtcNow.AddDays(-1)
+        });
+        await _repository.InsertAsync(new Product {
+            Id = "upcoming", Published = true, VisibleIndividually = true,
+            AvailableStartDateTimeUtc = DateTime.UtcNow.AddDays(1)
+        });
+
+        //Act
+        var result = await Handle();
+
+        //Assert
+        CollectionAssert.AreEquivalent(new[] { "open", "guests", "this-store" }, result.Select(p => p.Id).ToList());
+    }
+
+    [TestMethod]
+    public async Task Handle_ProjectsTheTranslatedSeNameFirstPictureAndLastChange()
+    {
+        //Arrange
+        var updatedOn = new DateTime(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc);
+        var product = new Product {
+            Id = "translated", SeName = "default-name", Published = true, VisibleIndividually = true,
+            Locales = [
+                new TranslationEntity { LanguageId = "lang2", LocaleKey = "SeName", LocaleValue = "other-name" },
+                new TranslationEntity { LanguageId = "lang1", LocaleKey = "Name", LocaleValue = "Name" },
+                new TranslationEntity { LanguageId = "lang1", LocaleKey = "SeName", LocaleValue = "translated-name" }
+            ]
+        };
+        product.ProductPictures.Add(new ProductPicture { PictureId = "picture1" });
+        product.ProductPictures.Add(new ProductPicture { PictureId = "picture2" });
+        await _repository.InsertAsync(product);
+        await _repository.UpdateField(product.Id, p => p.UpdatedOnUtc, updatedOn);
+        await _repository.InsertAsync(new Product {
+            Id = "untranslated", SeName = "default-only", Published = true, VisibleIndividually = true
+        });
+
+        //Act
+        var result = (await Handle()).ToDictionary(p => p.Id);
+
+        //Assert
+        Assert.AreEqual("translated-name", result["translated"].SeName);
+        Assert.AreEqual("picture1", result["translated"].PictureId);
+        Assert.AreEqual(updatedOn, result["translated"].UpdatedOnUtc.ToUniversalTime());
+        Assert.AreEqual("default-only", result["untranslated"].SeName);
+        Assert.IsNull(result["untranslated"].PictureId);
+    }
+
+    [TestMethod]
+    public async Task Handle_PagesByIdAfterTheLastProductOfTheBatch()
+    {
+        //Arrange
+        foreach (var id in new[] { "c", "a", "e", "b", "d" })
+            await _repository.InsertAsync(new Product { Id = id, Published = true, VisibleIndividually = true });
+
+        //Act
+        var first = await Handle(pageSize: 2);
+        var second = await Handle(first[^1].Id, 2);
+        var last = await Handle(second[^1].Id, 2);
+
+        //Assert
+        CollectionAssert.AreEqual(new[] { "a", "b" }, first.Select(p => p.Id).ToList());
+        CollectionAssert.AreEqual(new[] { "c", "d" }, second.Select(p => p.Id).ToList());
+        CollectionAssert.AreEqual(new[] { "e" }, last.Select(p => p.Id).ToList());
+    }
+
+    [TestMethod]
+    public async Task Handle_NoCustomerGroups_ReturnsOnlyProductsNotLimitedToGroups()
+    {
+        //Arrange
+        await _repository.InsertAsync(new Product { Id = "open", Published = true, VisibleIndividually = true });
+        await _repository.InsertAsync(new Product {
+            Id = "guests", Published = true, VisibleIndividually = true, LimitedToGroups = true,
+            CustomerGroups = [GuestsGroupId]
+        });
+
+        //Act
+        var result = await _handler.Handle(new GetSitemapProductsQuery { StoreId = StoreId, CustomerGroupIds = null },
+            CancellationToken.None);
+
+        //Assert
+        CollectionAssert.AreEqual(new[] { "open" }, result.Select(p => p.Id).ToList());
+    }
+
+    [TestMethod]
+    public async Task Handle_LiteDB_ReturnsTheRestOfTheCatalogInOneBatch()
+    {
+        //Arrange
+        var repository = new LiteDBRepositoryMock<Product>();
+        var handler = new GetSitemapProductsQueryHandler(repository, new CatalogSettings(), new AccessControlConfig());
+        foreach (var id in new[] { "c", "a", "e", "b", "d" })
+            await repository.InsertAsync(new Product { Id = id, Published = true, VisibleIndividually = true });
+
+        //Act
+        var result = await handler.Handle(new GetSitemapProductsQuery {
+            StoreId = StoreId, CustomerGroupIds = [GuestsGroupId], AfterId = "a", PageSize = 2
+        }, CancellationToken.None);
+
+        //Assert
+        CollectionAssert.AreEqual(new[] { "b", "c", "d", "e" }, result.Select(p => p.Id).ToList());
+    }
+}