From 83655f13e9ec6a31f0473bbc7c86f92e83472b62 Mon Sep 17 00:00:00 2001 From: Brian Bicknell Date: Sun, 1 Mar 2026 22:07:20 -0500 Subject: [PATCH] Updated various parts of scanning and ingestion, either for bug fixes, or for enhancements. --- .../DLSite/Models/VoiceWorkDetails.cs | 1 + .../Scanning/Contracts/DLSiteWork.cs | 4 +- .../Scanning/Contracts/VoiceWorkIngest.cs | 2 +- .../Scanning/ScanVoiceWorksHandler.cs | 5 +- .../Scanning/ScanVoiceWorksResponse.cs | 5 +- .../Circles/CircleSearchProvider.cs | 63 ---- .../Ingestion/VoiceWorkUpdater.cs | 2 +- .../DLSite/Mapping/DLSiteToDomainMapper.cs | 1 + .../Integrations/DLSite/Models/ProductInfo.cs | 6 + .../Extensions/DLSiteWorkExtensions.cs | 62 ++++ .../Scanning/VoiceWorksScanner.cs | 11 +- ...te_Upcoming_With_No_Expected_Date_Tests.cs | 62 ++++ .../Ingestion/VoiceWorkIngestionTests.cs | 15 +- .../Integrations/DLSite/DLSiteClientTests.cs | 7 + JSMR.Tests/Scanning/VoiceWorkScannerTests.cs | 50 +++- .../DLSiteWorkExpectedDateInferenceTests.cs | 283 ++++++++++++++++++ JSMR.Tests/Unit/DLSiteWorkTestFactory.cs | 39 +++ JSMR.Worker/Program.cs | 2 +- JSMR.Worker/Services/ScanRunner.cs | 22 ++ JSMR.Worker/appsettings.Development.json | 3 +- 20 files changed, 555 insertions(+), 90 deletions(-) create mode 100644 JSMR.Infrastructure/Scanning/Extensions/DLSiteWorkExtensions.cs create mode 100644 JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs create mode 100644 JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs create mode 100644 JSMR.Tests/Unit/DLSiteWorkTestFactory.cs diff --git a/JSMR.Application/Integrations/DLSite/Models/VoiceWorkDetails.cs b/JSMR.Application/Integrations/DLSite/Models/VoiceWorkDetails.cs index 9ab9ad8..56af692 100644 --- a/JSMR.Application/Integrations/DLSite/Models/VoiceWorkDetails.cs +++ b/JSMR.Application/Integrations/DLSite/Models/VoiceWorkDetails.cs @@ -5,6 +5,7 @@ namespace JSMR.Application.Integrations.DLSite.Models; public class VoiceWorkDetails { + public string? Title { get; init; } public VoiceWorkSeries? Series { get; init; } public VoiceWorkTranslation? Translation { get; init; } public AgeRating AgeRating { get; init; } diff --git a/JSMR.Application/Scanning/Contracts/DLSiteWork.cs b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs index 3ecb9a8..a390772 100644 --- a/JSMR.Application/Scanning/Contracts/DLSiteWork.cs +++ b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs @@ -5,10 +5,10 @@ namespace JSMR.Application.Scanning.Contracts; public class DLSiteWork { //public DLSiteWorkType Type { get; set; } - public DLSiteWorkCategory Category { get; set; } + //public DLSiteWorkCategory Category { get; set; } public required string ProductName { get; set; } public required string ProductId { get; set; } - public DateOnly? AnnouncedDate { get; set; } + //public DateOnly? AnnouncedDate { get; set; } public DateOnly? ExpectedDate { get; set; } public DateOnly? SalesDate { get; set; } public int Downloads { get; set; } diff --git a/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs b/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs index 401c954..a8691f4 100644 --- a/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs +++ b/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs @@ -36,7 +36,7 @@ public sealed record VoiceWorkIngest MakerId = work.MakerId, MakerName = work.Maker, ProductId = work.ProductId, - Title = work.ProductName, + Title = details?.Title ?? work.ProductName, Description = work.Description ?? string.Empty, Tags = work.Tags, Creators = work.Creators, diff --git a/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs index c3bcb85..94875c3 100644 --- a/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs +++ b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs @@ -48,6 +48,9 @@ public sealed class ScanVoiceWorksHandler( await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken); - return new(); + return new() + { + Results = upsertResults + }; } } \ No newline at end of file diff --git a/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs b/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs index da872a5..8e62841 100644 --- a/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs +++ b/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs @@ -1,7 +1,10 @@ -namespace JSMR.Application.Scanning; +using JSMR.Application.Scanning.Ports; + +namespace JSMR.Application.Scanning; public sealed class ScanVoiceWorksResponse { public int Inserted { get; init; } public int Updated { get; init; } + public VoiceWorkUpsertResult[] Results { get; init; } = []; } \ No newline at end of file diff --git a/JSMR.Infrastructure/Data/Repositories/Circles/CircleSearchProvider.cs b/JSMR.Infrastructure/Data/Repositories/Circles/CircleSearchProvider.cs index 8ccf9e6..98962a0 100644 --- a/JSMR.Infrastructure/Data/Repositories/Circles/CircleSearchProvider.cs +++ b/JSMR.Infrastructure/Data/Repositories/Circles/CircleSearchProvider.cs @@ -29,69 +29,6 @@ public class CircleSearchProvider(AppDbContext context) : SearchProvider GetBaseQuery() - //{ - // // Project from Circles so we can use correlated subqueries per CircleId. - // var q = - // from c in context.Circles.AsNoTracking() - // select new CircleSearchItem - // { - // CircleId = c.CircleId, - // Name = c.Name, - // MakerId = c.MakerId, - // Favorite = c.Favorite, - // Blacklisted = c.Blacklisted, - // Spam = c.Spam, - - // // Aggregates - // Downloads = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .Select(v => (int?)v.Downloads) // make nullable for Sum over empty set - // .Sum() ?? 0, - - // Releases = context.VoiceWorks - // .Count(v => v.CircleId == c.CircleId && v.SalesDate != null), - - // Pending = context.VoiceWorks - // .Count(v => v.CircleId == c.CircleId && v.ExpectedDate != null), - - // FirstReleaseDate = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .Select(v => v.SalesDate) - // .Min(), - - // LatestReleaseDate = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .Select(v => v.SalesDate) - // .Max(), - - // // "Latest" by ProductId length, then value - // LatestProductId = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .OrderByDescending(v => v.ProductId.Length) - // .ThenByDescending(v => v.ProductId) - // .Select(v => v.ProductId) - // .FirstOrDefault(), - - // // If you want these two in base query too: - // LatestVoiceWorkHasImage = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .OrderByDescending(v => v.ProductId.Length) - // .ThenByDescending(v => v.ProductId) - // .Select(v => (bool?)v.HasImage) - // .FirstOrDefault(), - - // LatestVoiceWorkSalesDate = context.VoiceWorks - // .Where(v => v.CircleId == c.CircleId) - // .OrderByDescending(v => v.ProductId.Length) - // .ThenByDescending(v => v.ProductId) - // .Select(v => v.SalesDate) - // .FirstOrDefault() - // }; - - // return q; - //} - protected override IQueryable ApplyFilters(IQueryable query, CircleSearchCriteria criteria) { if (!string.IsNullOrWhiteSpace(criteria.Name)) diff --git a/JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs b/JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs index 791cdec..b01f690 100644 --- a/JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs +++ b/JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs @@ -205,7 +205,7 @@ public class VoiceWorkUpdater(AppDbContext dbContext, ITimeProvider timeProvider else { voiceWork.SalesDate = null; - voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0)); + voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0)) ?? voiceWork.ExpectedDate; voiceWork.PlannedReleaseDate = ingest.RegistrationDate > upsertContext.CurrentScanAnchor ? ingest.RegistrationDate : null; voiceWork.Status = state.IsNewUpcoming ? (byte)VoiceWorkStatus.NewAndUpcoming : (byte)VoiceWorkStatus.Upcoming; } diff --git a/JSMR.Infrastructure/Integrations/DLSite/Mapping/DLSiteToDomainMapper.cs b/JSMR.Infrastructure/Integrations/DLSite/Mapping/DLSiteToDomainMapper.cs index 09679f6..5489a07 100644 --- a/JSMR.Infrastructure/Integrations/DLSite/Mapping/DLSiteToDomainMapper.cs +++ b/JSMR.Infrastructure/Integrations/DLSite/Mapping/DLSiteToDomainMapper.cs @@ -51,6 +51,7 @@ public static class DLSiteToDomainMapper return new VoiceWorkDetails { + Title = productInfo.WorkName, Series = MapSeries(productInfo), Translation = MapTranslation(productInfo, optionsSet), WishlistCount = productInfo.WishlistCount, diff --git a/JSMR.Infrastructure/Integrations/DLSite/Models/ProductInfo.cs b/JSMR.Infrastructure/Integrations/DLSite/Models/ProductInfo.cs index c5167bb..7ec1f68 100644 --- a/JSMR.Infrastructure/Integrations/DLSite/Models/ProductInfo.cs +++ b/JSMR.Infrastructure/Integrations/DLSite/Models/ProductInfo.cs @@ -112,6 +112,9 @@ public class ProductInfo [JsonPropertyName("title_name")] public string? TitleName { get; set; } + [JsonPropertyName("title_name_masked")] + public string? TitleNameMasked { get; set; } + [JsonPropertyName("title_volumn")] public int? TitleVolumeNumber { get; set; } @@ -166,6 +169,9 @@ public class ProductInfo [JsonPropertyName("work_name")] public string? WorkName { get; set; } + [JsonPropertyName("work_name_masked")] + public string? WorkNameMasked { get; set; } + [JsonPropertyName("work_image")] public string? WorkImage { get; set; } diff --git a/JSMR.Infrastructure/Scanning/Extensions/DLSiteWorkExtensions.cs b/JSMR.Infrastructure/Scanning/Extensions/DLSiteWorkExtensions.cs new file mode 100644 index 0000000..b5a252e --- /dev/null +++ b/JSMR.Infrastructure/Scanning/Extensions/DLSiteWorkExtensions.cs @@ -0,0 +1,62 @@ +using JSMR.Application.Scanning.Contracts; + +namespace JSMR.Infrastructure.Scanning.Extensions; + +public static class DLSiteWorkExtensions +{ + public static void InferAndUpdateExpectedDates(this DLSiteWork[] works) + { + // Precompute nearest known effective date on the left and right for each index. + var left = new DateOnly?[works.Length]; + var right = new DateOnly?[works.Length]; + + DateOnly? last = null; + for (int i = 0; i < works.Length; i++) + { + var effective = GetEffectiveDate(works[i]); + if (effective.HasValue) + last = effective; + + left[i] = last; + } + + DateOnly? next = null; + for (int i = works.Length - 1; i >= 0; i--) + { + var effective = GetEffectiveDate(works[i]); + if (effective.HasValue) + next = effective; + + right[i] = next; + } + + // Fill only when BOTH sides exist and match. + for (int i = 0; i < works.Length; i++) + { + DLSiteWork work = works[i]; + + if (work.SalesDate.HasValue || work.ExpectedDate.HasValue) + continue; + + DateOnly? previous = (i > 0) ? left[i - 1] : null; + DateOnly? nxt = (i < works.Length - 1) ? right[i + 1] : null; + + if (previous.HasValue && nxt.HasValue && previous.Value == nxt.Value) + work.ExpectedDate = previous.Value; + } + } + + private static DateOnly? GetEffectiveDate(DLSiteWork work) + { + if (work.ExpectedDate.HasValue) + return work.ExpectedDate.Value; + + if (!work.SalesDate.HasValue) + return null; + + // Bucket sales day to Early/Middle/Late => 1/11/21 + var d = work.SalesDate.Value; + int day = d.Day >= 21 ? 21 : d.Day >= 11 ? 11 : 1; + return new DateOnly(d.Year, d.Month, day); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs index 42189bb..9366d07 100644 --- a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs +++ b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs @@ -5,9 +5,9 @@ using JSMR.Application.Scanning.Ports; using JSMR.Domain.Enums; using JSMR.Domain.ValueObjects; using JSMR.Infrastructure.Http; +using JSMR.Infrastructure.Scanning.Extensions; using JSMR.Infrastructure.Scanning.Models; using System.Globalization; -using System.Text.Json; using System.Text.RegularExpressions; namespace JSMR.Infrastructure.Scanning; @@ -25,7 +25,10 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca DLSiteHtmlDocument document = await GetDLSiteHtmlCollectionAsync(options, cancellationToken); DLSiteHtmlNode[] nodes = document.GetDLSiteNodes(); - return GetDLSiteWorks(nodes, options); + DLSiteWork[] works = GetDLSiteWorks(nodes, options); + works.InferAndUpdateExpectedDates(); + + return works; } private async Task GetDLSiteHtmlCollectionAsync(VoiceWorkScanOptions options, CancellationToken cancellationToken) @@ -53,7 +56,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca return filterBuilder.BuildSearchQuery(options.PageNumber, options.PageSize); } - private List GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options) + private DLSiteWork[] GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options) { var works = new List(); @@ -67,7 +70,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca works.Add(work); } - return works; + return [.. works]; } private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node) diff --git a/JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs b/JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs new file mode 100644 index 0000000..db2b45a --- /dev/null +++ b/JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs @@ -0,0 +1,62 @@ +using JSMR.Application.Scanning.Contracts; +using JSMR.Domain.Entities; +using JSMR.Domain.Enums; +using JSMR.Domain.ValueObjects; +using JSMR.Infrastructure.Data; +using JSMR.Tests.Fixtures; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace JSMR.Tests.Ingestion.Japanese; + +public class Update_Upcoming_With_No_Expected_Date_Tests(MariaDbContainerFixture container) : JapaneseIngestionTestsBase(container) +{ + [Fact] + public async Task Update_Upcoming_With_No_Expected_Date() + { + VoiceWorkIngest ingest = new() + { + MakerId = "RG00001", + MakerName = "Some Maker", + ProductId = "RJ1000001", + Title = "Some Upcoming Work", + Description = "Something is coming.", + Tags = [], + Creators = [], + WishlistCount = 250, + Downloads = 0, + HasTrial = false, + HasDLPlay = false, + StarRating = null, + Votes = null, + AgeRating = AgeRating.R15, + HasImage = true, + SupportedLanguages = [SupportedLanguage.Japanese], + SalesDate = null, + ExpectedDate = new DateOnly(2025, 1, 21), + RegistrationDate = null + }; + + await using AppDbContext dbContext = await GetAppDbContextAsync(); + DateTime currentDateTime = TokyoLocalToUtc(2025, 01, 05, 10, 0, 0); + + await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21)); + + VoiceWorkIngest updatedIngest = ingest with + { + ExpectedDate = null + }; + + // Should be exactly the same + await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21)); + } + + private static async Task UpsertAndVerify(AppDbContext dbContext, DateTime dateTime, VoiceWorkIngest ingest, DateTime? expectedDate) + { + await UpsertAsync(dbContext, dateTime, [ingest]); + + VoiceWork? voiceWork = await dbContext.VoiceWorks.SingleAsync(v => v.ProductId == ingest.ProductId, TestContext.Current.CancellationToken); + voiceWork.ShouldNotBeNull(); + voiceWork.ExpectedDate.ShouldBe(expectedDate); + } +} \ No newline at end of file diff --git a/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs b/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs index 9e042c4..8d1b360 100644 --- a/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs +++ b/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs @@ -1,20 +1,7 @@ -using JSMR.Application.Common; -using JSMR.Application.Scanning.Contracts; -using JSMR.Application.Scanning.Ports; -using JSMR.Domain.Entities; -using JSMR.Infrastructure.Common.Time; -using JSMR.Infrastructure.Data; -using JSMR.Infrastructure.Ingestion; -using JSMR.Tests.Fixtures; -using JSMR.Tests.Ingestion.Japanese; -using Microsoft.EntityFrameworkCore; -using NSubstitute; -using Shouldly; +using JSMR.Tests.Fixtures; namespace JSMR.Tests.Ingestion; - - public class VoiceWorkIngestionTests(MariaDbContainerFixture container) : IngestionTestsBase(container) { diff --git a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs index b90bf15..3378aba 100644 --- a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs +++ b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs @@ -37,6 +37,7 @@ public class DLSiteClientTests result.Count.ShouldBe(2); result.ShouldContainKey("RJ01230163"); + result["RJ01230163"].Title.ShouldBe("[Azur Lane ASMR] Commander Pampering Team! Golden Hind's Tentacular Treatment"); result["RJ01230163"].HasTrial.ShouldBeTrue(); result["RJ01230163"].HasDLPlay.ShouldBeTrue(); result["RJ01230163"].HasReviews.ShouldBeTrue(); @@ -58,6 +59,8 @@ public class DLSiteClientTests "RG0001", new ProductInfo() { + WorkName = "Product 1", + WorkNameMasked = "Product 1 (Masked)", WishlistCount = 250, DownloadCount = 100, Options = ["TRI", "DLP", "JPN"], @@ -69,6 +72,8 @@ public class DLSiteClientTests "RG0002", new ProductInfo() { + WorkName = "Product 2", + WorkNameMasked = "Product 2 (Masked)", WishlistCount = 100, DownloadCount = 50, Options = ["TRI", "ENG", "DOT", "VET"], @@ -91,6 +96,7 @@ public class DLSiteClientTests // RG0001 VoiceWorkDetails voiceWorkDetails = mappedCollection["RG0001"]; + voiceWorkDetails.Title.ShouldBe("Product 1"); voiceWorkDetails.WishlistCount.ShouldBe(250); voiceWorkDetails.DownloadCount.ShouldBe(100); voiceWorkDetails.HasTrial.ShouldBe(true); @@ -105,6 +111,7 @@ public class DLSiteClientTests // RG0002 VoiceWorkDetails secondVoiceWorkDetails = mappedCollection["RG0002"]; + secondVoiceWorkDetails.Title.ShouldBe("Product 2"); secondVoiceWorkDetails.WishlistCount.ShouldBe(100); secondVoiceWorkDetails.DownloadCount.ShouldBe(50); secondVoiceWorkDetails.HasTrial.ShouldBe(true); diff --git a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs index da63d90..4d18f45 100644 --- a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs +++ b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs @@ -1,4 +1,7 @@ -using JSMR.Application.Scanning.Contracts; +using JSMR.Application.Integrations.DLSite.Models; +using JSMR.Application.Integrations.Ports; +using JSMR.Application.Scanning.Contracts; +using JSMR.Application.Scanning.Ports; using JSMR.Infrastructure.Http; using JSMR.Infrastructure.Scanning; using JSMR.Tests.Utilities; @@ -101,6 +104,51 @@ public class VoiceWorkScannerTests result[1].Tags.ShouldBe(["ASMR", "バイノーラル/ダミヘ", "色仕掛け", "浮気", "百合", "レズ/女同士", "ツルペタ", "貧乳/微乳"]); } + // (Scan) + (Integration) = ==> Ingestion + [Fact] + public async Task Scan_And_Integration_Ingest_Mapping_Test() + { + IVoiceWorksScanner scanner = Substitute.For(); + + IReadOnlyList scannedWorks = + [ + new() + { + ProductId = "RJ1", + ProductName = "Masked Product Title", + MakerId = "RG1", + Maker = "Some Maker", + ImageUrl = "https://site.com/image_main.png", + SmallImageUrl = "https://site.com/image_240x240.png" + } + ]; + + scanner.ScanPageAsync(Arg.Any(), CancellationToken.None) + .Returns(Task.FromResult(scannedWorks)); + + IDLSiteClient dlsiteClient = Substitute.For(); + + VoiceWorkDetailCollection detailCollection = new() + { + { + "RJ1", + new VoiceWorkDetails() + { + Title = "Product Title" + } + } + }; + + dlsiteClient.GetVoiceWorkDetailsAsync(Arg.Any(), CancellationToken.None) + .Returns(Task.FromResult(detailCollection)); + + VoiceWorkIngest ingest = VoiceWorkIngest.From(scannedWorks[0], detailCollection["RJ1"]); + + // TODO: Test other fields + ingest.Title.ShouldBe("Product Title"); + ingest.HasImage.ShouldBe(true); + } + [Fact] public async Task Scan_With_English_Locale() { diff --git a/JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs b/JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs new file mode 100644 index 0000000..beabfbe --- /dev/null +++ b/JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs @@ -0,0 +1,283 @@ +using JSMR.Application.Scanning.Contracts; +using JSMR.Infrastructure.Scanning.Extensions; +using Shouldly; + +namespace JSMR.Tests.Unit; + +public class DLSiteWorkExpectedDateInferenceTests +{ + [Fact] + public void Infers_From_Sandwich_Expected_Dates() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work(), + Work(expected: new DateOnly(2026, 5, 1)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1) + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void Infers_From_Sandwich_Sales_Dates_Early() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 2)), + Work(), + Work(sales: new DateOnly(2026, 5, 7)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + new DateOnly(2026, 5, 1), + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void Infers_From_Sandwich_Sales_Dates_Middle() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 12)), + Work(), + Work(sales: new DateOnly(2026, 5, 13)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + new DateOnly(2026, 5, 11), + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void Infers_From_Sandwich_Sales_Dates_Late() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 25)), + Work(), + Work(sales: new DateOnly(2026, 5, 27)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + new DateOnly(2026, 5, 21), + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void Infers_From_Sandwich_Mixed_Dates() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work(), + Work(sales: new DateOnly(2026, 5, 7)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void Infers_A_Run_Of_Missing_Items_When_Bounded_By_Same_Date() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work(), + Work(), + Work(), + Work(expected: new DateOnly(2026, 5, 1)), + ]; + + DateOnly?[] expected = + [ + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + new DateOnly(2026, 5, 1), + ]; + + AttemptInferAndVerify(works, expected); + } + + [Fact] + public void DoesNotInfer_When_Expected_Sandwich_Difference() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work(), + Work(expected: new DateOnly(2026, 5, 11)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + new DateOnly(2026, 5, 1), + null, + new DateOnly(2026, 5, 11) + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Middle() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 1)), + Work(), + Work(sales: new DateOnly(2026, 5, 12)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + null, + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Late() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 1)), + Work(), + Work(sales: new DateOnly(2026, 5, 22)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + null, + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNotInfer_When_Sales_Sandwich_Difference_Middle_Late() + { + DLSiteWork[] works = + [ + Work(sales: new DateOnly(2026, 5, 14)), + Work(), + Work(sales: new DateOnly(2026, 5, 22)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, // Sales date will not have expected date + null, + null // Sales date will not have expected date + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNotInfer_When_No_Left() + { + DLSiteWork[] works = + [ + Work(), + Work(expected: new DateOnly(2026, 5, 1)), + ]; + + DateOnly?[] expectedExpectedDates = + [ + null, + new DateOnly(2026, 5, 1) + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNotInfer_When_No_Right() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work() + ]; + + DateOnly?[] expectedExpectedDates = + [ + new DateOnly(2026, 5, 1), + null + ]; + + AttemptInferAndVerify(works, expectedExpectedDates); + } + + [Fact] + public void DoesNot_Overwrite_Existing_ExpectedDate() + { + DLSiteWork[] works = + [ + Work(expected: new DateOnly(2026, 5, 1)), + Work(expected: new DateOnly(2026, 4, 11)), // already set + Work(expected: new DateOnly(2026, 5, 21)), + ]; + + DateOnly?[] expected = + [ + new DateOnly(2026, 5, 1), + new DateOnly(2026, 4, 11), + new DateOnly(2026, 5, 21), + ]; + + AttemptInferAndVerify(works, expected); + } + + private static void AttemptInferAndVerify(DLSiteWork[] works, DateOnly?[] expectedExpectedDates) + { + // Act + works.InferAndUpdateExpectedDates(); + + // Assert + works.Length.ShouldBe(expectedExpectedDates.Length); + + for (int i = 0; i < works.Length; i++) + works[i].ExpectedDate.ShouldBe(expectedExpectedDates[i]); + } + + private static DLSiteWork Work(DateOnly? expected = null, DateOnly? sales = null) + => DLSiteWorkTestFactory.Create(expectedDate: expected, salesDate: sales); +} \ No newline at end of file diff --git a/JSMR.Tests/Unit/DLSiteWorkTestFactory.cs b/JSMR.Tests/Unit/DLSiteWorkTestFactory.cs new file mode 100644 index 0000000..38930df --- /dev/null +++ b/JSMR.Tests/Unit/DLSiteWorkTestFactory.cs @@ -0,0 +1,39 @@ +using JSMR.Application.Scanning.Contracts; +using JSMR.Domain.Enums; + +namespace JSMR.Tests.Unit; + +internal static class DLSiteWorkTestFactory +{ + private static int _counter = 0; + + public static DLSiteWork Create(DateOnly? expectedDate = null, DateOnly? salesDate = null) + { + int id = Interlocked.Increment(ref _counter); + + return new DLSiteWork + { + ProductName = $"Test Product {id}", + ProductId = $"RJ_TEST_{id:00000000}", + Maker = "Test Maker", + MakerId = "RG_TEST", + ImageUrl = "//img.dlsite.jp/test_main.jpg", + SmallImageUrl = "//img.dlsite.jp/test_sam.jpg", + AgeRating = AgeRating.AllAges, + + // Relevant fields for these tests: + ExpectedDate = expectedDate, + SalesDate = salesDate, + + // The rest can be safe defaults: + Downloads = 0, + StarRating = null, + Votes = null, + Description = null, + Genres = [], + Tags = [], + Creators = [], + HasTrial = false + }; + } +} \ No newline at end of file diff --git a/JSMR.Worker/Program.cs b/JSMR.Worker/Program.cs index b18e2fb..dd04d2f 100644 --- a/JSMR.Worker/Program.cs +++ b/JSMR.Worker/Program.cs @@ -130,7 +130,7 @@ schemaDumpCommand.SetAction(async (parseResult, cancellationToken) => var sql = db.Database.GenerateCreateScript(); var outPath = Path.GetFullPath("desired.sql"); - await File.WriteAllTextAsync(outPath, sql); + await File.WriteAllTextAsync(outPath, sql, cancellationToken); Console.WriteLine($"[OK] Wrote EF model create script to: {outPath}"); }); diff --git a/JSMR.Worker/Services/ScanRunner.cs b/JSMR.Worker/Services/ScanRunner.cs index 84e846c..4ae312e 100644 --- a/JSMR.Worker/Services/ScanRunner.cs +++ b/JSMR.Worker/Services/ScanRunner.cs @@ -1,5 +1,6 @@ using JSMR.Application.Enums; using JSMR.Application.Scanning; +using JSMR.Application.Scanning.Ports; using JSMR.Infrastructure.Common.Time; using JSMR.Worker.Options; using Microsoft.Extensions.DependencyInjection; @@ -48,6 +49,27 @@ public sealed class PagedScanRunner( ScanVoiceWorksResponse response = await handler.HandleAsync(request, cancellationToken); + //int newUpcoming = response.Results.Where(x => x.IsNewUpcoming == true).Count(); + + //if (newUpcoming > 0) + // updatedInfo.Add($"{newUpcoming} new upcoming work(s)"); + + //int newOnSale = result.ScannedVoiceWorks.Where(x => x.IsNewOnSale == true).Count(); + + //if (newOnSale > 0) + // updatedInfo.Add($"{newOnSale} new work(s) on sale"); + + IEnumerable resultsWithIssues = response.Results.Where(x => x.Issues.Count > 0); + + //foreach (VoiceWorkUpsertResult resultWithIssues in resultsWithIssues) + //{ + // log.LogWarning($"PRoblem with {resultWithIssues.}") + // string messageToDisplay = $"{scannedVoiceWork.ProductId} - {scannedVoiceWork.ProductName} -- {message}"; + + // Console.WriteLine(messageToDisplay); + // messages.Add(messageToDisplay); + //} + // Save checkpoint await checkpoints.SaveLastPageAsync(options.Locale, currentPage, cancellationToken); } diff --git a/JSMR.Worker/appsettings.Development.json b/JSMR.Worker/appsettings.Development.json index faa1e74..18bab23 100644 --- a/JSMR.Worker/appsettings.Development.json +++ b/JSMR.Worker/appsettings.Development.json @@ -1,7 +1,8 @@ { "Logging": { "LogLevel": { - "Default": "Information", + /*"Default": "Information",*/ + "Default": "Warning", "Microsoft.AspNetCore": "Warning" } },