Updated various parts of scanning and ingestion, either for bug fixes, or for enhancements.

2026-03-01 22:07:20 -05:00
parent 704a6fc433
commit 83655f13e9
20 changed files with 555 additions and 90 deletions
--- a/JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs
+++ b/JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs
@@ -0,0 +1,62 @@
+using JSMR.Application.Scanning.Contracts;
+using JSMR.Domain.Entities;
+using JSMR.Domain.Enums;
+using JSMR.Domain.ValueObjects;
+using JSMR.Infrastructure.Data;
+using JSMR.Tests.Fixtures;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+
+namespace JSMR.Tests.Ingestion.Japanese;
+
+public class Update_Upcoming_With_No_Expected_Date_Tests(MariaDbContainerFixture container) : JapaneseIngestionTestsBase(container)
+{
+    [Fact]
+    public async Task Update_Upcoming_With_No_Expected_Date()
+    {
+        VoiceWorkIngest ingest = new()
+        {
+            MakerId = "RG00001",
+            MakerName = "Some Maker",
+            ProductId = "RJ1000001",
+            Title = "Some Upcoming Work",
+            Description = "Something is coming.",
+            Tags = [],
+            Creators = [],
+            WishlistCount = 250,
+            Downloads = 0,
+            HasTrial = false,
+            HasDLPlay = false,
+            StarRating = null,
+            Votes = null,
+            AgeRating = AgeRating.R15,
+            HasImage = true,
+            SupportedLanguages = [SupportedLanguage.Japanese],
+            SalesDate = null,
+            ExpectedDate = new DateOnly(2025, 1, 21),
+            RegistrationDate = null
+        };
+
+        await using AppDbContext dbContext = await GetAppDbContextAsync();
+        DateTime currentDateTime = TokyoLocalToUtc(2025, 01, 05, 10, 0, 0);
+
+        await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21));
+
+        VoiceWorkIngest updatedIngest = ingest with
+        {
+            ExpectedDate = null
+        };
+
+        // Should be exactly the same
+        await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21));
+    }
+
+    private static async Task UpsertAndVerify(AppDbContext dbContext, DateTime dateTime, VoiceWorkIngest ingest, DateTime? expectedDate)
+    {
+        await UpsertAsync(dbContext, dateTime, [ingest]);
+
+        VoiceWork? voiceWork = await dbContext.VoiceWorks.SingleAsync(v => v.ProductId == ingest.ProductId, TestContext.Current.CancellationToken);
+        voiceWork.ShouldNotBeNull();
+        voiceWork.ExpectedDate.ShouldBe(expectedDate);
+    }
+}
--- a/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs
+++ b/JSMR.Tests/Ingestion/VoiceWorkIngestionTests.cs
@@ -1,20 +1,7 @@
-using JSMR.Application.Common;
-using JSMR.Application.Scanning.Contracts;
-using JSMR.Application.Scanning.Ports;
-using JSMR.Domain.Entities;
-using JSMR.Infrastructure.Common.Time;
-using JSMR.Infrastructure.Data;
-using JSMR.Infrastructure.Ingestion;
-using JSMR.Tests.Fixtures;
-using JSMR.Tests.Ingestion.Japanese;
-using Microsoft.EntityFrameworkCore;
-using NSubstitute;
-using Shouldly;
+using JSMR.Tests.Fixtures;

 namespace JSMR.Tests.Ingestion;

-
-
 public class VoiceWorkIngestionTests(MariaDbContainerFixture container) : IngestionTestsBase(container)
 {

--- a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs
+++ b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs
@@ -37,6 +37,7 @@ public class DLSiteClientTests
        result.Count.ShouldBe(2);

        result.ShouldContainKey("RJ01230163");
+        result["RJ01230163"].Title.ShouldBe("[Azur Lane ASMR] Commander Pampering Team! Golden Hind's Tentacular Treatment");
        result["RJ01230163"].HasTrial.ShouldBeTrue();
        result["RJ01230163"].HasDLPlay.ShouldBeTrue();
        result["RJ01230163"].HasReviews.ShouldBeTrue();
@@ -58,6 +59,8 @@ public class DLSiteClientTests
                "RG0001",
                new ProductInfo()
                {
+                    WorkName = "Product 1",
+                    WorkNameMasked = "Product 1 (Masked)",
                    WishlistCount = 250,
                    DownloadCount = 100,
                    Options = ["TRI", "DLP", "JPN"],
@@ -69,6 +72,8 @@ public class DLSiteClientTests
                "RG0002",
                new ProductInfo()
                {
+                    WorkName = "Product 2",
+                    WorkNameMasked = "Product 2 (Masked)",
                    WishlistCount = 100,
                    DownloadCount = 50,
                    Options = ["TRI", "ENG", "DOT", "VET"],
@@ -91,6 +96,7 @@ public class DLSiteClientTests

        // RG0001
        VoiceWorkDetails voiceWorkDetails = mappedCollection["RG0001"];
+        voiceWorkDetails.Title.ShouldBe("Product 1");
        voiceWorkDetails.WishlistCount.ShouldBe(250);
        voiceWorkDetails.DownloadCount.ShouldBe(100);
        voiceWorkDetails.HasTrial.ShouldBe(true);
@@ -105,6 +111,7 @@ public class DLSiteClientTests

        // RG0002
        VoiceWorkDetails secondVoiceWorkDetails = mappedCollection["RG0002"];
+        secondVoiceWorkDetails.Title.ShouldBe("Product 2");
        secondVoiceWorkDetails.WishlistCount.ShouldBe(100);
        secondVoiceWorkDetails.DownloadCount.ShouldBe(50);
        secondVoiceWorkDetails.HasTrial.ShouldBe(true);
--- a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs
+++ b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs
@@ -1,4 +1,7 @@
-using JSMR.Application.Scanning.Contracts;
+using JSMR.Application.Integrations.DLSite.Models;
+using JSMR.Application.Integrations.Ports;
+using JSMR.Application.Scanning.Contracts;
+using JSMR.Application.Scanning.Ports;
 using JSMR.Infrastructure.Http;
 using JSMR.Infrastructure.Scanning;
 using JSMR.Tests.Utilities;
@@ -101,6 +104,51 @@ public class VoiceWorkScannerTests
        result[1].Tags.ShouldBe(["ASMR", "バイノーラル/ダミヘ", "色仕掛け", "浮気", "百合", "レズ/女同士", "ツルペタ", "貧乳/微乳"]);
    }

+    // (Scan) + (Integration) = <This> ==> Ingestion
+    [Fact]
+    public async Task Scan_And_Integration_Ingest_Mapping_Test()
+    {
+        IVoiceWorksScanner scanner = Substitute.For<IVoiceWorksScanner>();
+
+        IReadOnlyList<DLSiteWork> scannedWorks =
+        [
+            new()
+            {
+                ProductId = "RJ1",
+                ProductName = "Masked Product Title",
+                MakerId = "RG1",
+                Maker = "Some Maker",
+                ImageUrl = "https://site.com/image_main.png",
+                SmallImageUrl = "https://site.com/image_240x240.png"
+            }
+        ];
+
+        scanner.ScanPageAsync(Arg.Any<VoiceWorkScanOptions>(), CancellationToken.None)
+            .Returns(Task.FromResult(scannedWorks));
+
+        IDLSiteClient dlsiteClient = Substitute.For<IDLSiteClient>();
+
+        VoiceWorkDetailCollection detailCollection = new()
+        {
+            {
+                "RJ1",
+                new VoiceWorkDetails()
+                {
+                    Title = "Product Title"
+                }
+            }
+        };
+
+        dlsiteClient.GetVoiceWorkDetailsAsync(Arg.Any<string[]>(), CancellationToken.None)
+            .Returns(Task.FromResult(detailCollection));
+
+        VoiceWorkIngest ingest = VoiceWorkIngest.From(scannedWorks[0], detailCollection["RJ1"]);
+
+        // TODO: Test other fields
+        ingest.Title.ShouldBe("Product Title");
+        ingest.HasImage.ShouldBe(true);
+    }
+
    [Fact]
    public async Task Scan_With_English_Locale()
    {
--- a/JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs
+++ b/JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs
@@ -0,0 +1,283 @@
+using JSMR.Application.Scanning.Contracts;
+using JSMR.Infrastructure.Scanning.Extensions;
+using Shouldly;
+
+namespace JSMR.Tests.Unit;
+
+public class DLSiteWorkExpectedDateInferenceTests
+{
+    [Fact]
+    public void Infers_From_Sandwich_Expected_Dates()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(expected: new DateOnly(2026, 5, 1)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1)
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void Infers_From_Sandwich_Sales_Dates_Early()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 2)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 7)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            new DateOnly(2026, 5, 1),
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void Infers_From_Sandwich_Sales_Dates_Middle()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 12)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 13)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            new DateOnly(2026, 5, 11),
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void Infers_From_Sandwich_Sales_Dates_Late()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 25)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 27)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            new DateOnly(2026, 5, 21),
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void Infers_From_Sandwich_Mixed_Dates()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 7)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void Infers_A_Run_Of_Missing_Items_When_Bounded_By_Same_Date()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(),
+            Work(),
+            Work(expected: new DateOnly(2026, 5, 1)),
+        ];
+
+        DateOnly?[] expected =
+        [
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 5, 1),
+        ];
+
+        AttemptInferAndVerify(works, expected);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_Expected_Sandwich_Difference()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(expected: new DateOnly(2026, 5, 11)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            new DateOnly(2026, 5, 1),
+            null,
+            new DateOnly(2026, 5, 11)
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Middle()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 12)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            null,
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Late()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 1)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 22)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            null,
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_Sales_Sandwich_Difference_Middle_Late()
+    {
+        DLSiteWork[] works =
+        [
+            Work(sales: new DateOnly(2026, 5, 14)),
+            Work(),
+            Work(sales: new DateOnly(2026, 5, 22)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null, // Sales date will not have expected date
+            null,
+            null // Sales date will not have expected date
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_No_Left()
+    {
+        DLSiteWork[] works =
+        [
+            Work(),
+            Work(expected: new DateOnly(2026, 5, 1)),
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            null,
+            new DateOnly(2026, 5, 1)
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNotInfer_When_No_Right()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work()
+        ];
+
+        DateOnly?[] expectedExpectedDates =
+        [
+            new DateOnly(2026, 5, 1),
+            null
+        ];
+
+        AttemptInferAndVerify(works, expectedExpectedDates);
+    }
+
+    [Fact]
+    public void DoesNot_Overwrite_Existing_ExpectedDate()
+    {
+        DLSiteWork[] works =
+        [
+            Work(expected: new DateOnly(2026, 5, 1)),
+            Work(expected: new DateOnly(2026, 4, 11)), // already set
+            Work(expected: new DateOnly(2026, 5, 21)),
+        ];
+
+        DateOnly?[] expected =
+        [
+            new DateOnly(2026, 5, 1),
+            new DateOnly(2026, 4, 11),
+            new DateOnly(2026, 5, 21),
+        ];
+
+        AttemptInferAndVerify(works, expected);
+    }
+
+    private static void AttemptInferAndVerify(DLSiteWork[] works, DateOnly?[] expectedExpectedDates)
+    {
+        // Act
+        works.InferAndUpdateExpectedDates();
+
+        // Assert
+        works.Length.ShouldBe(expectedExpectedDates.Length);
+
+        for (int i = 0; i < works.Length; i++)
+            works[i].ExpectedDate.ShouldBe(expectedExpectedDates[i]);
+    }
+
+    private static DLSiteWork Work(DateOnly? expected = null, DateOnly? sales = null)
+        => DLSiteWorkTestFactory.Create(expectedDate: expected, salesDate: sales);
+}
--- a/JSMR.Tests/Unit/DLSiteWorkTestFactory.cs
+++ b/JSMR.Tests/Unit/DLSiteWorkTestFactory.cs
@@ -0,0 +1,39 @@
+using JSMR.Application.Scanning.Contracts;
+using JSMR.Domain.Enums;
+
+namespace JSMR.Tests.Unit;
+
+internal static class DLSiteWorkTestFactory
+{
+    private static int _counter = 0;
+
+    public static DLSiteWork Create(DateOnly? expectedDate = null, DateOnly? salesDate = null)
+    {
+        int id = Interlocked.Increment(ref _counter);
+
+        return new DLSiteWork
+        {
+            ProductName = $"Test Product {id}",
+            ProductId = $"RJ_TEST_{id:00000000}",
+            Maker = "Test Maker",
+            MakerId = "RG_TEST",
+            ImageUrl = "//img.dlsite.jp/test_main.jpg",
+            SmallImageUrl = "//img.dlsite.jp/test_sam.jpg",
+            AgeRating = AgeRating.AllAges,
+
+            // Relevant fields for these tests:
+            ExpectedDate = expectedDate,
+            SalesDate = salesDate,
+
+            // The rest can be safe defaults:
+            Downloads = 0,
+            StarRating = null,
+            Votes = null,
+            Description = null,
+            Genres = [],
+            Tags = [],
+            Creators = [],
+            HasTrial = false
+        };
+    }
+}