Updated various parts of scanning and ingestion, either for bug fixes, or for enhancements.
This commit is contained in:
@@ -5,6 +5,7 @@ namespace JSMR.Application.Integrations.DLSite.Models;
|
||||
|
||||
public class VoiceWorkDetails
|
||||
{
|
||||
public string? Title { get; init; }
|
||||
public VoiceWorkSeries? Series { get; init; }
|
||||
public VoiceWorkTranslation? Translation { get; init; }
|
||||
public AgeRating AgeRating { get; init; }
|
||||
|
||||
@@ -5,10 +5,10 @@ namespace JSMR.Application.Scanning.Contracts;
|
||||
public class DLSiteWork
|
||||
{
|
||||
//public DLSiteWorkType Type { get; set; }
|
||||
public DLSiteWorkCategory Category { get; set; }
|
||||
//public DLSiteWorkCategory Category { get; set; }
|
||||
public required string ProductName { get; set; }
|
||||
public required string ProductId { get; set; }
|
||||
public DateOnly? AnnouncedDate { get; set; }
|
||||
//public DateOnly? AnnouncedDate { get; set; }
|
||||
public DateOnly? ExpectedDate { get; set; }
|
||||
public DateOnly? SalesDate { get; set; }
|
||||
public int Downloads { get; set; }
|
||||
|
||||
@@ -36,7 +36,7 @@ public sealed record VoiceWorkIngest
|
||||
MakerId = work.MakerId,
|
||||
MakerName = work.Maker,
|
||||
ProductId = work.ProductId,
|
||||
Title = work.ProductName,
|
||||
Title = details?.Title ?? work.ProductName,
|
||||
Description = work.Description ?? string.Empty,
|
||||
Tags = work.Tags,
|
||||
Creators = work.Creators,
|
||||
|
||||
@@ -48,6 +48,9 @@ public sealed class ScanVoiceWorksHandler(
|
||||
|
||||
await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
|
||||
|
||||
return new();
|
||||
return new()
|
||||
{
|
||||
Results = upsertResults
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
namespace JSMR.Application.Scanning;
|
||||
using JSMR.Application.Scanning.Ports;
|
||||
|
||||
namespace JSMR.Application.Scanning;
|
||||
|
||||
public sealed class ScanVoiceWorksResponse
|
||||
{
|
||||
public int Inserted { get; init; }
|
||||
public int Updated { get; init; }
|
||||
public VoiceWorkUpsertResult[] Results { get; init; } = [];
|
||||
}
|
||||
@@ -29,69 +29,6 @@ public class CircleSearchProvider(AppDbContext context) : SearchProvider<CircleS
|
||||
return q;
|
||||
}
|
||||
|
||||
//protected override IQueryable<CircleSearchItem> GetBaseQuery()
|
||||
//{
|
||||
// // Project from Circles so we can use correlated subqueries per CircleId.
|
||||
// var q =
|
||||
// from c in context.Circles.AsNoTracking()
|
||||
// select new CircleSearchItem
|
||||
// {
|
||||
// CircleId = c.CircleId,
|
||||
// Name = c.Name,
|
||||
// MakerId = c.MakerId,
|
||||
// Favorite = c.Favorite,
|
||||
// Blacklisted = c.Blacklisted,
|
||||
// Spam = c.Spam,
|
||||
|
||||
// // Aggregates
|
||||
// Downloads = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .Select(v => (int?)v.Downloads) // make nullable for Sum over empty set
|
||||
// .Sum() ?? 0,
|
||||
|
||||
// Releases = context.VoiceWorks
|
||||
// .Count(v => v.CircleId == c.CircleId && v.SalesDate != null),
|
||||
|
||||
// Pending = context.VoiceWorks
|
||||
// .Count(v => v.CircleId == c.CircleId && v.ExpectedDate != null),
|
||||
|
||||
// FirstReleaseDate = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .Select(v => v.SalesDate)
|
||||
// .Min(),
|
||||
|
||||
// LatestReleaseDate = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .Select(v => v.SalesDate)
|
||||
// .Max(),
|
||||
|
||||
// // "Latest" by ProductId length, then value
|
||||
// LatestProductId = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .OrderByDescending(v => v.ProductId.Length)
|
||||
// .ThenByDescending(v => v.ProductId)
|
||||
// .Select(v => v.ProductId)
|
||||
// .FirstOrDefault(),
|
||||
|
||||
// // If you want these two in base query too:
|
||||
// LatestVoiceWorkHasImage = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .OrderByDescending(v => v.ProductId.Length)
|
||||
// .ThenByDescending(v => v.ProductId)
|
||||
// .Select(v => (bool?)v.HasImage)
|
||||
// .FirstOrDefault(),
|
||||
|
||||
// LatestVoiceWorkSalesDate = context.VoiceWorks
|
||||
// .Where(v => v.CircleId == c.CircleId)
|
||||
// .OrderByDescending(v => v.ProductId.Length)
|
||||
// .ThenByDescending(v => v.ProductId)
|
||||
// .Select(v => v.SalesDate)
|
||||
// .FirstOrDefault()
|
||||
// };
|
||||
|
||||
// return q;
|
||||
//}
|
||||
|
||||
protected override IQueryable<CircleQuery> ApplyFilters(IQueryable<CircleQuery> query, CircleSearchCriteria criteria)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(criteria.Name))
|
||||
|
||||
@@ -205,7 +205,7 @@ public class VoiceWorkUpdater(AppDbContext dbContext, ITimeProvider timeProvider
|
||||
else
|
||||
{
|
||||
voiceWork.SalesDate = null;
|
||||
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0));
|
||||
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0)) ?? voiceWork.ExpectedDate;
|
||||
voiceWork.PlannedReleaseDate = ingest.RegistrationDate > upsertContext.CurrentScanAnchor ? ingest.RegistrationDate : null;
|
||||
voiceWork.Status = state.IsNewUpcoming ? (byte)VoiceWorkStatus.NewAndUpcoming : (byte)VoiceWorkStatus.Upcoming;
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@ public static class DLSiteToDomainMapper
|
||||
|
||||
return new VoiceWorkDetails
|
||||
{
|
||||
Title = productInfo.WorkName,
|
||||
Series = MapSeries(productInfo),
|
||||
Translation = MapTranslation(productInfo, optionsSet),
|
||||
WishlistCount = productInfo.WishlistCount,
|
||||
|
||||
@@ -112,6 +112,9 @@ public class ProductInfo
|
||||
[JsonPropertyName("title_name")]
|
||||
public string? TitleName { get; set; }
|
||||
|
||||
[JsonPropertyName("title_name_masked")]
|
||||
public string? TitleNameMasked { get; set; }
|
||||
|
||||
[JsonPropertyName("title_volumn")]
|
||||
public int? TitleVolumeNumber { get; set; }
|
||||
|
||||
@@ -166,6 +169,9 @@ public class ProductInfo
|
||||
[JsonPropertyName("work_name")]
|
||||
public string? WorkName { get; set; }
|
||||
|
||||
[JsonPropertyName("work_name_masked")]
|
||||
public string? WorkNameMasked { get; set; }
|
||||
|
||||
[JsonPropertyName("work_image")]
|
||||
public string? WorkImage { get; set; }
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
|
||||
namespace JSMR.Infrastructure.Scanning.Extensions;
|
||||
|
||||
public static class DLSiteWorkExtensions
|
||||
{
|
||||
public static void InferAndUpdateExpectedDates(this DLSiteWork[] works)
|
||||
{
|
||||
// Precompute nearest known effective date on the left and right for each index.
|
||||
var left = new DateOnly?[works.Length];
|
||||
var right = new DateOnly?[works.Length];
|
||||
|
||||
DateOnly? last = null;
|
||||
for (int i = 0; i < works.Length; i++)
|
||||
{
|
||||
var effective = GetEffectiveDate(works[i]);
|
||||
if (effective.HasValue)
|
||||
last = effective;
|
||||
|
||||
left[i] = last;
|
||||
}
|
||||
|
||||
DateOnly? next = null;
|
||||
for (int i = works.Length - 1; i >= 0; i--)
|
||||
{
|
||||
var effective = GetEffectiveDate(works[i]);
|
||||
if (effective.HasValue)
|
||||
next = effective;
|
||||
|
||||
right[i] = next;
|
||||
}
|
||||
|
||||
// Fill only when BOTH sides exist and match.
|
||||
for (int i = 0; i < works.Length; i++)
|
||||
{
|
||||
DLSiteWork work = works[i];
|
||||
|
||||
if (work.SalesDate.HasValue || work.ExpectedDate.HasValue)
|
||||
continue;
|
||||
|
||||
DateOnly? previous = (i > 0) ? left[i - 1] : null;
|
||||
DateOnly? nxt = (i < works.Length - 1) ? right[i + 1] : null;
|
||||
|
||||
if (previous.HasValue && nxt.HasValue && previous.Value == nxt.Value)
|
||||
work.ExpectedDate = previous.Value;
|
||||
}
|
||||
}
|
||||
|
||||
private static DateOnly? GetEffectiveDate(DLSiteWork work)
|
||||
{
|
||||
if (work.ExpectedDate.HasValue)
|
||||
return work.ExpectedDate.Value;
|
||||
|
||||
if (!work.SalesDate.HasValue)
|
||||
return null;
|
||||
|
||||
// Bucket sales day to Early/Middle/Late => 1/11/21
|
||||
var d = work.SalesDate.Value;
|
||||
int day = d.Day >= 21 ? 21 : d.Day >= 11 ? 11 : 1;
|
||||
return new DateOnly(d.Year, d.Month, day);
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,9 @@ using JSMR.Application.Scanning.Ports;
|
||||
using JSMR.Domain.Enums;
|
||||
using JSMR.Domain.ValueObjects;
|
||||
using JSMR.Infrastructure.Http;
|
||||
using JSMR.Infrastructure.Scanning.Extensions;
|
||||
using JSMR.Infrastructure.Scanning.Models;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace JSMR.Infrastructure.Scanning;
|
||||
@@ -25,7 +25,10 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
|
||||
DLSiteHtmlDocument document = await GetDLSiteHtmlCollectionAsync(options, cancellationToken);
|
||||
DLSiteHtmlNode[] nodes = document.GetDLSiteNodes();
|
||||
|
||||
return GetDLSiteWorks(nodes, options);
|
||||
DLSiteWork[] works = GetDLSiteWorks(nodes, options);
|
||||
works.InferAndUpdateExpectedDates();
|
||||
|
||||
return works;
|
||||
}
|
||||
|
||||
private async Task<DLSiteHtmlDocument> GetDLSiteHtmlCollectionAsync(VoiceWorkScanOptions options, CancellationToken cancellationToken)
|
||||
@@ -53,7 +56,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
|
||||
return filterBuilder.BuildSearchQuery(options.PageNumber, options.PageSize);
|
||||
}
|
||||
|
||||
private List<DLSiteWork> GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options)
|
||||
private DLSiteWork[] GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options)
|
||||
{
|
||||
var works = new List<DLSiteWork>();
|
||||
|
||||
@@ -67,7 +70,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
|
||||
works.Add(work);
|
||||
}
|
||||
|
||||
return works;
|
||||
return [.. works];
|
||||
}
|
||||
|
||||
private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node)
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Domain.Entities;
|
||||
using JSMR.Domain.Enums;
|
||||
using JSMR.Domain.ValueObjects;
|
||||
using JSMR.Infrastructure.Data;
|
||||
using JSMR.Tests.Fixtures;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
|
||||
namespace JSMR.Tests.Ingestion.Japanese;
|
||||
|
||||
public class Update_Upcoming_With_No_Expected_Date_Tests(MariaDbContainerFixture container) : JapaneseIngestionTestsBase(container)
|
||||
{
|
||||
[Fact]
|
||||
public async Task Update_Upcoming_With_No_Expected_Date()
|
||||
{
|
||||
VoiceWorkIngest ingest = new()
|
||||
{
|
||||
MakerId = "RG00001",
|
||||
MakerName = "Some Maker",
|
||||
ProductId = "RJ1000001",
|
||||
Title = "Some Upcoming Work",
|
||||
Description = "Something is coming.",
|
||||
Tags = [],
|
||||
Creators = [],
|
||||
WishlistCount = 250,
|
||||
Downloads = 0,
|
||||
HasTrial = false,
|
||||
HasDLPlay = false,
|
||||
StarRating = null,
|
||||
Votes = null,
|
||||
AgeRating = AgeRating.R15,
|
||||
HasImage = true,
|
||||
SupportedLanguages = [SupportedLanguage.Japanese],
|
||||
SalesDate = null,
|
||||
ExpectedDate = new DateOnly(2025, 1, 21),
|
||||
RegistrationDate = null
|
||||
};
|
||||
|
||||
await using AppDbContext dbContext = await GetAppDbContextAsync();
|
||||
DateTime currentDateTime = TokyoLocalToUtc(2025, 01, 05, 10, 0, 0);
|
||||
|
||||
await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21));
|
||||
|
||||
VoiceWorkIngest updatedIngest = ingest with
|
||||
{
|
||||
ExpectedDate = null
|
||||
};
|
||||
|
||||
// Should be exactly the same
|
||||
await UpsertAndVerify(dbContext, TokyoLocalToUtc(2025, 01, 05, 10, 0, 0), ingest, new DateTime(2025, 1, 21));
|
||||
}
|
||||
|
||||
private static async Task UpsertAndVerify(AppDbContext dbContext, DateTime dateTime, VoiceWorkIngest ingest, DateTime? expectedDate)
|
||||
{
|
||||
await UpsertAsync(dbContext, dateTime, [ingest]);
|
||||
|
||||
VoiceWork? voiceWork = await dbContext.VoiceWorks.SingleAsync(v => v.ProductId == ingest.ProductId, TestContext.Current.CancellationToken);
|
||||
voiceWork.ShouldNotBeNull();
|
||||
voiceWork.ExpectedDate.ShouldBe(expectedDate);
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,7 @@
|
||||
using JSMR.Application.Common;
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Application.Scanning.Ports;
|
||||
using JSMR.Domain.Entities;
|
||||
using JSMR.Infrastructure.Common.Time;
|
||||
using JSMR.Infrastructure.Data;
|
||||
using JSMR.Infrastructure.Ingestion;
|
||||
using JSMR.Tests.Fixtures;
|
||||
using JSMR.Tests.Ingestion.Japanese;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using JSMR.Tests.Fixtures;
|
||||
|
||||
namespace JSMR.Tests.Ingestion;
|
||||
|
||||
|
||||
|
||||
public class VoiceWorkIngestionTests(MariaDbContainerFixture container) : IngestionTestsBase(container)
|
||||
{
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ public class DLSiteClientTests
|
||||
result.Count.ShouldBe(2);
|
||||
|
||||
result.ShouldContainKey("RJ01230163");
|
||||
result["RJ01230163"].Title.ShouldBe("[Azur Lane ASMR] Commander Pampering Team! Golden Hind's Tentacular Treatment");
|
||||
result["RJ01230163"].HasTrial.ShouldBeTrue();
|
||||
result["RJ01230163"].HasDLPlay.ShouldBeTrue();
|
||||
result["RJ01230163"].HasReviews.ShouldBeTrue();
|
||||
@@ -58,6 +59,8 @@ public class DLSiteClientTests
|
||||
"RG0001",
|
||||
new ProductInfo()
|
||||
{
|
||||
WorkName = "Product 1",
|
||||
WorkNameMasked = "Product 1 (Masked)",
|
||||
WishlistCount = 250,
|
||||
DownloadCount = 100,
|
||||
Options = ["TRI", "DLP", "JPN"],
|
||||
@@ -69,6 +72,8 @@ public class DLSiteClientTests
|
||||
"RG0002",
|
||||
new ProductInfo()
|
||||
{
|
||||
WorkName = "Product 2",
|
||||
WorkNameMasked = "Product 2 (Masked)",
|
||||
WishlistCount = 100,
|
||||
DownloadCount = 50,
|
||||
Options = ["TRI", "ENG", "DOT", "VET"],
|
||||
@@ -91,6 +96,7 @@ public class DLSiteClientTests
|
||||
|
||||
// RG0001
|
||||
VoiceWorkDetails voiceWorkDetails = mappedCollection["RG0001"];
|
||||
voiceWorkDetails.Title.ShouldBe("Product 1");
|
||||
voiceWorkDetails.WishlistCount.ShouldBe(250);
|
||||
voiceWorkDetails.DownloadCount.ShouldBe(100);
|
||||
voiceWorkDetails.HasTrial.ShouldBe(true);
|
||||
@@ -105,6 +111,7 @@ public class DLSiteClientTests
|
||||
|
||||
// RG0002
|
||||
VoiceWorkDetails secondVoiceWorkDetails = mappedCollection["RG0002"];
|
||||
secondVoiceWorkDetails.Title.ShouldBe("Product 2");
|
||||
secondVoiceWorkDetails.WishlistCount.ShouldBe(100);
|
||||
secondVoiceWorkDetails.DownloadCount.ShouldBe(50);
|
||||
secondVoiceWorkDetails.HasTrial.ShouldBe(true);
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Application.Integrations.DLSite.Models;
|
||||
using JSMR.Application.Integrations.Ports;
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Application.Scanning.Ports;
|
||||
using JSMR.Infrastructure.Http;
|
||||
using JSMR.Infrastructure.Scanning;
|
||||
using JSMR.Tests.Utilities;
|
||||
@@ -101,6 +104,51 @@ public class VoiceWorkScannerTests
|
||||
result[1].Tags.ShouldBe(["ASMR", "バイノーラル/ダミヘ", "色仕掛け", "浮気", "百合", "レズ/女同士", "ツルペタ", "貧乳/微乳"]);
|
||||
}
|
||||
|
||||
// (Scan) + (Integration) = <This> ==> Ingestion
|
||||
[Fact]
|
||||
public async Task Scan_And_Integration_Ingest_Mapping_Test()
|
||||
{
|
||||
IVoiceWorksScanner scanner = Substitute.For<IVoiceWorksScanner>();
|
||||
|
||||
IReadOnlyList<DLSiteWork> scannedWorks =
|
||||
[
|
||||
new()
|
||||
{
|
||||
ProductId = "RJ1",
|
||||
ProductName = "Masked Product Title",
|
||||
MakerId = "RG1",
|
||||
Maker = "Some Maker",
|
||||
ImageUrl = "https://site.com/image_main.png",
|
||||
SmallImageUrl = "https://site.com/image_240x240.png"
|
||||
}
|
||||
];
|
||||
|
||||
scanner.ScanPageAsync(Arg.Any<VoiceWorkScanOptions>(), CancellationToken.None)
|
||||
.Returns(Task.FromResult(scannedWorks));
|
||||
|
||||
IDLSiteClient dlsiteClient = Substitute.For<IDLSiteClient>();
|
||||
|
||||
VoiceWorkDetailCollection detailCollection = new()
|
||||
{
|
||||
{
|
||||
"RJ1",
|
||||
new VoiceWorkDetails()
|
||||
{
|
||||
Title = "Product Title"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
dlsiteClient.GetVoiceWorkDetailsAsync(Arg.Any<string[]>(), CancellationToken.None)
|
||||
.Returns(Task.FromResult(detailCollection));
|
||||
|
||||
VoiceWorkIngest ingest = VoiceWorkIngest.From(scannedWorks[0], detailCollection["RJ1"]);
|
||||
|
||||
// TODO: Test other fields
|
||||
ingest.Title.ShouldBe("Product Title");
|
||||
ingest.HasImage.ShouldBe(true);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Scan_With_English_Locale()
|
||||
{
|
||||
|
||||
283
JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs
Normal file
283
JSMR.Tests/Unit/DLSiteWorkExpectedDateInferenceTests.cs
Normal file
@@ -0,0 +1,283 @@
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Infrastructure.Scanning.Extensions;
|
||||
using Shouldly;
|
||||
|
||||
namespace JSMR.Tests.Unit;
|
||||
|
||||
public class DLSiteWorkExpectedDateInferenceTests
|
||||
{
|
||||
[Fact]
|
||||
public void Infers_From_Sandwich_Expected_Dates()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1)
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Infers_From_Sandwich_Sales_Dates_Early()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 2)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 7)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
new DateOnly(2026, 5, 1),
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Infers_From_Sandwich_Sales_Dates_Middle()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 12)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 13)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
new DateOnly(2026, 5, 11),
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Infers_From_Sandwich_Sales_Dates_Late()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 25)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 27)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
new DateOnly(2026, 5, 21),
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Infers_From_Sandwich_Mixed_Dates()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 7)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Infers_A_Run_Of_Missing_Items_When_Bounded_By_Same_Date()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(),
|
||||
Work(),
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
];
|
||||
|
||||
DateOnly?[] expected =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 5, 1),
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_Expected_Sandwich_Difference()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(expected: new DateOnly(2026, 5, 11)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
null,
|
||||
new DateOnly(2026, 5, 11)
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Middle()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 12)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
null,
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_Sales_Sandwich_Difference_Early_Late()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 1)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 22)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
null,
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_Sales_Sandwich_Difference_Middle_Late()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(sales: new DateOnly(2026, 5, 14)),
|
||||
Work(),
|
||||
Work(sales: new DateOnly(2026, 5, 22)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null, // Sales date will not have expected date
|
||||
null,
|
||||
null // Sales date will not have expected date
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_No_Left()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(),
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
null,
|
||||
new DateOnly(2026, 5, 1)
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotInfer_When_No_Right()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work()
|
||||
];
|
||||
|
||||
DateOnly?[] expectedExpectedDates =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
null
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expectedExpectedDates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNot_Overwrite_Existing_ExpectedDate()
|
||||
{
|
||||
DLSiteWork[] works =
|
||||
[
|
||||
Work(expected: new DateOnly(2026, 5, 1)),
|
||||
Work(expected: new DateOnly(2026, 4, 11)), // already set
|
||||
Work(expected: new DateOnly(2026, 5, 21)),
|
||||
];
|
||||
|
||||
DateOnly?[] expected =
|
||||
[
|
||||
new DateOnly(2026, 5, 1),
|
||||
new DateOnly(2026, 4, 11),
|
||||
new DateOnly(2026, 5, 21),
|
||||
];
|
||||
|
||||
AttemptInferAndVerify(works, expected);
|
||||
}
|
||||
|
||||
private static void AttemptInferAndVerify(DLSiteWork[] works, DateOnly?[] expectedExpectedDates)
|
||||
{
|
||||
// Act
|
||||
works.InferAndUpdateExpectedDates();
|
||||
|
||||
// Assert
|
||||
works.Length.ShouldBe(expectedExpectedDates.Length);
|
||||
|
||||
for (int i = 0; i < works.Length; i++)
|
||||
works[i].ExpectedDate.ShouldBe(expectedExpectedDates[i]);
|
||||
}
|
||||
|
||||
private static DLSiteWork Work(DateOnly? expected = null, DateOnly? sales = null)
|
||||
=> DLSiteWorkTestFactory.Create(expectedDate: expected, salesDate: sales);
|
||||
}
|
||||
39
JSMR.Tests/Unit/DLSiteWorkTestFactory.cs
Normal file
39
JSMR.Tests/Unit/DLSiteWorkTestFactory.cs
Normal file
@@ -0,0 +1,39 @@
|
||||
using JSMR.Application.Scanning.Contracts;
|
||||
using JSMR.Domain.Enums;
|
||||
|
||||
namespace JSMR.Tests.Unit;
|
||||
|
||||
internal static class DLSiteWorkTestFactory
|
||||
{
|
||||
private static int _counter = 0;
|
||||
|
||||
public static DLSiteWork Create(DateOnly? expectedDate = null, DateOnly? salesDate = null)
|
||||
{
|
||||
int id = Interlocked.Increment(ref _counter);
|
||||
|
||||
return new DLSiteWork
|
||||
{
|
||||
ProductName = $"Test Product {id}",
|
||||
ProductId = $"RJ_TEST_{id:00000000}",
|
||||
Maker = "Test Maker",
|
||||
MakerId = "RG_TEST",
|
||||
ImageUrl = "//img.dlsite.jp/test_main.jpg",
|
||||
SmallImageUrl = "//img.dlsite.jp/test_sam.jpg",
|
||||
AgeRating = AgeRating.AllAges,
|
||||
|
||||
// Relevant fields for these tests:
|
||||
ExpectedDate = expectedDate,
|
||||
SalesDate = salesDate,
|
||||
|
||||
// The rest can be safe defaults:
|
||||
Downloads = 0,
|
||||
StarRating = null,
|
||||
Votes = null,
|
||||
Description = null,
|
||||
Genres = [],
|
||||
Tags = [],
|
||||
Creators = [],
|
||||
HasTrial = false
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -130,7 +130,7 @@ schemaDumpCommand.SetAction(async (parseResult, cancellationToken) =>
|
||||
|
||||
var sql = db.Database.GenerateCreateScript();
|
||||
var outPath = Path.GetFullPath("desired.sql");
|
||||
await File.WriteAllTextAsync(outPath, sql);
|
||||
await File.WriteAllTextAsync(outPath, sql, cancellationToken);
|
||||
|
||||
Console.WriteLine($"[OK] Wrote EF model create script to: {outPath}");
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using JSMR.Application.Enums;
|
||||
using JSMR.Application.Scanning;
|
||||
using JSMR.Application.Scanning.Ports;
|
||||
using JSMR.Infrastructure.Common.Time;
|
||||
using JSMR.Worker.Options;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -48,6 +49,27 @@ public sealed class PagedScanRunner(
|
||||
|
||||
ScanVoiceWorksResponse response = await handler.HandleAsync(request, cancellationToken);
|
||||
|
||||
//int newUpcoming = response.Results.Where(x => x.IsNewUpcoming == true).Count();
|
||||
|
||||
//if (newUpcoming > 0)
|
||||
// updatedInfo.Add($"{newUpcoming} new upcoming work(s)");
|
||||
|
||||
//int newOnSale = result.ScannedVoiceWorks.Where(x => x.IsNewOnSale == true).Count();
|
||||
|
||||
//if (newOnSale > 0)
|
||||
// updatedInfo.Add($"{newOnSale} new work(s) on sale");
|
||||
|
||||
IEnumerable<VoiceWorkUpsertResult> resultsWithIssues = response.Results.Where(x => x.Issues.Count > 0);
|
||||
|
||||
//foreach (VoiceWorkUpsertResult resultWithIssues in resultsWithIssues)
|
||||
//{
|
||||
// log.LogWarning($"PRoblem with {resultWithIssues.}")
|
||||
// string messageToDisplay = $"{scannedVoiceWork.ProductId} - {scannedVoiceWork.ProductName} -- {message}";
|
||||
|
||||
// Console.WriteLine(messageToDisplay);
|
||||
// messages.Add(messageToDisplay);
|
||||
//}
|
||||
|
||||
// Save checkpoint
|
||||
await checkpoints.SaveLastPageAsync(options.Locale, currentPage, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
/*"Default": "Information",*/
|
||||
"Default": "Warning",
|
||||
"Microsoft.AspNetCore": "Warning"
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user