brister/jsmr
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Updated various parts of scanning and ingestion, either for bug fixes, or for enhancements.
All checks were successful
ci / build-test (push) Successful in 2m22s
Details
ci / publish-image (push) Has been skipped
Details

Browse Source
This commit is contained in:
Brian Bicknell
2026-03-01 22:07:20 -05:00
parent 704a6fc433
commit 83655f13e9
20 changed files with 555 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
JSMR.Infrastructure/Data/Repositories/Circles/CircleSearchProvider.cs
View File

@@ -29,69 +29,6 @@ public class CircleSearchProvider(AppDbContext context) : SearchProvider<CircleS
return q;
}
//protected override IQueryable<CircleSearchItem> GetBaseQuery()
//{
// // Project from Circles so we can use correlated subqueries per CircleId.
// var q =
// from c in context.Circles.AsNoTracking()
// select new CircleSearchItem
// {
// CircleId = c.CircleId,
// Name = c.Name,
// MakerId = c.MakerId,
// Favorite = c.Favorite,
// Blacklisted = c.Blacklisted,
// Spam = c.Spam,
// // Aggregates
// Downloads = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .Select(v => (int?)v.Downloads) // make nullable for Sum over empty set
// .Sum() ?? 0,
// Releases = context.VoiceWorks
// .Count(v => v.CircleId == c.CircleId && v.SalesDate != null),
// Pending = context.VoiceWorks
// .Count(v => v.CircleId == c.CircleId && v.ExpectedDate != null),
// FirstReleaseDate = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .Select(v => v.SalesDate)
// .Min(),
// LatestReleaseDate = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .Select(v => v.SalesDate)
// .Max(),
// // "Latest" by ProductId length, then value
// LatestProductId = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .OrderByDescending(v => v.ProductId.Length)
// .ThenByDescending(v => v.ProductId)
// .Select(v => v.ProductId)
// .FirstOrDefault(),
// // If you want these two in base query too:
// LatestVoiceWorkHasImage = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .OrderByDescending(v => v.ProductId.Length)
// .ThenByDescending(v => v.ProductId)
// .Select(v => (bool?)v.HasImage)
// .FirstOrDefault(),
// LatestVoiceWorkSalesDate = context.VoiceWorks
// .Where(v => v.CircleId == c.CircleId)
// .OrderByDescending(v => v.ProductId.Length)
// .ThenByDescending(v => v.ProductId)
// .Select(v => v.SalesDate)
// .FirstOrDefault()
// };
// return q;
//}
protected override IQueryable<CircleQuery> ApplyFilters(IQueryable<CircleQuery> query, CircleSearchCriteria criteria)
{
if (!string.IsNullOrWhiteSpace(criteria.Name))

2
JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs
View File

@@ -205,7 +205,7 @@ public class VoiceWorkUpdater(AppDbContext dbContext, ITimeProvider timeProvider
else
{
voiceWork.SalesDate = null;
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0));
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0)) ?? voiceWork.ExpectedDate;
voiceWork.PlannedReleaseDate = ingest.RegistrationDate > upsertContext.CurrentScanAnchor ? ingest.RegistrationDate : null;
voiceWork.Status = state.IsNewUpcoming ? (byte)VoiceWorkStatus.NewAndUpcoming : (byte)VoiceWorkStatus.Upcoming;
}

1
JSMR.Infrastructure/Integrations/DLSite/Mapping/DLSiteToDomainMapper.cs
View File

@@ -51,6 +51,7 @@ public static class DLSiteToDomainMapper
return new VoiceWorkDetails
{
Title = productInfo.WorkName,
Series = MapSeries(productInfo),
Translation = MapTranslation(productInfo, optionsSet),
WishlistCount = productInfo.WishlistCount,

6
JSMR.Infrastructure/Integrations/DLSite/Models/ProductInfo.cs
View File

@@ -112,6 +112,9 @@ public class ProductInfo
[JsonPropertyName("title_name")]
public string? TitleName { get; set; }
[JsonPropertyName("title_name_masked")]
public string? TitleNameMasked { get; set; }
[JsonPropertyName("title_volumn")]
public int? TitleVolumeNumber { get; set; }
@@ -166,6 +169,9 @@ public class ProductInfo
[JsonPropertyName("work_name")]
public string? WorkName { get; set; }
[JsonPropertyName("work_name_masked")]
public string? WorkNameMasked { get; set; }
[JsonPropertyName("work_image")]
public string? WorkImage { get; set; }

62
JSMR.Infrastructure/Scanning/Extensions/DLSiteWorkExtensions.cs Normal file
View File

@@ -0,0 +1,62 @@
using JSMR.Application.Scanning.Contracts;
namespace JSMR.Infrastructure.Scanning.Extensions;
public static class DLSiteWorkExtensions
{
public static void InferAndUpdateExpectedDates(this DLSiteWork[] works)
{
// Precompute nearest known effective date on the left and right for each index.
var left = new DateOnly?[works.Length];
var right = new DateOnly?[works.Length];
DateOnly? last = null;
for (int i = 0; i < works.Length; i++)
{
var effective = GetEffectiveDate(works[i]);
if (effective.HasValue)
last = effective;
left[i] = last;
}
DateOnly? next = null;
for (int i = works.Length - 1; i >= 0; i--)
{
var effective = GetEffectiveDate(works[i]);
if (effective.HasValue)
next = effective;
right[i] = next;
}
// Fill only when BOTH sides exist and match.
for (int i = 0; i < works.Length; i++)
{
DLSiteWork work = works[i];
if (work.SalesDate.HasValue || work.ExpectedDate.HasValue)
continue;
DateOnly? previous = (i > 0) ? left[i - 1] : null;
DateOnly? nxt = (i < works.Length - 1) ? right[i + 1] : null;
if (previous.HasValue && nxt.HasValue && previous.Value == nxt.Value)
work.ExpectedDate = previous.Value;
}
}
private static DateOnly? GetEffectiveDate(DLSiteWork work)
{
if (work.ExpectedDate.HasValue)
return work.ExpectedDate.Value;
if (!work.SalesDate.HasValue)
return null;
// Bucket sales day to Early/Middle/Late => 1/11/21
var d = work.SalesDate.Value;
int day = d.Day >= 21 ? 21 : d.Day >= 11 ? 11 : 1;
return new DateOnly(d.Year, d.Month, day);
}
}

11
JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs
View File

@@ -5,9 +5,9 @@ using JSMR.Application.Scanning.Ports;
using JSMR.Domain.Enums;
using JSMR.Domain.ValueObjects;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Scanning.Extensions;
using JSMR.Infrastructure.Scanning.Models;
using System.Globalization;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace JSMR.Infrastructure.Scanning;
@@ -25,7 +25,10 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
DLSiteHtmlDocument document = await GetDLSiteHtmlCollectionAsync(options, cancellationToken);
DLSiteHtmlNode[] nodes = document.GetDLSiteNodes();
return GetDLSiteWorks(nodes, options);
DLSiteWork[] works = GetDLSiteWorks(nodes, options);
works.InferAndUpdateExpectedDates();
return works;
}
private async Task<DLSiteHtmlDocument> GetDLSiteHtmlCollectionAsync(VoiceWorkScanOptions options, CancellationToken cancellationToken)
@@ -53,7 +56,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
return filterBuilder.BuildSearchQuery(options.PageNumber, options.PageSize);
}
private List<DLSiteWork> GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options)
private DLSiteWork[] GetDLSiteWorks(DLSiteHtmlNode[] nodes, VoiceWorkScanOptions options)
{
var works = new List<DLSiteWork>();
@@ -67,7 +70,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
works.Add(work);
}
return works;
return [.. works];
}
private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node)