Added English voice work updater.

This commit is contained in:
2025-10-23 10:07:50 -04:00
parent 3a115bc7b8
commit 36fcd5379a
19 changed files with 651 additions and 401 deletions

View File

@@ -3,26 +3,51 @@ using JSMR.Application.Integrations.DLSite.Models;
namespace JSMR.Application.Scanning.Contracts;
public record VoiceWorkIngest(DLSiteWork Work, VoiceWorkDetails? Details)
public sealed record VoiceWorkIngest
{
public string MakerId { get; init; } = Work.MakerId;
public string MakerName { get; init; } = Work.Maker;
public string ProductId { get; init; } = Work.ProductId;
public string Title { get; init; } = Work.ProductName;
public string? Description { get; init; } = Work.Description;
public ICollection<string> Tags { get; init; } = Work.Tags;
public ICollection<string> Creators { get; init; } = Work.Creators;
public int WishlistCount { get; init; } = Details?.WishlistCount ?? 0;
public int Downloads { get; init; } = Math.Max(Work.Downloads, Details?.DownloadCount ?? 0);
public bool HasTrial { get; init; } = Work.HasTrial || (Details?.HasTrial ?? false);
public bool HasDLPlay { get; init; } = Details?.HasDLPlay ?? false;
public byte? StarRating { get; init; } = Work.StarRating;
public int? Votes { get; init; } = Work.Votes;
public AgeRating AgeRating { get; init; } = Details?.AgeRating ?? Work.AgeRating;
public bool HasImage { get; init; } = Work.ImageUrl.Contains("no_img") == false;
public ICollection<ISupportedLanguage> SupportedLanguages { get; init; } = Details?.SupportedLanguages ?? [];
public DateOnly? ExpectedDate { get; init; } = Work.ExpectedDate;
public DateOnly? SalesDate { get; init; } = Work.SalesDate;
public DateTime? RegistrationDate { get; init; } = Details?.RegistrationDate;
// TODO: Other properties
public required string MakerId { get; init; }
public required string MakerName { get; init; }
public required string ProductId { get; init; }
public required string Title { get; init; }
public required string Description { get; init; }
public ICollection<string> Tags { get; init; } = [];
public ICollection<string> Creators { get; init; } = [];
public int WishlistCount { get; init; }
public int Downloads { get; init; }
public bool HasTrial { get; init; }
public bool HasDLPlay { get; init; }
public byte? StarRating { get; init; }
public int? Votes { get; init; }
public AgeRating AgeRating { get; init; }
public bool HasImage { get; init; }
public ICollection<ISupportedLanguage> SupportedLanguages { get; init; } = [];
public DateOnly? ExpectedDate { get; init; }
public DateOnly? SalesDate { get; init; }
public DateTime? RegistrationDate { get; init; }
public static VoiceWorkIngest From(DLSiteWork work, VoiceWorkDetails? details)
{
return new VoiceWorkIngest()
{
MakerId = work.MakerId,
MakerName = work.Maker,
ProductId = work.ProductId,
Title = work.ProductName,
Description = work.Description ?? string.Empty,
Tags = work.Tags,
Creators = work.Creators,
WishlistCount = details?.WishlistCount ?? 0,
Downloads = Math.Max(work.Downloads, details?.DownloadCount ?? 0),
HasTrial = work.HasTrial || (details?.HasTrial ?? false),
HasDLPlay = details?.HasDLPlay ?? false,
StarRating = work.StarRating,
Votes = work.Votes,
AgeRating = details?.AgeRating ?? work.AgeRating,
HasImage = !string.IsNullOrEmpty(work.ImageUrl) && !work.ImageUrl.Contains("no_img", StringComparison.OrdinalIgnoreCase),
SupportedLanguages = details?.SupportedLanguages ?? [],
ExpectedDate = work.ExpectedDate,
SalesDate = work.SalesDate,
RegistrationDate = details?.RegistrationDate
};
}
}

View File

@@ -0,0 +1,8 @@
using JSMR.Application.Scanning.Contracts;
namespace JSMR.Application.Scanning.Ports;
public interface IVoiceWorkUpdater
{
Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken);
}

View File

@@ -3,7 +3,6 @@ using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Ports;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports;
using JSMR.Application.VoiceWorks.Ports;
using Microsoft.Extensions.DependencyInjection;
namespace JSMR.Application.Scanning;
@@ -12,14 +11,14 @@ public sealed class ScanVoiceWorksHandler(
IServiceProvider serviceProvider,
IDLSiteClient dlsiteClient,
ISpamCircleCache spamCircleCache,
IVoiceWorkWriter writer,
IVoiceWorkSearchUpdater searchUpdater)
{
public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
{
IVoiceWorksScanner? scanner = serviceProvider.GetKeyedService<IVoiceWorksScanner>(request.Locale);
IVoiceWorkUpdater? updater = serviceProvider.GetKeyedService<IVoiceWorkUpdater>(request.Locale);
if (scanner is null)
if (scanner is null || updater is null)
return new();
VoiceWorkScanOptions options = new(
@@ -41,10 +40,10 @@ public sealed class ScanVoiceWorksHandler(
VoiceWorkIngest[] ingests = [.. works.Select(work =>
{
voiceWorkDetails.TryGetValue(work.ProductId!, out VoiceWorkDetails? value);
return new VoiceWorkIngest(work, value);
return VoiceWorkIngest.From(work, value);
})];
int[] voiceWorkIds = await writer.UpsertAsync(ingests, cancellationToken);
int[] voiceWorkIds = await updater.UpsertAsync(ingests, cancellationToken);
await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
return new();

View File

@@ -1,10 +1,8 @@
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
namespace JSMR.Application.VoiceWorks.Ports;
public interface IVoiceWorkWriter
{
Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken);
Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken);
}

View File

@@ -29,4 +29,35 @@ public sealed class SearchVoiceWorksHandler(IVoiceWorkSearchProvider provider, I
return new SearchVoiceWorksResponse(results);
}
}
}
//public record SearchProviderContext(
//);
//public class SearchHandler<TSearchResult, TCriteria, TSortField>(ISearchProvider<TSearchResult, TCriteria, TSortField> searchProvider, ILogger logger)
// where TCriteria : notnull, new()
// where TSortField : struct, Enum
//{
// public async Task<SearchVoiceWorksResponse> HandleAsync(SearchOptions<TCriteria, TSortField> options, CancellationToken cancellationToken)
// {
// Stopwatch stopWatch = Stopwatch.StartNew();
// SearchResult<TSearchResult> results = await searchProvider.SearchAsync(options, cancellationToken);
// long elapsedMilliseconds = stopWatch.ElapsedMilliseconds;
// LogEvents.SearchCompleted(
// logger,
// Elapsed: elapsedMilliseconds,
// Items: results.Items.Length,
// Total: results.TotalItems,
// Page: options.PageNumber,
// Size: options.PageSize,
// Sort: options.SortOptions.ToLogObject(),
// Criteria: options.Criteria.ToLogObject()
// );
// return new SearchVoiceWorksResponse(results);
// }
//}

View File

@@ -0,0 +1,8 @@
namespace JSMR.Infrastructure.Common.Time;
public interface ITimeProvider
{
DateTimeOffset Now();
DateTimeOffset Local(int year, int month, int day, int hour);
DateTimeOffset Local(DateTimeOffset offset);
}

View File

@@ -1,12 +1,5 @@
namespace JSMR.Infrastructure.Common.Time;
public interface ITimeProvider
{
DateTimeOffset Now();
DateTimeOffset Local(int year, int month, int day, int hour);
DateTimeOffset Local(DateTimeOffset offset);
}
public abstract class TimeProvider : ITimeProvider
{
protected abstract string Id { get; }
@@ -42,28 +35,4 @@ public abstract class TimeProvider : ITimeProvider
return new DateTimeOffset(local, offset);
}
public DateTimeOffset CurrentScanAnchor()
{
DateTimeOffset now = Now();
DateTimeOffset midnight = Local(now.Year, now.Month, now.Day, 0);
DateTimeOffset fourPm = Local(now.Year, now.Month, now.Day, 16);
return now >= fourPm ? fourPm : midnight;
}
public DateTimeOffset PreviousScanAnchor(DateTimeOffset scanAnchorTokyo)
{
// Normalize to Tokyo (no-op if already)
var a = TimeZoneInfo.ConvertTime(scanAnchorTokyo, _timeZone);
return a.Hour == 16
? Local(a.Year, a.Month, a.Day, 0)
: Local(a.AddDays(-1).Year, a.AddDays(-1).Month, a.AddDays(-1).Day, 16);
}
}
public class TokyoTimeProvider(IClock clock) : TimeProvider(clock)
{
protected override string Id => "Tokyo Standard Time";
protected override string[] TimeZoneIds => ["Tokyo Standard Time", "Asia/Tokyo"];
}

View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Common.Time;
public class TokyoTimeProvider(IClock clock) : TimeProvider(clock)
{
protected override string Id => "Tokyo Standard Time";
protected override string[] TimeZoneIds => ["Tokyo Standard Time", "Asia/Tokyo"];
}

View File

@@ -20,6 +20,8 @@ using JSMR.Infrastructure.Data.Repositories.Creators;
using JSMR.Infrastructure.Data.Repositories.Tags;
using JSMR.Infrastructure.Data.Repositories.VoiceWorks;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Ingest;
using JSMR.Infrastructure.Ingestions;
using JSMR.Infrastructure.Scanning;
using Microsoft.Extensions.DependencyInjection;
@@ -36,9 +38,13 @@ public static class InfrastructureServiceCollectionExtensions
services.AddScoped<IVoiceWorkSearchProvider, VoiceWorkSearchProvider>();
services.AddScoped<IVoiceWorkFullTextSearch, MySqlVoiceWorkFullTextSearch>();
services.AddScoped<IVoiceWorkWriter, VoiceWorkWriter>();
services.AddKeyedScoped<IVoiceWorksScanner, JapaneseVoiceWorksScanner>(Locale.Japanese);
services.AddKeyedScoped<IVoiceWorksScanner, EnglishVoiceWorksScanner>(Locale.English);
services.AddKeyedScoped<IVoiceWorkUpdater, VoiceWorkUpdater>(Locale.Japanese);
services.AddKeyedScoped<IVoiceWorkUpdater, EnglishVoiceWorkUpdater>(Locale.English);
services.AddKeyedScoped<ISupportedLanguage, JapaneseLanguage>(Locale.Japanese);
services.AddKeyedScoped<ISupportedLanguage, EnglishLanguage>(Locale.English);
services.AddKeyedScoped<ISupportedLanguage, SimplifiedChineseLanguage>(Locale.ChineseSimplified);

View File

@@ -1,12 +0,0 @@
using JSMR.Domain.Entities;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public record VoiceWorkUpsertContext(
DateTimeOffset CurrentScanAnchor,
DateTimeOffset PreviousScanAnchor,
Dictionary<string, Circle> Circles,
Dictionary<string, VoiceWork> VoiceWorks,
Dictionary<string, Tag> Tags,
Dictionary<string, Creator> Creators
);

View File

@@ -1,6 +1,4 @@
using JSMR.Application.Common;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
using JSMR.Application.VoiceWorks.Ports;
using JSMR.Domain.Entities;
using JSMR.Infrastructure.Common.Time;
@@ -8,286 +6,8 @@ using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public class VoiceWorkWriter(AppDbContext dbContext, ITimeProvider timeProvider) : IVoiceWorkWriter
public class VoiceWorkWriter(AppDbContext dbContext) : IVoiceWorkWriter
{
public async Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
VoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken);
foreach (VoiceWorkIngest ingest in ingests)
{
Upsert(ingest, upsertContext);
}
await dbContext.SaveChangesAsync(cancellationToken);
return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)];
}
private async Task<VoiceWorkUpsertContext> CreateUpsertContextAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()];
string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()];
string[] tagNames = [.. ingests.SelectMany(i => i.Tags).Distinct()];
string[] creatorNames = [.. ingests.SelectMany(i => i.Creators).Distinct()];
DateTimeOffset currentScanAnchor = GetCurrentScanAnchor();
DateTimeOffset previousScanAnchor = PreviousScanAnchor(currentScanAnchor);
VoiceWorkUpsertContext upsertContext = new(
CurrentScanAnchor: currentScanAnchor,
PreviousScanAnchor: previousScanAnchor,
Circles: await dbContext.Circles
.Where(c => makerIds.Contains(c.MakerId))
.ToDictionaryAsync(c => c.MakerId, cancellationToken),
VoiceWorks: await dbContext.VoiceWorks
.Where(v => productIds.Contains(v.ProductId))
.Include(v => v.Creators)
.Include(v => v.Tags)
.Include(v => v.Localizations)
.Include(v => v.SupportedLanguages)
.ToDictionaryAsync(v => v.ProductId, cancellationToken),
Tags: await dbContext.Tags
.Where(t => tagNames.Contains(t.Name))
.ToDictionaryAsync(t => t.Name, cancellationToken),
Creators: await dbContext.Creators
.Where(cr => creatorNames.Contains(cr.Name))
.ToDictionaryAsync(cr => cr.Name, cancellationToken)
);
return upsertContext;
}
private DateTimeOffset GetCurrentScanAnchor()
{
DateTimeOffset now = timeProvider.Now();
DateTimeOffset midnight = timeProvider.Local(now.Year, now.Month, now.Day, 0);
DateTimeOffset fourPm = timeProvider.Local(now.Year, now.Month, now.Day, 16);
return now >= fourPm ? fourPm : midnight;
}
private DateTimeOffset PreviousScanAnchor(DateTimeOffset scanAnchorTokyo)
{
// Normalize to Tokyo (no-op if already)
var a = timeProvider.Local(scanAnchorTokyo);
return a.Hour == 16
? timeProvider.Local(a.Year, a.Month, a.Day, 0)
: timeProvider.Local(a.AddDays(-1).Year, a.AddDays(-1).Month, a.AddDays(-1).Day, 16);
}
private void Upsert(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
UpsertCircle(ingest, upsertContext);
UpsertVoiceWork(ingest, upsertContext);
UpsertTags(ingest, upsertContext);
UpsertVoiceWorkTags(ingest, upsertContext);
UpsertCreators(ingest, upsertContext);
UpsertVoiceWorkCreators(ingest, upsertContext);
UpsertVoiceWorkSupportedLanguages(ingest, upsertContext);
}
private void UpsertCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
Circle circle = GetOrAddCircle(ingest, upsertContext);
circle.Name = ingest.MakerName;
}
private Circle GetOrAddCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle))
{
circle = new Circle
{
MakerId = ingest.MakerId,
Name = ingest.MakerName,
};
dbContext.Circles.Add(circle);
upsertContext.Circles[ingest.MakerId] = circle;
}
return circle;
}
private void UpsertVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = GetOrAddVoiceWork(ingest, upsertContext);
bool isAdded = dbContext.Entry(voiceWork).State == EntityState.Added;
bool isWithinCurrentScanAnchor = voiceWork.LastScannedDate == upsertContext.CurrentScanAnchor;
bool isNewOnSale = voiceWork.SalesDate is null && ingest.SalesDate is not null;
bool isNew = isAdded || isWithinCurrentScanAnchor || isNewOnSale;
voiceWork.Circle = upsertContext.Circles[ingest.MakerId];
voiceWork.ProductName = ingest.Title;
voiceWork.Description = ingest.Description;
voiceWork.HasImage = ingest.HasImage;
voiceWork.Rating = (int)ingest.AgeRating;
voiceWork.Downloads = ingest.Downloads;
voiceWork.WishlistCount = ingest.WishlistCount;
voiceWork.HasTrial = ingest.HasTrial;
voiceWork.HasChobit = ingest.HasDLPlay;
voiceWork.StarRating = ingest.StarRating;
voiceWork.Votes = ingest.Votes;
voiceWork.IsValid = true;
if (ingest.SalesDate.HasValue)
{
voiceWork.SalesDate = ingest.SalesDate.Value.ToDateTime(new TimeOnly(0, 0));
voiceWork.ExpectedDate = null;
voiceWork.PlannedReleaseDate = null;
voiceWork.Status = isNew ? (byte)VoiceWorkStatus.NewRelease : (byte)VoiceWorkStatus.Available;
}
else
{
voiceWork.SalesDate = null;
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0));
voiceWork.PlannedReleaseDate = ingest.RegistrationDate > upsertContext.CurrentScanAnchor ? ingest.RegistrationDate : null;
voiceWork.Status = isNew ? (byte)VoiceWorkStatus.NewAndUpcoming : (byte)VoiceWorkStatus.Upcoming;
}
}
private VoiceWork GetOrAddVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork))
{
voiceWork = new VoiceWork
{
ProductId = ingest.ProductId
};
dbContext.VoiceWorks.Add(voiceWork);
upsertContext.VoiceWorks[ingest.ProductId] = voiceWork;
}
return voiceWork;
}
private void UpsertTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
foreach (string tagName in ingest.Tags)
{
GetOrAddTag(tagName, upsertContext);
}
}
private Tag GetOrAddTag(string tagName, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Tags.TryGetValue(tagName, out Tag? tag))
{
tag = new Tag
{
Name = tagName
};
dbContext.Tags.Add(tag);
upsertContext.Tags[tagName] = tag;
}
return tag;
}
private void UpsertCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
foreach (string creatorName in ingest.Creators)
{
GetOrAddCreator(creatorName, upsertContext);
}
}
private Creator GetOrAddCreator(string creatorName, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Creators.TryGetValue(creatorName, out Creator? creator))
{
creator = new Creator
{
Name = creatorName
};
dbContext.Creators.Add(creator);
upsertContext.Creators[creatorName] = creator;
}
return creator;
}
private void UpsertVoiceWorkTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkTag> existingTagLinks = voiceWork.Tags.ToDictionary(x => x.TagId);
int position = 1;
foreach (string tagName in ingest.Tags)
{
Tag tag = upsertContext.Tags[tagName];
if (!existingTagLinks.TryGetValue(tag.TagId, out VoiceWorkTag? voiceWorkTag))
{
voiceWorkTag = new VoiceWorkTag
{
VoiceWork = voiceWork,
Tag = tag
};
dbContext.VoiceWorkTags.Add(voiceWorkTag);
}
voiceWorkTag.Position = position++;
voiceWorkTag.IsValid = true;
}
}
private void UpsertVoiceWorkCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkCreator> existingCreatorLinks = voiceWork.Creators.ToDictionary(x => x.CreatorId);
int position = 1;
foreach (string creatorName in ingest.Creators)
{
Creator creator = upsertContext.Creators[creatorName];
if (!existingCreatorLinks.TryGetValue(creator.CreatorId, out VoiceWorkCreator? voiceWorkCreator))
{
voiceWorkCreator = new VoiceWorkCreator
{
VoiceWork = voiceWork,
Creator = creator
};
dbContext.VoiceWorkCreators.Add(voiceWorkCreator);
}
voiceWorkCreator.Position = position++;
voiceWorkCreator.IsValid = true;
}
}
private void UpsertVoiceWorkSupportedLanguages(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<string, VoiceWorkSupportedLanguage> existingLanguageLinks = voiceWork.SupportedLanguages.ToDictionary(x => x.Language);
foreach (ISupportedLanguage supportedLanguage in ingest.SupportedLanguages)
{
if (!existingLanguageLinks.TryGetValue(supportedLanguage.Code, out VoiceWorkSupportedLanguage? voiceWorkSupportedLanguage))
{
voiceWorkSupportedLanguage = new VoiceWorkSupportedLanguage
{
VoiceWork = voiceWork,
Language = supportedLanguage.Code
};
dbContext.VoiceWorkSupportedLanguages.Add(voiceWorkSupportedLanguage);
}
}
}
public async Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken)
{
VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken);

View File

@@ -0,0 +1,118 @@
using JSMR.Application.Common;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports;
using JSMR.Domain.Entities;
using JSMR.Infrastructure.Common.Languages;
using JSMR.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Ingestion;
public class EnglishVoiceWorkUpdater(AppDbContext dbContext, ILanguageIdentifier languageIdentifier) : IVoiceWorkUpdater
{
public async Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
EnglishVoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken);
foreach (VoiceWorkIngest ingest in ingests)
{
Validate(ingest, upsertContext);
VoiceWorkUpsertResult result = upsertContext.Results[ingest.ProductId];
if (result.Issues.Count > 0)
{
result.Status = VoiceWorkUpsertStatus.Skipped;
continue;
}
UpsertEnglishVoiceWork(ingest, upsertContext);
result.Status = VoiceWorkUpsertStatus.Updated;
}
await dbContext.SaveChangesAsync(cancellationToken);
return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)];
}
private async Task<EnglishVoiceWorkUpsertContext> CreateUpsertContextAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()];
string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()];
EnglishVoiceWorkUpsertContext upsertContext = new(
Circles: await dbContext.Circles
.Where(c => makerIds.Contains(c.MakerId))
.ToDictionaryAsync(c => c.MakerId, cancellationToken),
VoiceWorks: await dbContext.VoiceWorks
.Where(v => productIds.Contains(v.ProductId))
.Include(v => v.EnglishVoiceWorks)
.Include(v => v.Localizations)
.ToDictionaryAsync(v => v.ProductId, cancellationToken),
Results: productIds.ToDictionary(
productId => productId,
productId => new VoiceWorkUpsertResult()
)
);
return upsertContext;
}
private void Validate(VoiceWorkIngest ingest, EnglishVoiceWorkUpsertContext upsertContext)
{
VoiceWorkUpsertResult result = upsertContext.Results[ingest.ProductId];
bool isTitleEnglish = languageIdentifier.GetLanguage(ingest.Title) == Language.English;
bool isDescriptionEnglish = !string.IsNullOrWhiteSpace(ingest.Description) && languageIdentifier.GetLanguage(ingest.Description) == Language.English;
if (!isTitleEnglish && !isDescriptionEnglish)
{
string message = $"Prouct title and/or description is not in English";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Information));
return;
}
if (upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle) == false)
{
string message = $"Unable to find circle for maker id: {ingest.MakerId}";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Error));
return;
}
if (upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork) == false)
{
string message = $"Unable to find voice work for product id: {ingest.ProductId}";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Error));
}
}
private void UpsertEnglishVoiceWork(VoiceWorkIngest ingest, EnglishVoiceWorkUpsertContext upsertContext)
{
EnglishVoiceWork englishVoiceWork = GetOrAddEnglishVoiceWork(ingest, upsertContext);
englishVoiceWork.ProductName = ingest.Title;
englishVoiceWork.Description = ingest.Description;
englishVoiceWork.IsValid = true;
}
private EnglishVoiceWork GetOrAddEnglishVoiceWork(VoiceWorkIngest ingest, EnglishVoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
EnglishVoiceWork? englishVoiceWork = voiceWork.EnglishVoiceWorks.FirstOrDefault();
if (englishVoiceWork is null)
{
englishVoiceWork = new EnglishVoiceWork
{
VoiceWork = voiceWork,
ProductName = string.Empty,
Description = string.Empty
};
dbContext.EnglishVoiceWorks.Add(englishVoiceWork);
//upsertContext.VoiceWorks[ingest.ProductId] = voiceWork;
}
return englishVoiceWork;
}
}

View File

@@ -0,0 +1,9 @@
using JSMR.Domain.Entities;
namespace JSMR.Infrastructure.Ingestion;
public record EnglishVoiceWorkUpsertContext(
Dictionary<string, Circle> Circles,
Dictionary<string, VoiceWork> VoiceWorks,
Dictionary<string, VoiceWorkUpsertResult> Results
);

View File

@@ -1,9 +1,10 @@
using JSMR.Application.Scanning.Ports;
using JSMR.Domain.Entities;
using JSMR.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;
using System.Text;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
namespace JSMR.Infrastructure.Ingestion;
public class VoiceWorkSearchUpdater(AppDbContext dbContext) : IVoiceWorkSearchUpdater
{

View File

@@ -0,0 +1,337 @@
using JSMR.Application.Common;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports;
using JSMR.Domain.Entities;
using JSMR.Infrastructure.Common.Time;
using JSMR.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Ingestion;
public class VoiceWorkUpdater(AppDbContext dbContext, ITimeProvider timeProvider) : IVoiceWorkUpdater
{
public async Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
VoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken);
foreach (VoiceWorkIngest ingest in ingests)
{
Validate(ingest, upsertContext);
VoiceWorkUpsertResult result = upsertContext.Results[ingest.ProductId];
if (result.Issues.Count > 0)
{
result.Status = VoiceWorkUpsertStatus.Skipped;
continue;
}
Upsert(ingest, upsertContext);
}
await dbContext.SaveChangesAsync(cancellationToken);
return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)];
}
private async Task<VoiceWorkUpsertContext> CreateUpsertContextAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()];
string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()];
string[] tagNames = [.. ingests.SelectMany(i => i.Tags).Distinct()];
string[] creatorNames = [.. ingests.SelectMany(i => i.Creators).Distinct()];
DateTimeOffset currentScanAnchor = GetCurrentScanAnchor();
DateTimeOffset previousScanAnchor = PreviousScanAnchor(currentScanAnchor);
VoiceWorkUpsertContext upsertContext = new(
CurrentScanAnchor: currentScanAnchor,
PreviousScanAnchor: previousScanAnchor,
Circles: await dbContext.Circles
.Where(c => makerIds.Contains(c.MakerId))
.ToDictionaryAsync(c => c.MakerId, cancellationToken),
VoiceWorks: await dbContext.VoiceWorks
.Where(v => productIds.Contains(v.ProductId))
.Include(v => v.Creators)
.Include(v => v.Tags)
.Include(v => v.Localizations)
.Include(v => v.SupportedLanguages)
.ToDictionaryAsync(v => v.ProductId, cancellationToken),
Tags: await dbContext.Tags
.Where(t => tagNames.Contains(t.Name))
.ToDictionaryAsync(t => t.Name, cancellationToken),
Creators: await dbContext.Creators
.Where(cr => creatorNames.Contains(cr.Name))
.ToDictionaryAsync(cr => cr.Name, cancellationToken),
Results: productIds.ToDictionary(
productId => productId,
productId => new VoiceWorkUpsertResult()
)
);
return upsertContext;
}
private DateTimeOffset GetCurrentScanAnchor()
{
DateTimeOffset now = timeProvider.Now();
DateTimeOffset midnight = timeProvider.Local(now.Year, now.Month, now.Day, 0);
DateTimeOffset fourPm = timeProvider.Local(now.Year, now.Month, now.Day, 16);
return now >= fourPm ? fourPm : midnight;
}
private DateTimeOffset PreviousScanAnchor(DateTimeOffset scanAnchorTokyo)
{
// Normalize to Tokyo (no-op if already)
var a = timeProvider.Local(scanAnchorTokyo);
return a.Hour == 16
? timeProvider.Local(a.Year, a.Month, a.Day, 0)
: timeProvider.Local(a.AddDays(-1).Year, a.AddDays(-1).Month, a.AddDays(-1).Day, 16);
}
private static void Validate(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWorkUpsertResult result = upsertContext.Results[ingest.ProductId];
if (upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle) == false)
return;
if (circle.Spam)
{
string message = $"Circle {ingest.MakerName} ({ingest.MakerId}) is a spam circle";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Error));
return;
}
if (upsertContext.VoiceWorks.TryGetValue(ingest.MakerId, out VoiceWork? voiceWork) == false)
return;
int ingestDownloads = ingest.Downloads;
int voiceWorkDownloads = voiceWork.Downloads ?? 0;
if (ingestDownloads < voiceWorkDownloads)
{
string message = $"Downloads have decreased from {voiceWorkDownloads} to {ingestDownloads}";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Error));
}
if (voiceWork.SalesDate is not null && ingest.SalesDate is null)
{
string message = $"Voice work has a sales date of {voiceWork.SalesDate.Value.ToShortDateString()}, but parsed ingest does not";
result.Issues.Add(new(message, VoiceWorkUpsertIssueSeverity.Error));
}
}
private void Upsert(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
UpsertCircle(ingest, upsertContext);
UpsertVoiceWork(ingest, upsertContext);
UpsertTags(ingest, upsertContext);
UpsertVoiceWorkTags(ingest, upsertContext);
UpsertCreators(ingest, upsertContext);
UpsertVoiceWorkCreators(ingest, upsertContext);
UpsertVoiceWorkSupportedLanguages(ingest, upsertContext);
}
private void UpsertCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
Circle circle = GetOrAddCircle(ingest, upsertContext);
circle.Name = ingest.MakerName;
}
private Circle GetOrAddCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle))
{
circle = new Circle
{
MakerId = ingest.MakerId,
Name = ingest.MakerName,
};
dbContext.Circles.Add(circle);
upsertContext.Circles[ingest.MakerId] = circle;
}
return circle;
}
private void UpsertVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = GetOrAddVoiceWork(ingest, upsertContext);
bool isAdded = dbContext.Entry(voiceWork).State == EntityState.Added;
bool isWithinCurrentScanAnchor = voiceWork.LastScannedDate == upsertContext.CurrentScanAnchor;
bool isNewOnSale = voiceWork.SalesDate is null && ingest.SalesDate is not null;
bool isNew = isAdded || isWithinCurrentScanAnchor || isNewOnSale;
voiceWork.Circle = upsertContext.Circles[ingest.MakerId];
voiceWork.ProductName = ingest.Title;
voiceWork.Description = ingest.Description;
voiceWork.HasImage = ingest.HasImage;
voiceWork.Rating = (int)ingest.AgeRating;
voiceWork.Downloads = ingest.Downloads;
voiceWork.WishlistCount = ingest.WishlistCount;
voiceWork.HasTrial = ingest.HasTrial;
voiceWork.HasChobit = ingest.HasDLPlay;
voiceWork.StarRating = ingest.StarRating;
voiceWork.Votes = ingest.Votes;
voiceWork.IsValid = true;
if (ingest.SalesDate.HasValue)
{
voiceWork.SalesDate = ingest.SalesDate.Value.ToDateTime(new TimeOnly(0, 0));
voiceWork.ExpectedDate = null;
voiceWork.PlannedReleaseDate = null;
voiceWork.Status = isNew ? (byte)VoiceWorkStatus.NewRelease : (byte)VoiceWorkStatus.Available;
}
else
{
voiceWork.SalesDate = null;
voiceWork.ExpectedDate = ingest.ExpectedDate?.ToDateTime(new TimeOnly(0, 0));
voiceWork.PlannedReleaseDate = ingest.RegistrationDate > upsertContext.CurrentScanAnchor ? ingest.RegistrationDate : null;
voiceWork.Status = isNew ? (byte)VoiceWorkStatus.NewAndUpcoming : (byte)VoiceWorkStatus.Upcoming;
}
}
private VoiceWork GetOrAddVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork))
{
voiceWork = new VoiceWork
{
ProductId = ingest.ProductId
};
dbContext.VoiceWorks.Add(voiceWork);
upsertContext.VoiceWorks[ingest.ProductId] = voiceWork;
}
return voiceWork;
}
private void UpsertTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
foreach (string tagName in ingest.Tags)
{
GetOrAddTag(tagName, upsertContext);
}
}
private Tag GetOrAddTag(string tagName, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Tags.TryGetValue(tagName, out Tag? tag))
{
tag = new Tag
{
Name = tagName
};
dbContext.Tags.Add(tag);
upsertContext.Tags[tagName] = tag;
}
return tag;
}
private void UpsertCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
foreach (string creatorName in ingest.Creators)
{
GetOrAddCreator(creatorName, upsertContext);
}
}
private Creator GetOrAddCreator(string creatorName, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Creators.TryGetValue(creatorName, out Creator? creator))
{
creator = new Creator
{
Name = creatorName
};
dbContext.Creators.Add(creator);
upsertContext.Creators[creatorName] = creator;
}
return creator;
}
private void UpsertVoiceWorkTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkTag> existingTagLinks = voiceWork.Tags.ToDictionary(x => x.TagId);
int position = 1;
foreach (string tagName in ingest.Tags)
{
Tag tag = upsertContext.Tags[tagName];
if (!existingTagLinks.TryGetValue(tag.TagId, out VoiceWorkTag? voiceWorkTag))
{
voiceWorkTag = new VoiceWorkTag
{
VoiceWork = voiceWork,
Tag = tag
};
dbContext.VoiceWorkTags.Add(voiceWorkTag);
}
voiceWorkTag.Position = position++;
voiceWorkTag.IsValid = true;
}
}
private void UpsertVoiceWorkCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkCreator> existingCreatorLinks = voiceWork.Creators.ToDictionary(x => x.CreatorId);
int position = 1;
foreach (string creatorName in ingest.Creators)
{
Creator creator = upsertContext.Creators[creatorName];
if (!existingCreatorLinks.TryGetValue(creator.CreatorId, out VoiceWorkCreator? voiceWorkCreator))
{
voiceWorkCreator = new VoiceWorkCreator
{
VoiceWork = voiceWork,
Creator = creator
};
dbContext.VoiceWorkCreators.Add(voiceWorkCreator);
}
voiceWorkCreator.Position = position++;
voiceWorkCreator.IsValid = true;
}
}
private void UpsertVoiceWorkSupportedLanguages(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = upsertContext.VoiceWorks[ingest.ProductId];
Dictionary<string, VoiceWorkSupportedLanguage> existingLanguageLinks = voiceWork.SupportedLanguages.ToDictionary(x => x.Language);
foreach (ISupportedLanguage supportedLanguage in ingest.SupportedLanguages)
{
if (!existingLanguageLinks.TryGetValue(supportedLanguage.Code, out VoiceWorkSupportedLanguage? voiceWorkSupportedLanguage))
{
voiceWorkSupportedLanguage = new VoiceWorkSupportedLanguage
{
VoiceWork = voiceWork,
Language = supportedLanguage.Code
};
dbContext.VoiceWorkSupportedLanguages.Add(voiceWorkSupportedLanguage);
}
}
}
}

View File

@@ -0,0 +1,40 @@
using JSMR.Domain.Entities;
namespace JSMR.Infrastructure.Ingestion;
public record VoiceWorkUpsertContext(
DateTimeOffset CurrentScanAnchor,
DateTimeOffset PreviousScanAnchor,
Dictionary<string, Circle> Circles,
Dictionary<string, VoiceWork> VoiceWorks,
Dictionary<string, Tag> Tags,
Dictionary<string, Creator> Creators,
Dictionary<string, VoiceWorkUpsertResult> Results
);
public class VoiceWorkUpsertResult
{
public int? VoiceWorkId { get; set; }
public ICollection<VoiceWorkUpsertIssue> Issues { get; } = [];
public VoiceWorkUpsertStatus Status { get; set; } = VoiceWorkUpsertStatus.Unchanged;
}
public record VoiceWorkUpsertIssue(
string Message,
VoiceWorkUpsertIssueSeverity Severity
);
public enum VoiceWorkUpsertIssueSeverity
{
Information,
Warning,
Error
}
public enum VoiceWorkUpsertStatus
{
Unchanged,
Inserted,
Updated,
Skipped
}

View File

@@ -129,24 +129,9 @@ public class VoiceWorkUpsertFixture : MariaDbFixture
public VoiceWorkIngest[] GetFirstRoundIngests()
{
DLSiteWork work1 = new()
{
ProductId = "001",
ProductName = "fdsfs",
MakerId = "RG00001",
Maker = "",
ImageUrl = "",
SmallImageUrl = ""
};
VoiceWorkDetails details1 = new()
{
AgeRating = AgeRating.R18,
};
VoiceWorkIngest[] ingests =
[
new(work1, details1)
new()
{
MakerId = "RG00001",
MakerName = "Good Dreams",
@@ -162,7 +147,7 @@ public class VoiceWorkUpsertFixture : MariaDbFixture
HasTrial = true,
HasDLPlay = true
},
new(work1, details1)
new()
{
MakerId = "RG00002",
MakerName = "Sweet Dreams",
@@ -178,7 +163,7 @@ public class VoiceWorkUpsertFixture : MariaDbFixture
HasTrial = true,
HasDLPlay = true
},
new(work1, details1)
new()
{
MakerId = "RG00003",
MakerName = "Nightmare Fuel",

View File

@@ -1,8 +1,7 @@
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Contracts;
using JSMR.Infrastructure.Common.Time;
using JSMR.Infrastructure.Data;
using JSMR.Infrastructure.Data.Repositories.VoiceWorks;
using JSMR.Infrastructure.Ingestion;
using JSMR.Tests.Fixtures;
using NSubstitute;
using Shouldly;
@@ -12,35 +11,37 @@ namespace JSMR.Tests.Integration;
public class VoiceWorkUpsertTests(VoiceWorkUpsertFixture fixture) : IClassFixture<VoiceWorkUpsertFixture>
{
[Fact]
public async Task Filter_None()
public async Task Simple_Upsert()
{
await using AppDbContext context = fixture.CreateDbContext();
ITimeProvider timeProvider = Substitute.For<ITimeProvider>();
timeProvider.Now().Returns(new DateTimeOffset(2025, 10, 1, 0, 0, 0, 0, TimeSpan.FromSeconds(0)));
DLSiteWork work1 = new()
{
ProductId = "001",
ProductName = "fdsfs",
MakerId = "RG00001",
Maker = "",
ImageUrl = "",
SmallImageUrl = ""
};
VoiceWorkDetails details1 = new()
{
AgeRating = Application.Common.AgeRating.R18,
};
VoiceWorkIngest[] ingests =
[
new(work1, details1)
new()
{
MakerId = "RG00001",
MakerName = "Good Dreams",
ProductId = "A Newly Announced Work",
Title = "",
Description = ""
},
new()
{
MakerId = "RG00001",
MakerName = "Sweet Dreams",
ProductId = "",
Title = "",
Description = ""
}
];
VoiceWorkWriter writer = new(context, timeProvider);
await writer.UpsertAsync(ingests, CancellationToken.None);
IClock clock = Substitute.For<IClock>();
clock.UtcNow.Returns(new DateTimeOffset(2025, 10, 1, 0, 0, 0, 0, TimeSpan.FromSeconds(0)));
TokyoTimeProvider timeProvider = new(clock);
VoiceWorkUpdater updater = new(context, timeProvider);
await updater.UpsertAsync(ingests, CancellationToken.None);
context.VoiceWorks.Count().ShouldBe(2);
}

View File

@@ -28,8 +28,8 @@
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0" />
<PackageReference Include="Shouldly" Version="4.3.0" />
<PackageReference Include="Testcontainers" Version="4.8.0" />
<PackageReference Include="Testcontainers.MariaDb" Version="4.8.0" />
<PackageReference Include="Testcontainers" Version="4.8.1" />
<PackageReference Include="Testcontainers.MariaDb" Version="4.8.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
<PrivateAssets>all</PrivateAssets>