Updated scanner/upsert logic.

2025-09-17 09:40:24 -04:00
parent 3b2a0e3491
commit db0c3349a2
14 changed files with 434 additions and 66 deletions
--- a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkSearchUpdater.cs
+++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkSearchUpdater.cs
@@ -0,0 +1,107 @@
+using JSMR.Application.Scanning.Ports;
+using JSMR.Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using System.Text;
+
+namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
+
+public class VoiceWorkSearchUpdater(AppDbContext dbContext) : IVoiceWorkSearchUpdater
+{
+    public async Task UpdateAsync(int[] voiceWorkIds, CancellationToken cancellationToken)
+    {
+        List<VoiceWork> batch = await dbContext.VoiceWorks
+                .Include(vw => vw.Circle)
+                .Include(vw => vw.VoiceWorkTags)
+                    .ThenInclude(vwt => vwt.Tag)
+                .Include(vw => vw.VoiceWorkCreators)
+                    .ThenInclude(vwc => vwc.Creator)
+                .Include(vw => vw.EnglishVoiceWorks)
+                .Where(vw => voiceWorkIds.Contains(vw.VoiceWorkId))
+                .ToListAsync(cancellationToken);
+
+        foreach (var voiceWork in batch)
+        {
+            try
+            {
+                UpdateSearchText(voiceWork);
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"⚠️ Error updating VoiceWorkId {voiceWork.VoiceWorkId}: {ex.Message}");
+            }
+        }
+
+        dbContext.SaveChanges();
+    }
+
+    private void UpdateSearchText(VoiceWork voiceWork)
+    {
+        string searchText = GetSearchText(voiceWork);
+
+        var searchEntry = dbContext.VoiceWorkSearches
+            .FirstOrDefault(s => s.VoiceWorkId == voiceWork.VoiceWorkId);
+
+        if (searchEntry == null)
+        {
+            dbContext.VoiceWorkSearches.Add(new VoiceWorkSearch
+            {
+                VoiceWorkId = voiceWork.VoiceWorkId,
+                SearchText = searchText
+            });
+        }
+        else
+        {
+            searchEntry.SearchText = searchText;
+        }
+    }
+
+    private string GetSearchText(VoiceWork voiceWork)
+    {
+        var english = voiceWork.EnglishVoiceWorks.FirstOrDefault();
+
+        var sb = new StringBuilder();
+
+        AppendRaw(sb, voiceWork.ProductId);
+        AppendRaw(sb, voiceWork.Circle?.MakerId);
+
+        AppendRaw(sb, english?.ProductName);
+        AppendRaw(sb, english?.Description);
+
+        AppendRaw(sb, voiceWork.ProductName);
+        AppendRaw(sb, voiceWork.Description);
+        AppendRaw(sb, voiceWork.Circle?.Name);
+
+        foreach (var tag in voiceWork.VoiceWorkTags.Select(vwt => vwt.Tag))
+        {
+            if (tag is null)
+                continue;
+
+            AppendRaw(sb, tag.Name);
+
+            var englishTag = dbContext.EnglishTags.FirstOrDefault(et => et.TagId == tag.TagId);
+
+            if (englishTag is null)
+                continue;
+
+            AppendRaw(sb, englishTag?.Name);
+        }
+
+        foreach (var creator in voiceWork.VoiceWorkCreators.Select(vwc => vwc.Creator))
+        {
+            if (creator is null)
+                continue;
+
+            AppendRaw(sb, creator?.Name);
+        }
+
+        string searchText = sb.ToString().Trim();
+
+        return searchText;
+    }
+
+    private static void AppendRaw(StringBuilder sb, string? text)
+    {
+        if (!string.IsNullOrWhiteSpace(text))
+            sb.Append(text).Append(' ');
+    }
+}
--- a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkUpsertContext.cs
+++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkUpsertContext.cs
@@ -0,0 +1,10 @@
+using JSMR.Domain.Entities;
+
+namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
+
+public record VoiceWorkUpsertContext(
+    Dictionary<string, Circle> Circles,
+    Dictionary<string, VoiceWork> VoiceWorks,
+    Dictionary<string, Tag> Tags,
+    Dictionary<string, Creator> Creators
+);
--- a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs
+++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs
@@ -1,25 +1,240 @@
-using JSMR.Application.VoiceWorks.Commands.SetFavorite;
+using JSMR.Application.Scanning.Contracts;
+using JSMR.Application.VoiceWorks.Commands.SetFavorite;
 using JSMR.Application.VoiceWorks.Ports;
 using JSMR.Domain.Entities;
 using Microsoft.EntityFrameworkCore;

 namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;

-public class VoiceWorkWriter(AppDbContext context) : IVoiceWorkWriter
+public class VoiceWorkWriter(AppDbContext dbContext) : IVoiceWorkWriter
 {
+    public async Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
+    {
+        VoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken);
+
+        foreach (VoiceWorkIngest ingest in ingests)
+        {
+            Upsert(ingest, upsertContext);
+        }
+
+        await dbContext.SaveChangesAsync(cancellationToken);
+
+        return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)];
+    }
+
+    private async Task<VoiceWorkUpsertContext> CreateUpsertContextAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
+    {
+        string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()];
+        string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()];
+        string[] tagNames = [.. ingests.SelectMany(i => i.Tags).Distinct()];
+        string[] creatorNames = [.. ingests.SelectMany(i => i.Creators).Distinct()];
+
+        VoiceWorkUpsertContext upsertContext = new(
+            Circles: await dbContext.Circles
+                .Where(c => makerIds.Contains(c.MakerId))
+                .ToDictionaryAsync(c => c.MakerId, cancellationToken),
+            VoiceWorks: await dbContext.VoiceWorks
+                .Where(v => productIds.Contains(v.ProductId))
+                .Include(v => v.VoiceWorkCreators)
+                .Include(v => v.VoiceWorkTags)
+                .ToDictionaryAsync(v => v.ProductId, cancellationToken),
+            Tags: await dbContext.Tags
+                .Where(t => tagNames.Contains(t.Name))
+                .ToDictionaryAsync(t => t.Name, cancellationToken),
+            Creators: await dbContext.Creators
+                .Where(cr => creatorNames.Contains(cr.Name))
+                .ToDictionaryAsync(cr => cr.Name, cancellationToken)
+        );
+
+        return upsertContext;
+    }
+
+    private void Upsert(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        UpsertCircle(ingest, upsertContext);
+        UpsertVoiceWork(ingest, upsertContext);
+        UpsertTags(ingest, upsertContext);
+        UpsertVoiceWorkTags(ingest, upsertContext);
+        UpsertCreators(ingest, upsertContext);
+        UpsertVoiceWorkCreators(ingest, upsertContext);
+    }
+
+    private void UpsertCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        Circle circle = GetOrAddCircle(ingest, upsertContext);
+        circle.Name = ingest.MakerName;
+    }
+
+    private Circle GetOrAddCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        if (!upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle))
+        {
+            circle = new Circle
+            {
+                MakerId = ingest.MakerId,
+                Name = ingest.MakerName,
+            };
+
+            dbContext.Circles.Add(circle);
+            upsertContext.Circles[ingest.MakerId] = circle;
+        }
+
+        return circle;
+    }
+
+    private void UpsertVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        VoiceWork voiceWork = GetOrAddVoiceWork(ingest, upsertContext);
+        voiceWork.Circle = upsertContext.Circles[ingest.MakerId];
+        voiceWork.ProductName = ingest.Title;
+        voiceWork.Description = ingest.Description;
+        voiceWork.Downloads = ingest.Downloads;
+        voiceWork.WishlistCount = ingest.WishlistCount;
+        voiceWork.HasTrial = ingest.HasTrial;
+        voiceWork.HasChobit = ingest.HasDLPlay;
+        voiceWork.StarRating = ingest.StarRating;
+        voiceWork.Votes = ingest.Votes;
+
+        voiceWork.IsValid = true;
+
+            //var est = i.EstimatedDate?.ToDateTime(new TimeOnly(0, 0));
+            //if (vw2.ExpectedDate != est) { vw2.ExpectedDate = est; changed = true; }
+
+            //var sales = i.SalesDate?.ToDateTime(new TimeOnly(0, 0));
+            //if (vw2.SalesDate != sales) { vw2.SalesDate = sales; changed = true; }
+    }
+
+    private VoiceWork GetOrAddVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        if (!upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork))
+        {
+            voiceWork = new VoiceWork
+            {
+                ProductId = ingest.ProductId
+            };
+
+            dbContext.VoiceWorks.Add(voiceWork);
+            upsertContext.VoiceWorks[ingest.ProductId] = voiceWork;
+        }
+
+        return voiceWork;
+    }
+
+    private void UpsertTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
+    {
+        foreach (string tagName in ingest.Tags)
+        {
+            GetOrAddTag(tagName, upsertContext);
+        }
+    }
+
+    private Tag GetOrAddTag(string tagName, VoiceWorkUpsertContext upsertContext)
+    {
+        if (!upsertContext.Tags.TryGetValue(tagName, out Tag? tag))
+        {
+            tag = new Tag
+            {
+                Name = tagName
+            };
+
+            dbContext.Tags.Add(tag);
+            upsertContext.Tags[tagName] = tag;
+        }
+
+        return tag;
+    }
+
+    private void UpsertCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
+    {
+        foreach (string creatorName in ingest.Creators)
+        {
+            GetOrAddCreator(creatorName, something);
+        }
+    }
+
+    private Creator GetOrAddCreator(string creatorName, VoiceWorkUpsertContext something)
+    {
+        if (!something.Creators.TryGetValue(creatorName, out Creator? creator))
+        {
+            creator = new Creator
+            {
+                Name = creatorName
+            };
+
+            dbContext.Creators.Add(creator);
+            something.Creators[creatorName] = creator;
+        }
+
+        return creator;
+    }
+
+    private void UpsertVoiceWorkTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
+    {
+        VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId];
+        Dictionary<int, VoiceWorkTag> existingTagLinks = voiceWork.VoiceWorkTags.ToDictionary(x => x.TagId);
+
+        int position = 1;
+
+        foreach (string tagName in ingest.Tags)
+        {
+            Tag tag = something.Tags[tagName];
+
+            if (!existingTagLinks.TryGetValue(tag.TagId, out VoiceWorkTag? voiceWorkTag))
+            {
+                voiceWorkTag = new VoiceWorkTag
+                {
+                    VoiceWork = voiceWork,
+                    Tag = tag
+                };
+
+                dbContext.VoiceWorkTags.Add(voiceWorkTag);
+            }
+            
+            voiceWorkTag.Position = position++;
+            voiceWorkTag.IsValid = true;
+        }
+    }
+
+    private void UpsertVoiceWorkCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
+    {
+        VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId];
+        Dictionary<int, VoiceWorkCreator> existingCreatorLinks = voiceWork.VoiceWorkCreators.ToDictionary(x => x.CreatorId);
+
+        int position = 1;
+
+        foreach (string creatorName in ingest.Creators)
+        {
+            Creator creator = something.Creators[creatorName];
+
+            if (!existingCreatorLinks.TryGetValue(creator.CreatorId, out VoiceWorkCreator? voiceWorkCreator))
+            {
+                voiceWorkCreator = new VoiceWorkCreator
+                {
+                    VoiceWork = voiceWork,
+                    Creator = creator
+                };
+
+                dbContext.VoiceWorkCreators.Add(voiceWorkCreator);
+            }
+
+            voiceWorkCreator.Position = position++;
+            voiceWorkCreator.IsValid = true;
+        }
+    }
+
    public async Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken)
    {
        VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken);
        voiceWork.Favorite = request.IsFavorite;

-        await context.SaveChangesAsync(cancellationToken);
+        await dbContext.SaveChangesAsync(cancellationToken);

        return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite);
    }

    private async Task<VoiceWork> GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken)
    {
-        return await context.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken)
+        return await dbContext.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken)
            ?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found.");
    }
 }
--- a/JSMR.Infrastructure/JSMR.Infrastructure.csproj
+++ b/JSMR.Infrastructure/JSMR.Infrastructure.csproj
@@ -7,7 +7,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.12.2" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.12.3" />
    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.9" />
    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.9" />
    <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.9" />
--- a/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs
+++ b/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs
@@ -5,8 +5,14 @@ using System.Text.RegularExpressions;

 namespace JSMR.Infrastructure.Scanning;

-public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader)
+public partial class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader)
 {
+    [GeneratedRegex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase, "en-US")]
+    private static partial Regex EstimatedDateRegex();
+
+    [GeneratedRegex("販売日:&nbsp;(.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase, "en-US")]
+    private static partial Regex SalesDateRegex();
+
    protected override ILocale Locale => new JapaneseLocale();

    protected override ISupportedLanguage[] SupportedLanguages =>
@@ -24,7 +30,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l
        if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定"))
            return null;

-        Regex textRegex = new Regex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase);
+        Regex textRegex = EstimatedDateRegex();
        MatchCollection textMatches = textRegex.Matches(expectedDate);

        if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
@@ -57,7 +63,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l

    protected override DateOnly? GetSalesDate(string salesDate)
    {
-        Regex textRegex = new Regex("販売日:&nbsp;(.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase);
+        Regex textRegex = SalesDateRegex();
        MatchCollection textMatches = textRegex.Matches(salesDate);

        if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
--- a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs
+++ b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs
@@ -70,17 +70,28 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca

    private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node)
    {
-        DLSiteWork work = new();
-
-        work.ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode);
-        work.ProductUrl = node.ProductLinkNode.Attributes["href"].Value;
-        work.ProductId = ScannerUtilities.GetTextBetween(work.ProductUrl, "product_id/", ".html");
-        work.Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode);
-
+        string productUrl = node.ProductLinkNode.Attributes["href"].Value;
        string makerUrl = node.MakerLinkNode.Attributes["href"].Value;
-        work.MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html");
+        string imageSource = ScannerUtilities.GetImageSource(node.ImageNode);
+        string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif");
+        ScannedRating? rating = GetScannedRating(node.StarRatingNode);

-        work.Description = ScannerUtilities.GetDecodedText(node.DescriptionNode);
+        DLSiteWork work = new()
+        {
+            ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode),
+            Description = ScannerUtilities.GetDecodedText(node.DescriptionNode),
+            ProductId = ScannerUtilities.GetTextBetween(productUrl, "product_id/", ".html"),
+            Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode),
+            MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"),
+            Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes),
+            Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes),
+            Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes),
+            SmallImageUrl = imageSource,
+            ImageUrl = imageUrl,
+            Type = imageUrl.Contains("ana/doujin") ? DLSiteWorkType.Announced : DLSiteWorkType.Released,
+            StarRating = rating?.Score,
+            Votes = rating?.Votes
+        };

        if (node.ExpectedDateNode != null)
        {
@@ -97,25 +108,6 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
            work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands);
        }

-        var rating = GetScannedRating(node.StarRatingNode);
-
-        if (rating != null)
-        {
-            work.StarRating = rating.Score;
-            work.Votes = rating.Votes;
-        }
-
-        work.Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes);
-        work.Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes);
-        work.Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes);
-
-        string imageSource = ScannerUtilities.GetImageSource(node.ImageNode);
-        string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif");
-
-        work.SmallImageUrl = imageSource;
-        work.ImageUrl = imageUrl;
-        work.Type = imageUrl.Contains("ana/doujin") ? "Ana" : "Work";
-
        return work;
    }

@@ -138,9 +130,11 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
        if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2)
            return null;

-        ScannedRating rating = new ScannedRating();
-        rating.Score = Convert.ToByte(ratingClass.Replace("star_", ""));
-        rating.Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands);
+        ScannedRating rating = new()
+        {
+            Score = Convert.ToByte(ratingClass.Replace("star_", "")),
+            Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands)
+        };

        return rating;
    }