diff --git a/JSMR.Application/Scanning/Contracts/DLSiteWork.cs b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs index ffc2fce..64205d1 100644 --- a/JSMR.Application/Scanning/Contracts/DLSiteWork.cs +++ b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs @@ -2,24 +2,23 @@ public class DLSiteWork { - public DLSiteWorkType WorkType { get; set; } + public DLSiteWorkType Type { get; set; } public DLSiteWorkCategory Category { get; set; } - public string? ProductName { get; set; } - public string? ProductUrl { get; set; } - public string? ProductId { get; set; } + public required string ProductName { get; set; } + public required string ProductId { get; set; } public DateOnly? AnnouncedDate { get; set; } public DateOnly? ExpectedDate { get; set; } public DateOnly? SalesDate { get; set; } - public int? Downloads { get; set; } + public int Downloads { get; set; } public byte? StarRating { get; set; } public int? Votes { get; set; } - public string? Maker { get; set; } - public string? MakerId { get; set; } + public required string Maker { get; set; } + public required string MakerId { get; set; } public string? Description { get; set; } public ICollection Genres { get; set; } = []; + public bool HasTrial { get; set;} public ICollection Tags { get; set; } = []; public ICollection Creators { get; set; } = []; public string? ImageUrl { get; set; } public string? SmallImageUrl { get; set; } - public string? Type { get; set; } } \ No newline at end of file diff --git a/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs b/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs new file mode 100644 index 0000000..0b7eab0 --- /dev/null +++ b/JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs @@ -0,0 +1,21 @@ +using JSMR.Application.Integrations.DLSite.Models; + +namespace JSMR.Application.Scanning.Contracts; + +public record VoiceWorkIngest(DLSiteWork Work, VoiceWorkDetails? Details) +{ + public string MakerId { get; init; } = Work.MakerId; + public string MakerName { get; init; } = Work.Maker; + public string ProductId { get; init; } = Work.ProductId; + public string Title { get; init; } = Work.ProductName; + public string? Description { get; init; } = Work.Description; + public ICollection Tags { get; init; } = Work.Tags; + public ICollection Creators { get; init; } = Work.Creators; + public int WishlistCount { get; init; } = Details?.WishlistCount ?? 0; + public int Downloads { get; init; } = Math.Max(Work.Downloads, Details?.DownloadCount ?? 0); + public bool HasTrial { get; init; } = Work.HasTrial || (Details?.HasTrial ?? false); + public bool HasDLPlay { get; init; } = Details?.HasDLPlay ?? false; + public byte? StarRating { get; init; } = Work.StarRating; + public int? Votes { get; init; } = Work.Votes; + // TODO: Other properties +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/Ports/IVoiceWorkSearchUpdater.cs b/JSMR.Application/Scanning/Ports/IVoiceWorkSearchUpdater.cs new file mode 100644 index 0000000..bbaabc9 --- /dev/null +++ b/JSMR.Application/Scanning/Ports/IVoiceWorkSearchUpdater.cs @@ -0,0 +1,6 @@ +namespace JSMR.Application.Scanning.Ports; + +public interface IVoiceWorkSearchUpdater +{ + Task UpdateAsync(int[] voiceWorkIds, CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs index 5f8549e..f72d385 100644 --- a/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs +++ b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs @@ -3,11 +3,17 @@ using JSMR.Application.Integrations.DLSite.Models; using JSMR.Application.Integrations.Ports; using JSMR.Application.Scanning.Contracts; using JSMR.Application.Scanning.Ports; +using JSMR.Application.VoiceWorks.Ports; using Microsoft.Extensions.DependencyInjection; namespace JSMR.Application.Scanning; -public sealed class ScanVoiceWorksHandler(IServiceProvider serviceProvider, IDLSiteClient dlsiteClient, ISpamCircleCache spamCircleCache) +public sealed class ScanVoiceWorksHandler( + IServiceProvider serviceProvider, + IDLSiteClient dlsiteClient, + ISpamCircleCache spamCircleCache, + IVoiceWorkWriter writer, + IVoiceWorkSearchUpdater searchUpdater) { public async Task HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken) { @@ -32,21 +38,14 @@ public sealed class ScanVoiceWorksHandler(IServiceProvider serviceProvider, IDLS string[] productIds = [.. works.Where(x => !string.IsNullOrWhiteSpace(x.ProductId)).Select(x => x.ProductId!)]; VoiceWorkDetailCollection voiceWorkDetails = await dlsiteClient.GetVoiceWorkDetailsAsync(productIds, cancellationToken); - // TODO - - /* - var ingests = works.Select(VoiceWorkIngest.From).ToList(); - var upsert = await _writer.UpsertAsync(ingests, ct); - - // only update search text for affected rows - await _search.UpdateAsync(upsert.AffectedVoiceWorkIds, ct); - - return new ScanVoiceWorksResponse + List ingests = [.. works.Select(work => { - Inserted = upsert.Inserted, - Updated = upsert.Updated - }; - */ + voiceWorkDetails.TryGetValue(work.ProductId!, out VoiceWorkDetails? value); + return new VoiceWorkIngest(work, value); + })]; + + int[] voiceWorkIds = await writer.UpsertAsync(ingests, cancellationToken); + await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken); return new(); } diff --git a/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs b/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs index 50c70dd..612afa6 100644 --- a/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs +++ b/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs @@ -1,8 +1,10 @@ -using JSMR.Application.VoiceWorks.Commands.SetFavorite; +using JSMR.Application.Scanning.Contracts; +using JSMR.Application.VoiceWorks.Commands.SetFavorite; namespace JSMR.Application.VoiceWorks.Ports; public interface IVoiceWorkWriter { + Task UpsertAsync(IReadOnlyCollection ingests, CancellationToken cancellationToken); Task SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken); } \ No newline at end of file diff --git a/JSMR.Domain/Entities/VoiceWork.cs b/JSMR.Domain/Entities/VoiceWork.cs index 8839280..ca78f2f 100644 --- a/JSMR.Domain/Entities/VoiceWork.cs +++ b/JSMR.Domain/Entities/VoiceWork.cs @@ -5,7 +5,7 @@ public class VoiceWork public int VoiceWorkId { get; set; } public required string ProductId { get; set; } public string? OriginalProductId { get; set; } - public required string ProductName { get; set; } + public string ProductName { get; set; } = string.Empty; public string? Description { get; set; } public int Rating { get; set; } public bool HasImage { get; set; } diff --git a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkSearchUpdater.cs b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkSearchUpdater.cs new file mode 100644 index 0000000..0005260 --- /dev/null +++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkSearchUpdater.cs @@ -0,0 +1,107 @@ +using JSMR.Application.Scanning.Ports; +using JSMR.Domain.Entities; +using Microsoft.EntityFrameworkCore; +using System.Text; + +namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; + +public class VoiceWorkSearchUpdater(AppDbContext dbContext) : IVoiceWorkSearchUpdater +{ + public async Task UpdateAsync(int[] voiceWorkIds, CancellationToken cancellationToken) + { + List batch = await dbContext.VoiceWorks + .Include(vw => vw.Circle) + .Include(vw => vw.VoiceWorkTags) + .ThenInclude(vwt => vwt.Tag) + .Include(vw => vw.VoiceWorkCreators) + .ThenInclude(vwc => vwc.Creator) + .Include(vw => vw.EnglishVoiceWorks) + .Where(vw => voiceWorkIds.Contains(vw.VoiceWorkId)) + .ToListAsync(cancellationToken); + + foreach (var voiceWork in batch) + { + try + { + UpdateSearchText(voiceWork); + } + catch (Exception ex) + { + Console.WriteLine($"⚠️ Error updating VoiceWorkId {voiceWork.VoiceWorkId}: {ex.Message}"); + } + } + + dbContext.SaveChanges(); + } + + private void UpdateSearchText(VoiceWork voiceWork) + { + string searchText = GetSearchText(voiceWork); + + var searchEntry = dbContext.VoiceWorkSearches + .FirstOrDefault(s => s.VoiceWorkId == voiceWork.VoiceWorkId); + + if (searchEntry == null) + { + dbContext.VoiceWorkSearches.Add(new VoiceWorkSearch + { + VoiceWorkId = voiceWork.VoiceWorkId, + SearchText = searchText + }); + } + else + { + searchEntry.SearchText = searchText; + } + } + + private string GetSearchText(VoiceWork voiceWork) + { + var english = voiceWork.EnglishVoiceWorks.FirstOrDefault(); + + var sb = new StringBuilder(); + + AppendRaw(sb, voiceWork.ProductId); + AppendRaw(sb, voiceWork.Circle?.MakerId); + + AppendRaw(sb, english?.ProductName); + AppendRaw(sb, english?.Description); + + AppendRaw(sb, voiceWork.ProductName); + AppendRaw(sb, voiceWork.Description); + AppendRaw(sb, voiceWork.Circle?.Name); + + foreach (var tag in voiceWork.VoiceWorkTags.Select(vwt => vwt.Tag)) + { + if (tag is null) + continue; + + AppendRaw(sb, tag.Name); + + var englishTag = dbContext.EnglishTags.FirstOrDefault(et => et.TagId == tag.TagId); + + if (englishTag is null) + continue; + + AppendRaw(sb, englishTag?.Name); + } + + foreach (var creator in voiceWork.VoiceWorkCreators.Select(vwc => vwc.Creator)) + { + if (creator is null) + continue; + + AppendRaw(sb, creator?.Name); + } + + string searchText = sb.ToString().Trim(); + + return searchText; + } + + private static void AppendRaw(StringBuilder sb, string? text) + { + if (!string.IsNullOrWhiteSpace(text)) + sb.Append(text).Append(' '); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkUpsertContext.cs b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkUpsertContext.cs new file mode 100644 index 0000000..d8d9f6e --- /dev/null +++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkUpsertContext.cs @@ -0,0 +1,10 @@ +using JSMR.Domain.Entities; + +namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; + +public record VoiceWorkUpsertContext( + Dictionary Circles, + Dictionary VoiceWorks, + Dictionary Tags, + Dictionary Creators +); \ No newline at end of file diff --git a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs index 32eaa3b..71bd115 100644 --- a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs +++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs @@ -1,25 +1,240 @@ -using JSMR.Application.VoiceWorks.Commands.SetFavorite; +using JSMR.Application.Scanning.Contracts; +using JSMR.Application.VoiceWorks.Commands.SetFavorite; using JSMR.Application.VoiceWorks.Ports; using JSMR.Domain.Entities; using Microsoft.EntityFrameworkCore; namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; -public class VoiceWorkWriter(AppDbContext context) : IVoiceWorkWriter +public class VoiceWorkWriter(AppDbContext dbContext) : IVoiceWorkWriter { + public async Task UpsertAsync(IReadOnlyCollection ingests, CancellationToken cancellationToken) + { + VoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken); + + foreach (VoiceWorkIngest ingest in ingests) + { + Upsert(ingest, upsertContext); + } + + await dbContext.SaveChangesAsync(cancellationToken); + + return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)]; + } + + private async Task CreateUpsertContextAsync(IReadOnlyCollection ingests, CancellationToken cancellationToken) + { + string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()]; + string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()]; + string[] tagNames = [.. ingests.SelectMany(i => i.Tags).Distinct()]; + string[] creatorNames = [.. ingests.SelectMany(i => i.Creators).Distinct()]; + + VoiceWorkUpsertContext upsertContext = new( + Circles: await dbContext.Circles + .Where(c => makerIds.Contains(c.MakerId)) + .ToDictionaryAsync(c => c.MakerId, cancellationToken), + VoiceWorks: await dbContext.VoiceWorks + .Where(v => productIds.Contains(v.ProductId)) + .Include(v => v.VoiceWorkCreators) + .Include(v => v.VoiceWorkTags) + .ToDictionaryAsync(v => v.ProductId, cancellationToken), + Tags: await dbContext.Tags + .Where(t => tagNames.Contains(t.Name)) + .ToDictionaryAsync(t => t.Name, cancellationToken), + Creators: await dbContext.Creators + .Where(cr => creatorNames.Contains(cr.Name)) + .ToDictionaryAsync(cr => cr.Name, cancellationToken) + ); + + return upsertContext; + } + + private void Upsert(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + UpsertCircle(ingest, upsertContext); + UpsertVoiceWork(ingest, upsertContext); + UpsertTags(ingest, upsertContext); + UpsertVoiceWorkTags(ingest, upsertContext); + UpsertCreators(ingest, upsertContext); + UpsertVoiceWorkCreators(ingest, upsertContext); + } + + private void UpsertCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + Circle circle = GetOrAddCircle(ingest, upsertContext); + circle.Name = ingest.MakerName; + } + + private Circle GetOrAddCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + if (!upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle)) + { + circle = new Circle + { + MakerId = ingest.MakerId, + Name = ingest.MakerName, + }; + + dbContext.Circles.Add(circle); + upsertContext.Circles[ingest.MakerId] = circle; + } + + return circle; + } + + private void UpsertVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + VoiceWork voiceWork = GetOrAddVoiceWork(ingest, upsertContext); + voiceWork.Circle = upsertContext.Circles[ingest.MakerId]; + voiceWork.ProductName = ingest.Title; + voiceWork.Description = ingest.Description; + voiceWork.Downloads = ingest.Downloads; + voiceWork.WishlistCount = ingest.WishlistCount; + voiceWork.HasTrial = ingest.HasTrial; + voiceWork.HasChobit = ingest.HasDLPlay; + voiceWork.StarRating = ingest.StarRating; + voiceWork.Votes = ingest.Votes; + + voiceWork.IsValid = true; + + //var est = i.EstimatedDate?.ToDateTime(new TimeOnly(0, 0)); + //if (vw2.ExpectedDate != est) { vw2.ExpectedDate = est; changed = true; } + + //var sales = i.SalesDate?.ToDateTime(new TimeOnly(0, 0)); + //if (vw2.SalesDate != sales) { vw2.SalesDate = sales; changed = true; } + } + + private VoiceWork GetOrAddVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + if (!upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork)) + { + voiceWork = new VoiceWork + { + ProductId = ingest.ProductId + }; + + dbContext.VoiceWorks.Add(voiceWork); + upsertContext.VoiceWorks[ingest.ProductId] = voiceWork; + } + + return voiceWork; + } + + private void UpsertTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext) + { + foreach (string tagName in ingest.Tags) + { + GetOrAddTag(tagName, upsertContext); + } + } + + private Tag GetOrAddTag(string tagName, VoiceWorkUpsertContext upsertContext) + { + if (!upsertContext.Tags.TryGetValue(tagName, out Tag? tag)) + { + tag = new Tag + { + Name = tagName + }; + + dbContext.Tags.Add(tag); + upsertContext.Tags[tagName] = tag; + } + + return tag; + } + + private void UpsertCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something) + { + foreach (string creatorName in ingest.Creators) + { + GetOrAddCreator(creatorName, something); + } + } + + private Creator GetOrAddCreator(string creatorName, VoiceWorkUpsertContext something) + { + if (!something.Creators.TryGetValue(creatorName, out Creator? creator)) + { + creator = new Creator + { + Name = creatorName + }; + + dbContext.Creators.Add(creator); + something.Creators[creatorName] = creator; + } + + return creator; + } + + private void UpsertVoiceWorkTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext something) + { + VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId]; + Dictionary existingTagLinks = voiceWork.VoiceWorkTags.ToDictionary(x => x.TagId); + + int position = 1; + + foreach (string tagName in ingest.Tags) + { + Tag tag = something.Tags[tagName]; + + if (!existingTagLinks.TryGetValue(tag.TagId, out VoiceWorkTag? voiceWorkTag)) + { + voiceWorkTag = new VoiceWorkTag + { + VoiceWork = voiceWork, + Tag = tag + }; + + dbContext.VoiceWorkTags.Add(voiceWorkTag); + } + + voiceWorkTag.Position = position++; + voiceWorkTag.IsValid = true; + } + } + + private void UpsertVoiceWorkCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something) + { + VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId]; + Dictionary existingCreatorLinks = voiceWork.VoiceWorkCreators.ToDictionary(x => x.CreatorId); + + int position = 1; + + foreach (string creatorName in ingest.Creators) + { + Creator creator = something.Creators[creatorName]; + + if (!existingCreatorLinks.TryGetValue(creator.CreatorId, out VoiceWorkCreator? voiceWorkCreator)) + { + voiceWorkCreator = new VoiceWorkCreator + { + VoiceWork = voiceWork, + Creator = creator + }; + + dbContext.VoiceWorkCreators.Add(voiceWorkCreator); + } + + voiceWorkCreator.Position = position++; + voiceWorkCreator.IsValid = true; + } + } + public async Task SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken) { VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken); voiceWork.Favorite = request.IsFavorite; - await context.SaveChangesAsync(cancellationToken); + await dbContext.SaveChangesAsync(cancellationToken); return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite); } private async Task GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken) { - return await context.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken) + return await dbContext.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken) ?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found."); } } \ No newline at end of file diff --git a/JSMR.Infrastructure/JSMR.Infrastructure.csproj b/JSMR.Infrastructure/JSMR.Infrastructure.csproj index 3effbb7..ea94910 100644 --- a/JSMR.Infrastructure/JSMR.Infrastructure.csproj +++ b/JSMR.Infrastructure/JSMR.Infrastructure.csproj @@ -7,7 +7,7 @@ - + diff --git a/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs index 22c341f..1fc8d56 100644 --- a/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs +++ b/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs @@ -5,8 +5,14 @@ using System.Text.RegularExpressions; namespace JSMR.Infrastructure.Scanning; -public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader) +public partial class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader) { + [GeneratedRegex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase, "en-US")] + private static partial Regex EstimatedDateRegex(); + + [GeneratedRegex("販売日: (.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase, "en-US")] + private static partial Regex SalesDateRegex(); + protected override ILocale Locale => new JapaneseLocale(); protected override ISupportedLanguage[] SupportedLanguages => @@ -24,7 +30,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定")) return null; - Regex textRegex = new Regex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase); + Regex textRegex = EstimatedDateRegex(); MatchCollection textMatches = textRegex.Matches(expectedDate); if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) @@ -57,7 +63,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l protected override DateOnly? GetSalesDate(string salesDate) { - Regex textRegex = new Regex("販売日: (.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase); + Regex textRegex = SalesDateRegex(); MatchCollection textMatches = textRegex.Matches(salesDate); if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) diff --git a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs index 7a07d29..e282355 100644 --- a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs +++ b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs @@ -70,17 +70,28 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node) { - DLSiteWork work = new(); - - work.ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode); - work.ProductUrl = node.ProductLinkNode.Attributes["href"].Value; - work.ProductId = ScannerUtilities.GetTextBetween(work.ProductUrl, "product_id/", ".html"); - work.Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode); - + string productUrl = node.ProductLinkNode.Attributes["href"].Value; string makerUrl = node.MakerLinkNode.Attributes["href"].Value; - work.MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"); + string imageSource = ScannerUtilities.GetImageSource(node.ImageNode); + string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif"); + ScannedRating? rating = GetScannedRating(node.StarRatingNode); - work.Description = ScannerUtilities.GetDecodedText(node.DescriptionNode); + DLSiteWork work = new() + { + ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode), + Description = ScannerUtilities.GetDecodedText(node.DescriptionNode), + ProductId = ScannerUtilities.GetTextBetween(productUrl, "product_id/", ".html"), + Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode), + MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"), + Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes), + Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes), + Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes), + SmallImageUrl = imageSource, + ImageUrl = imageUrl, + Type = imageUrl.Contains("ana/doujin") ? DLSiteWorkType.Announced : DLSiteWorkType.Released, + StarRating = rating?.Score, + Votes = rating?.Votes + }; if (node.ExpectedDateNode != null) { @@ -97,25 +108,6 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands); } - var rating = GetScannedRating(node.StarRatingNode); - - if (rating != null) - { - work.StarRating = rating.Score; - work.Votes = rating.Votes; - } - - work.Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes); - work.Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes); - work.Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes); - - string imageSource = ScannerUtilities.GetImageSource(node.ImageNode); - string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif"); - - work.SmallImageUrl = imageSource; - work.ImageUrl = imageUrl; - work.Type = imageUrl.Contains("ana/doujin") ? "Ana" : "Work"; - return work; } @@ -138,9 +130,11 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2) return null; - ScannedRating rating = new ScannedRating(); - rating.Score = Convert.ToByte(ratingClass.Replace("star_", "")); - rating.Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands); + ScannedRating rating = new() + { + Score = Convert.ToByte(ratingClass.Replace("star_", "")), + Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands) + }; return rating; } diff --git a/JSMR.Tests/Scanning/English-Page.html b/JSMR.Tests/Scanning/English-Page.html index b3fcccc..d1b6cf0 100644 --- a/JSMR.Tests/Scanning/English-Page.html +++ b/JSMR.Tests/Scanning/English-Page.html @@ -65,6 +65,8 @@
The Maker + / + Some Creator
@@ -136,9 +138,9 @@
- Title of Product + Title of Product
- Title of Product + Title of Product
Voice / ASMR
diff --git a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs index 8e97640..d2f7ac7 100644 --- a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs +++ b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs @@ -44,10 +44,17 @@ public class VoiceWorkScannerTests result[0].ProductId.ShouldBe("RJ00000001"); result[0].ProductName.ShouldBe("Title of Product"); result[0].Description.ShouldBe("Description of the product."); + result[0].Maker.ShouldBe("The Maker"); + result[0].MakerId.ShouldBe("RG00001"); + result[0].Creators.ShouldBe(["Some Creator"]); + result[0].Genres.ShouldBe(["Voice", "Trial version"]); + result[0].Tags.ShouldBe(["Male Protagonist", "Gal", "Uniform", "Harem", "Big Breasts", "Tanned Skin / Suntan"]); + result[0].Type.ShouldBe(DLSiteWorkType.Released); result[0].Downloads.ShouldBe(1000); result[1].ExpectedDate.ShouldBe(new DateOnly(2025, 10, 11)); result[1].SalesDate.ShouldBeNull(); result[1].ProductId.ShouldBe("RJ00000002"); + result[1].Type.ShouldBe(DLSiteWorkType.Announced); } } \ No newline at end of file