Updated scanner/upsert logic.

This commit is contained in:
2025-09-17 09:40:24 -04:00
parent 3b2a0e3491
commit db0c3349a2
14 changed files with 434 additions and 66 deletions

View File

@@ -2,24 +2,23 @@
public class DLSiteWork public class DLSiteWork
{ {
public DLSiteWorkType WorkType { get; set; } public DLSiteWorkType Type { get; set; }
public DLSiteWorkCategory Category { get; set; } public DLSiteWorkCategory Category { get; set; }
public string? ProductName { get; set; } public required string ProductName { get; set; }
public string? ProductUrl { get; set; } public required string ProductId { get; set; }
public string? ProductId { get; set; }
public DateOnly? AnnouncedDate { get; set; } public DateOnly? AnnouncedDate { get; set; }
public DateOnly? ExpectedDate { get; set; } public DateOnly? ExpectedDate { get; set; }
public DateOnly? SalesDate { get; set; } public DateOnly? SalesDate { get; set; }
public int? Downloads { get; set; } public int Downloads { get; set; }
public byte? StarRating { get; set; } public byte? StarRating { get; set; }
public int? Votes { get; set; } public int? Votes { get; set; }
public string? Maker { get; set; } public required string Maker { get; set; }
public string? MakerId { get; set; } public required string MakerId { get; set; }
public string? Description { get; set; } public string? Description { get; set; }
public ICollection<string> Genres { get; set; } = []; public ICollection<string> Genres { get; set; } = [];
public bool HasTrial { get; set;}
public ICollection<string> Tags { get; set; } = []; public ICollection<string> Tags { get; set; } = [];
public ICollection<string> Creators { get; set; } = []; public ICollection<string> Creators { get; set; } = [];
public string? ImageUrl { get; set; } public string? ImageUrl { get; set; }
public string? SmallImageUrl { get; set; } public string? SmallImageUrl { get; set; }
public string? Type { get; set; }
} }

View File

@@ -0,0 +1,21 @@
using JSMR.Application.Integrations.DLSite.Models;
namespace JSMR.Application.Scanning.Contracts;
public record VoiceWorkIngest(DLSiteWork Work, VoiceWorkDetails? Details)
{
public string MakerId { get; init; } = Work.MakerId;
public string MakerName { get; init; } = Work.Maker;
public string ProductId { get; init; } = Work.ProductId;
public string Title { get; init; } = Work.ProductName;
public string? Description { get; init; } = Work.Description;
public ICollection<string> Tags { get; init; } = Work.Tags;
public ICollection<string> Creators { get; init; } = Work.Creators;
public int WishlistCount { get; init; } = Details?.WishlistCount ?? 0;
public int Downloads { get; init; } = Math.Max(Work.Downloads, Details?.DownloadCount ?? 0);
public bool HasTrial { get; init; } = Work.HasTrial || (Details?.HasTrial ?? false);
public bool HasDLPlay { get; init; } = Details?.HasDLPlay ?? false;
public byte? StarRating { get; init; } = Work.StarRating;
public int? Votes { get; init; } = Work.Votes;
// TODO: Other properties
}

View File

@@ -0,0 +1,6 @@
namespace JSMR.Application.Scanning.Ports;
public interface IVoiceWorkSearchUpdater
{
Task UpdateAsync(int[] voiceWorkIds, CancellationToken cancellationToken);
}

View File

@@ -3,11 +3,17 @@ using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Ports; using JSMR.Application.Integrations.Ports;
using JSMR.Application.Scanning.Contracts; using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports; using JSMR.Application.Scanning.Ports;
using JSMR.Application.VoiceWorks.Ports;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
namespace JSMR.Application.Scanning; namespace JSMR.Application.Scanning;
public sealed class ScanVoiceWorksHandler(IServiceProvider serviceProvider, IDLSiteClient dlsiteClient, ISpamCircleCache spamCircleCache) public sealed class ScanVoiceWorksHandler(
IServiceProvider serviceProvider,
IDLSiteClient dlsiteClient,
ISpamCircleCache spamCircleCache,
IVoiceWorkWriter writer,
IVoiceWorkSearchUpdater searchUpdater)
{ {
public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken) public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
{ {
@@ -32,21 +38,14 @@ public sealed class ScanVoiceWorksHandler(IServiceProvider serviceProvider, IDLS
string[] productIds = [.. works.Where(x => !string.IsNullOrWhiteSpace(x.ProductId)).Select(x => x.ProductId!)]; string[] productIds = [.. works.Where(x => !string.IsNullOrWhiteSpace(x.ProductId)).Select(x => x.ProductId!)];
VoiceWorkDetailCollection voiceWorkDetails = await dlsiteClient.GetVoiceWorkDetailsAsync(productIds, cancellationToken); VoiceWorkDetailCollection voiceWorkDetails = await dlsiteClient.GetVoiceWorkDetailsAsync(productIds, cancellationToken);
// TODO List<VoiceWorkIngest> ingests = [.. works.Select(work =>
/*
var ingests = works.Select(VoiceWorkIngest.From).ToList();
var upsert = await _writer.UpsertAsync(ingests, ct);
// only update search text for affected rows
await _search.UpdateAsync(upsert.AffectedVoiceWorkIds, ct);
return new ScanVoiceWorksResponse
{ {
Inserted = upsert.Inserted, voiceWorkDetails.TryGetValue(work.ProductId!, out VoiceWorkDetails? value);
Updated = upsert.Updated return new VoiceWorkIngest(work, value);
}; })];
*/
int[] voiceWorkIds = await writer.UpsertAsync(ingests, cancellationToken);
await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
return new(); return new();
} }

View File

@@ -1,8 +1,10 @@
using JSMR.Application.VoiceWorks.Commands.SetFavorite; using JSMR.Application.Scanning.Contracts;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
namespace JSMR.Application.VoiceWorks.Ports; namespace JSMR.Application.VoiceWorks.Ports;
public interface IVoiceWorkWriter public interface IVoiceWorkWriter
{ {
Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken);
Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken); Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken);
} }

View File

@@ -5,7 +5,7 @@ public class VoiceWork
public int VoiceWorkId { get; set; } public int VoiceWorkId { get; set; }
public required string ProductId { get; set; } public required string ProductId { get; set; }
public string? OriginalProductId { get; set; } public string? OriginalProductId { get; set; }
public required string ProductName { get; set; } public string ProductName { get; set; } = string.Empty;
public string? Description { get; set; } public string? Description { get; set; }
public int Rating { get; set; } public int Rating { get; set; }
public bool HasImage { get; set; } public bool HasImage { get; set; }

View File

@@ -0,0 +1,107 @@
using JSMR.Application.Scanning.Ports;
using JSMR.Domain.Entities;
using Microsoft.EntityFrameworkCore;
using System.Text;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public class VoiceWorkSearchUpdater(AppDbContext dbContext) : IVoiceWorkSearchUpdater
{
public async Task UpdateAsync(int[] voiceWorkIds, CancellationToken cancellationToken)
{
List<VoiceWork> batch = await dbContext.VoiceWorks
.Include(vw => vw.Circle)
.Include(vw => vw.VoiceWorkTags)
.ThenInclude(vwt => vwt.Tag)
.Include(vw => vw.VoiceWorkCreators)
.ThenInclude(vwc => vwc.Creator)
.Include(vw => vw.EnglishVoiceWorks)
.Where(vw => voiceWorkIds.Contains(vw.VoiceWorkId))
.ToListAsync(cancellationToken);
foreach (var voiceWork in batch)
{
try
{
UpdateSearchText(voiceWork);
}
catch (Exception ex)
{
Console.WriteLine($"⚠️ Error updating VoiceWorkId {voiceWork.VoiceWorkId}: {ex.Message}");
}
}
dbContext.SaveChanges();
}
private void UpdateSearchText(VoiceWork voiceWork)
{
string searchText = GetSearchText(voiceWork);
var searchEntry = dbContext.VoiceWorkSearches
.FirstOrDefault(s => s.VoiceWorkId == voiceWork.VoiceWorkId);
if (searchEntry == null)
{
dbContext.VoiceWorkSearches.Add(new VoiceWorkSearch
{
VoiceWorkId = voiceWork.VoiceWorkId,
SearchText = searchText
});
}
else
{
searchEntry.SearchText = searchText;
}
}
private string GetSearchText(VoiceWork voiceWork)
{
var english = voiceWork.EnglishVoiceWorks.FirstOrDefault();
var sb = new StringBuilder();
AppendRaw(sb, voiceWork.ProductId);
AppendRaw(sb, voiceWork.Circle?.MakerId);
AppendRaw(sb, english?.ProductName);
AppendRaw(sb, english?.Description);
AppendRaw(sb, voiceWork.ProductName);
AppendRaw(sb, voiceWork.Description);
AppendRaw(sb, voiceWork.Circle?.Name);
foreach (var tag in voiceWork.VoiceWorkTags.Select(vwt => vwt.Tag))
{
if (tag is null)
continue;
AppendRaw(sb, tag.Name);
var englishTag = dbContext.EnglishTags.FirstOrDefault(et => et.TagId == tag.TagId);
if (englishTag is null)
continue;
AppendRaw(sb, englishTag?.Name);
}
foreach (var creator in voiceWork.VoiceWorkCreators.Select(vwc => vwc.Creator))
{
if (creator is null)
continue;
AppendRaw(sb, creator?.Name);
}
string searchText = sb.ToString().Trim();
return searchText;
}
private static void AppendRaw(StringBuilder sb, string? text)
{
if (!string.IsNullOrWhiteSpace(text))
sb.Append(text).Append(' ');
}
}

View File

@@ -0,0 +1,10 @@
using JSMR.Domain.Entities;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public record VoiceWorkUpsertContext(
Dictionary<string, Circle> Circles,
Dictionary<string, VoiceWork> VoiceWorks,
Dictionary<string, Tag> Tags,
Dictionary<string, Creator> Creators
);

View File

@@ -1,25 +1,240 @@
using JSMR.Application.VoiceWorks.Commands.SetFavorite; using JSMR.Application.Scanning.Contracts;
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
using JSMR.Application.VoiceWorks.Ports; using JSMR.Application.VoiceWorks.Ports;
using JSMR.Domain.Entities; using JSMR.Domain.Entities;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public class VoiceWorkWriter(AppDbContext context) : IVoiceWorkWriter public class VoiceWorkWriter(AppDbContext dbContext) : IVoiceWorkWriter
{ {
public async Task<int[]> UpsertAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
VoiceWorkUpsertContext upsertContext = await CreateUpsertContextAsync(ingests, cancellationToken);
foreach (VoiceWorkIngest ingest in ingests)
{
Upsert(ingest, upsertContext);
}
await dbContext.SaveChangesAsync(cancellationToken);
return [.. upsertContext.VoiceWorks.Select(x => x.Value.VoiceWorkId)];
}
private async Task<VoiceWorkUpsertContext> CreateUpsertContextAsync(IReadOnlyCollection<VoiceWorkIngest> ingests, CancellationToken cancellationToken)
{
string[] makerIds = [.. ingests.Select(i => i.MakerId).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct()];
string[] productIds = [.. ingests.Select(i => i.ProductId).Distinct()];
string[] tagNames = [.. ingests.SelectMany(i => i.Tags).Distinct()];
string[] creatorNames = [.. ingests.SelectMany(i => i.Creators).Distinct()];
VoiceWorkUpsertContext upsertContext = new(
Circles: await dbContext.Circles
.Where(c => makerIds.Contains(c.MakerId))
.ToDictionaryAsync(c => c.MakerId, cancellationToken),
VoiceWorks: await dbContext.VoiceWorks
.Where(v => productIds.Contains(v.ProductId))
.Include(v => v.VoiceWorkCreators)
.Include(v => v.VoiceWorkTags)
.ToDictionaryAsync(v => v.ProductId, cancellationToken),
Tags: await dbContext.Tags
.Where(t => tagNames.Contains(t.Name))
.ToDictionaryAsync(t => t.Name, cancellationToken),
Creators: await dbContext.Creators
.Where(cr => creatorNames.Contains(cr.Name))
.ToDictionaryAsync(cr => cr.Name, cancellationToken)
);
return upsertContext;
}
private void Upsert(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
UpsertCircle(ingest, upsertContext);
UpsertVoiceWork(ingest, upsertContext);
UpsertTags(ingest, upsertContext);
UpsertVoiceWorkTags(ingest, upsertContext);
UpsertCreators(ingest, upsertContext);
UpsertVoiceWorkCreators(ingest, upsertContext);
}
private void UpsertCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
Circle circle = GetOrAddCircle(ingest, upsertContext);
circle.Name = ingest.MakerName;
}
private Circle GetOrAddCircle(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Circles.TryGetValue(ingest.MakerId, out Circle? circle))
{
circle = new Circle
{
MakerId = ingest.MakerId,
Name = ingest.MakerName,
};
dbContext.Circles.Add(circle);
upsertContext.Circles[ingest.MakerId] = circle;
}
return circle;
}
private void UpsertVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
VoiceWork voiceWork = GetOrAddVoiceWork(ingest, upsertContext);
voiceWork.Circle = upsertContext.Circles[ingest.MakerId];
voiceWork.ProductName = ingest.Title;
voiceWork.Description = ingest.Description;
voiceWork.Downloads = ingest.Downloads;
voiceWork.WishlistCount = ingest.WishlistCount;
voiceWork.HasTrial = ingest.HasTrial;
voiceWork.HasChobit = ingest.HasDLPlay;
voiceWork.StarRating = ingest.StarRating;
voiceWork.Votes = ingest.Votes;
voiceWork.IsValid = true;
//var est = i.EstimatedDate?.ToDateTime(new TimeOnly(0, 0));
//if (vw2.ExpectedDate != est) { vw2.ExpectedDate = est; changed = true; }
//var sales = i.SalesDate?.ToDateTime(new TimeOnly(0, 0));
//if (vw2.SalesDate != sales) { vw2.SalesDate = sales; changed = true; }
}
private VoiceWork GetOrAddVoiceWork(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.VoiceWorks.TryGetValue(ingest.ProductId, out VoiceWork? voiceWork))
{
voiceWork = new VoiceWork
{
ProductId = ingest.ProductId
};
dbContext.VoiceWorks.Add(voiceWork);
upsertContext.VoiceWorks[ingest.ProductId] = voiceWork;
}
return voiceWork;
}
private void UpsertTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext upsertContext)
{
foreach (string tagName in ingest.Tags)
{
GetOrAddTag(tagName, upsertContext);
}
}
private Tag GetOrAddTag(string tagName, VoiceWorkUpsertContext upsertContext)
{
if (!upsertContext.Tags.TryGetValue(tagName, out Tag? tag))
{
tag = new Tag
{
Name = tagName
};
dbContext.Tags.Add(tag);
upsertContext.Tags[tagName] = tag;
}
return tag;
}
private void UpsertCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
{
foreach (string creatorName in ingest.Creators)
{
GetOrAddCreator(creatorName, something);
}
}
private Creator GetOrAddCreator(string creatorName, VoiceWorkUpsertContext something)
{
if (!something.Creators.TryGetValue(creatorName, out Creator? creator))
{
creator = new Creator
{
Name = creatorName
};
dbContext.Creators.Add(creator);
something.Creators[creatorName] = creator;
}
return creator;
}
private void UpsertVoiceWorkTags(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
{
VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkTag> existingTagLinks = voiceWork.VoiceWorkTags.ToDictionary(x => x.TagId);
int position = 1;
foreach (string tagName in ingest.Tags)
{
Tag tag = something.Tags[tagName];
if (!existingTagLinks.TryGetValue(tag.TagId, out VoiceWorkTag? voiceWorkTag))
{
voiceWorkTag = new VoiceWorkTag
{
VoiceWork = voiceWork,
Tag = tag
};
dbContext.VoiceWorkTags.Add(voiceWorkTag);
}
voiceWorkTag.Position = position++;
voiceWorkTag.IsValid = true;
}
}
private void UpsertVoiceWorkCreators(VoiceWorkIngest ingest, VoiceWorkUpsertContext something)
{
VoiceWork voiceWork = something.VoiceWorks[ingest.ProductId];
Dictionary<int, VoiceWorkCreator> existingCreatorLinks = voiceWork.VoiceWorkCreators.ToDictionary(x => x.CreatorId);
int position = 1;
foreach (string creatorName in ingest.Creators)
{
Creator creator = something.Creators[creatorName];
if (!existingCreatorLinks.TryGetValue(creator.CreatorId, out VoiceWorkCreator? voiceWorkCreator))
{
voiceWorkCreator = new VoiceWorkCreator
{
VoiceWork = voiceWork,
Creator = creator
};
dbContext.VoiceWorkCreators.Add(voiceWorkCreator);
}
voiceWorkCreator.Position = position++;
voiceWorkCreator.IsValid = true;
}
}
public async Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken) public async Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken)
{ {
VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken); VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken);
voiceWork.Favorite = request.IsFavorite; voiceWork.Favorite = request.IsFavorite;
await context.SaveChangesAsync(cancellationToken); await dbContext.SaveChangesAsync(cancellationToken);
return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite); return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite);
} }
private async Task<VoiceWork> GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken) private async Task<VoiceWork> GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken)
{ {
return await context.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken) return await dbContext.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken)
?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found."); ?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found.");
} }
} }

View File

@@ -7,7 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.12.2" /> <PackageReference Include="HtmlAgilityPack" Version="1.12.3" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.9" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.9" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.9" /> <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.9" />

View File

@@ -5,8 +5,14 @@ using System.Text.RegularExpressions;
namespace JSMR.Infrastructure.Scanning; namespace JSMR.Infrastructure.Scanning;
public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader) public partial class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(loader)
{ {
[GeneratedRegex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase, "en-US")]
private static partial Regex EstimatedDateRegex();
[GeneratedRegex("販売日:&nbsp;(.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase, "en-US")]
private static partial Regex SalesDateRegex();
protected override ILocale Locale => new JapaneseLocale(); protected override ILocale Locale => new JapaneseLocale();
protected override ISupportedLanguage[] SupportedLanguages => protected override ISupportedLanguage[] SupportedLanguages =>
@@ -24,7 +30,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l
if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定")) if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定"))
return null; return null;
Regex textRegex = new Regex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase); Regex textRegex = EstimatedDateRegex();
MatchCollection textMatches = textRegex.Matches(expectedDate); MatchCollection textMatches = textRegex.Matches(expectedDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
@@ -57,7 +63,7 @@ public class JapaneseVoiceWorksScanner(IHtmlLoader loader) : VoiceWorksScanner(l
protected override DateOnly? GetSalesDate(string salesDate) protected override DateOnly? GetSalesDate(string salesDate)
{ {
Regex textRegex = new Regex("販売日:&nbsp;(.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase); Regex textRegex = SalesDateRegex();
MatchCollection textMatches = textRegex.Matches(salesDate); MatchCollection textMatches = textRegex.Matches(salesDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)

View File

@@ -70,17 +70,28 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node) private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node)
{ {
DLSiteWork work = new(); string productUrl = node.ProductLinkNode.Attributes["href"].Value;
work.ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode);
work.ProductUrl = node.ProductLinkNode.Attributes["href"].Value;
work.ProductId = ScannerUtilities.GetTextBetween(work.ProductUrl, "product_id/", ".html");
work.Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode);
string makerUrl = node.MakerLinkNode.Attributes["href"].Value; string makerUrl = node.MakerLinkNode.Attributes["href"].Value;
work.MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"); string imageSource = ScannerUtilities.GetImageSource(node.ImageNode);
string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif");
ScannedRating? rating = GetScannedRating(node.StarRatingNode);
work.Description = ScannerUtilities.GetDecodedText(node.DescriptionNode); DLSiteWork work = new()
{
ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode),
Description = ScannerUtilities.GetDecodedText(node.DescriptionNode),
ProductId = ScannerUtilities.GetTextBetween(productUrl, "product_id/", ".html"),
Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode),
MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"),
Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes),
Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes),
Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes),
SmallImageUrl = imageSource,
ImageUrl = imageUrl,
Type = imageUrl.Contains("ana/doujin") ? DLSiteWorkType.Announced : DLSiteWorkType.Released,
StarRating = rating?.Score,
Votes = rating?.Votes
};
if (node.ExpectedDateNode != null) if (node.ExpectedDateNode != null)
{ {
@@ -97,25 +108,6 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands); work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands);
} }
var rating = GetScannedRating(node.StarRatingNode);
if (rating != null)
{
work.StarRating = rating.Score;
work.Votes = rating.Votes;
}
work.Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes);
work.Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes);
work.Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes);
string imageSource = ScannerUtilities.GetImageSource(node.ImageNode);
string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif");
work.SmallImageUrl = imageSource;
work.ImageUrl = imageUrl;
work.Type = imageUrl.Contains("ana/doujin") ? "Ana" : "Work";
return work; return work;
} }
@@ -138,9 +130,11 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca
if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2) if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2)
return null; return null;
ScannedRating rating = new ScannedRating(); ScannedRating rating = new()
rating.Score = Convert.ToByte(ratingClass.Replace("star_", "")); {
rating.Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands); Score = Convert.ToByte(ratingClass.Replace("star_", "")),
Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands)
};
return rating; return rating;
} }

View File

@@ -65,6 +65,8 @@
</dt> </dt>
<dd class="maker_name"> <dd class="maker_name">
<a href="https://www.dlsite.com/maniax/circle/profile/=/maker_id/RG00001.html">The Maker</a> <a href="https://www.dlsite.com/maniax/circle/profile/=/maker_id/RG00001.html">The Maker</a>
<span class="separator">/</span>
<span class="author"><a href="https://www.dlsite.com/maniax/fsr/=/keyword_creater/some_creator" class="">Some Creator</a></span>
</dd> </dd>
<dd class="work_price_wrap"> <dd class="work_price_wrap">
<span class="work_price"> <span class="work_price">
@@ -136,9 +138,9 @@
<td class="work_1col_thumb"> <td class="work_1col_thumb">
<div class="work_thumb"> <div class="work_thumb">
<a href="https://www.dlsite.com/maniax/work/=/product_id/RJ01464588.html" id="_link_RJ00000002" class="work_thumb_inner"> <a href="https://www.dlsite.com/maniax/work/=/product_id/RJ01464588.html" id="_link_RJ00000002" class="work_thumb_inner">
<img alt="Title of Product" src="//img.dlsite.jp/resize/images2/work/doujin/RJ00000000/RJ00000002_img_main_240x240.jpg" class="lazy"> <img alt="Title of Product" src="//img.dlsite.jp/resize/images2/ana/doujin/RJ00000000/RJ00000002_img_main_240x240.jpg" class="lazy">
<div class="work_img_popover"> <div class="work_img_popover">
<img src="//img.dlsite.jp/modpub/images2/work/doujin/RJ00000000/RJ00000002_img_main.jpg" alt="Title of Product"> <img src="//img.dlsite.jp/modpub/images2/ana/doujin/RJ00000000/RJ00000002_img_main.jpg" alt="Title of Product">
</div> </div>
</a> </a>
<div class="work_category type_SOU"><a href="https://www.dlsite.com/maniax/fsr/=/work_type/SOU">Voice / ASMR</a></div> <div class="work_category type_SOU"><a href="https://www.dlsite.com/maniax/fsr/=/work_type/SOU">Voice / ASMR</a></div>

View File

@@ -44,10 +44,17 @@ public class VoiceWorkScannerTests
result[0].ProductId.ShouldBe("RJ00000001"); result[0].ProductId.ShouldBe("RJ00000001");
result[0].ProductName.ShouldBe("Title of Product"); result[0].ProductName.ShouldBe("Title of Product");
result[0].Description.ShouldBe("Description of the product."); result[0].Description.ShouldBe("Description of the product.");
result[0].Maker.ShouldBe("The Maker");
result[0].MakerId.ShouldBe("RG00001");
result[0].Creators.ShouldBe(["Some Creator"]);
result[0].Genres.ShouldBe(["Voice", "Trial version"]);
result[0].Tags.ShouldBe(["Male Protagonist", "Gal", "Uniform", "Harem", "Big Breasts", "Tanned Skin / Suntan"]);
result[0].Type.ShouldBe(DLSiteWorkType.Released);
result[0].Downloads.ShouldBe(1000); result[0].Downloads.ShouldBe(1000);
result[1].ExpectedDate.ShouldBe(new DateOnly(2025, 10, 11)); result[1].ExpectedDate.ShouldBe(new DateOnly(2025, 10, 11));
result[1].SalesDate.ShouldBeNull(); result[1].SalesDate.ShouldBeNull();
result[1].ProductId.ShouldBe("RJ00000002"); result[1].ProductId.ShouldBe("RJ00000002");
result[1].Type.ShouldBe(DLSiteWorkType.Announced);
} }
} }