brister/jsmr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Initial implementation of voice works scanning.

Browse Source
This commit is contained in:
Brian Bicknell
2025-09-11 00:07:49 -04:00
parent f250276a99
commit 3c0a39b324
50 changed files with 1351 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
JSMR.Api/JSMR.Api.http
View File

@@ -45,7 +45,7 @@ Content-Type: {{contentType}}
{
"options": {
"criteria": {
"keywords": "tsundere"
"keywords": "maid harem"
},
"pageNumber": 1,
"pageSize": 100,

25
JSMR.Application/Scanning/Contracts/DLSiteWork.cs Normal file
View File

@@ -0,0 +1,25 @@
namespace JSMR.Application.Scanning.Contracts;
public class DLSiteWork
{
public DLSiteWorkType WorkType { get; set; }
public DLSiteWorkCategory Category { get; set; }
public string? ProductName { get; set; }
public string? ProductUrl { get; set; }
public string? ProductId { get; set; }
public DateOnly? AnnouncedDate { get; set; }
public DateTime? ExpectedDate { get; set; }
public DateTime? SalesDate { get; set; }
public int? Downloads { get; set; }
public byte? StarRating { get; set; }
public int? Votes { get; set; }
public string? Maker { get; set; }
public string? MakerId { get; set; }
public string? Description { get; set; }
public ICollection<string> Genres { get; set; } = [];
public ICollection<string> Tags { get; set; } = [];
public ICollection<string> Creators { get; set; } = [];
public string? ImageUrl { get; set; }
public string? SmallImageUrl { get; set; }
public string? Type { get; set; }
}

13
JSMR.Application/Scanning/Contracts/DLSiteWorkCategory.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace JSMR.Application.Scanning.Contracts;
public enum DLSiteWorkCategory
{
Unknown,
VoiceASMR,
Manga,
RPG,
Video,
Adventure,
Simulation,
CG
}

7
JSMR.Application/Scanning/Contracts/DLSiteWorkType.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Application.Scanning.Contracts;
public enum DLSiteWorkType
{
Released,
Announced
}

8
JSMR.Application/Scanning/Ports/IVoiceWorksScanner.cs Normal file
View File

@@ -0,0 +1,8 @@
using JSMR.Application.Scanning.Contracts;
namespace JSMR.Application.Scanning.Ports;
public interface IVoiceWorksScanner
{
Task<IReadOnlyList<DLSiteWork>> ScanPageAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken = default);
}

26
JSMR.Application/Scanning/ScanVoiceWorksHandler.cs Normal file
View File

@@ -0,0 +1,26 @@
using JSMR.Application.Scanning.Ports;
namespace JSMR.Application.Scanning;
public sealed class ScanVoiceWorksHandler(IVoiceWorksScanner scanner)
{
//public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
//{
// var works = await scanner.ScanPageAsync(request, cancellationToken);
// if (works.Count == 0)
// return new ScanVoiceWorksResponse();
// var ingests = works.Select(VoiceWorkIngest.From).ToList();
// var upsert = await _writer.UpsertAsync(ingests, ct);
// // only update search text for affected rows
// await _search.UpdateAsync(upsert.AffectedVoiceWorkIds, ct);
// return new ScanVoiceWorksResponse
// {
// Inserted = upsert.Inserted,
// Updated = upsert.Updated
// };
//}
}

5
JSMR.Application/Scanning/ScanVoiceWorksRequest.cs Normal file
View File

@@ -0,0 +1,5 @@
using JSMR.Application.Common;
namespace JSMR.Application.Scanning;
public sealed record ScanVoiceWorksRequest(int PageNumber, int PageSize, Locale Locale);

7
JSMR.Application/Scanning/ScanVoiceWorksResponse.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Application.Scanning;
public sealed class ScanVoiceWorksResponse
{
public int Inserted { get; init; }
public int Updated { get; init; }
}

11
JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteHandler.cs Normal file
View File

@@ -0,0 +1,11 @@
using JSMR.Application.VoiceWorks.Ports;
namespace JSMR.Application.VoiceWorks.Commands.SetFavorite;
public sealed class SetVoiceWorkFavoriteHandler(IVoiceWorkWriter writer)
{
public async Task<SetVoiceWorkFavoriteResponse> HandleAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken = default)
{
return await writer.SetFavoriteAsync(request, cancellationToken);
}
}

3
JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteRequest.cs Normal file
View File

@@ -0,0 +1,3 @@
namespace JSMR.Application.VoiceWorks.Commands.SetFavorite;
public sealed record SetVoiceWorkFavoriteRequest(int VoiceWorkId, bool IsFavorite);

3
JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteResponse.cs Normal file
View File

@@ -0,0 +1,3 @@
namespace JSMR.Application.VoiceWorks.Commands.SetFavorite;
public sealed record SetVoiceWorkFavoriteResponse(int VoiceWorkId, bool IsFavorite);

8
JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs Normal file
View File

@@ -0,0 +1,8 @@
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
namespace JSMR.Application.VoiceWorks.Ports;
public interface IVoiceWorkWriter
{
Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken);
}

2
JSMR.Infrastructure/Caching/DistributedCacheAdapter.cs → JSMR.Infrastructure/Caching/Adapters/DistributedCacheAdapter.cs
View File

@@ -3,7 +3,7 @@ using Microsoft.Extensions.Caching.Distributed;
using System.Text;
using System.Text.Json;
namespace JSMR.Infrastructure.Caching;
namespace JSMR.Infrastructure.Caching.Adapters;
public class DistributedCacheAdapter(IDistributedCache distributedCache) : ICache
{

2
JSMR.Infrastructure/Caching/MemoryCacheAdapter.cs → JSMR.Infrastructure/Caching/Adapters/MemoryCacheAdapter.cs
View File

@@ -1,7 +1,7 @@
using JSMR.Application.Common.Caching;
using Microsoft.Extensions.Caching.Memory;
namespace JSMR.Infrastructure.Caching;
namespace JSMR.Infrastructure.Caching.Adapters;
public class MemoryCacheAdapter(IMemoryCache memoryCache) : ICache
{

55
JSMR.Infrastructure/Caching/ICacheObject.cs Normal file
View File

@@ -0,0 +1,55 @@
using JSMR.Application.Common.Caching;
using JSMR.Infrastructure.Data;
using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Caching;
public interface ICacheObject<T>
{
Task<T> GetAsync(CancellationToken cancellationToken = default);
Task<T> RefreshAsync(CancellationToken cancellationToken = default);
}
public abstract class CacheObject<T>(ICache cache) : ICacheObject<T>
{
protected abstract string Key { get; }
protected abstract Task<T> FetchAsync(CancellationToken cancellationToken = default);
protected virtual CacheEntryOptions Options => new() { SlidingExpiration = TimeSpan.FromHours(1) };
public async virtual Task<T> GetAsync(CancellationToken cancellationToken = default)
{
return await cache.GetAsync<T>(Key, cancellationToken) ?? await RefreshAsync(cancellationToken);
}
public async virtual Task<T> RefreshAsync(CancellationToken cancellationToken = default)
{
T cacheObject = await FetchAsync(cancellationToken);
await cache.SetAsync(Key, cacheObject, Options, cancellationToken);
return cacheObject;
}
}
public interface ISpamCircleCache : ICacheObject<string[]>
{
}
public class SpamCircleCache(IDbContextFactory<AppDbContext> contextFactory, ICache cache) : CacheObject<string[]>(cache), ISpamCircleCache
{
protected override string Key => "SpamCircles";
protected override async Task<string[]> FetchAsync(CancellationToken cancellationToken = default)
{
using var context = contextFactory.CreateDbContext();
return await context.Circles
.AsNoTracking()
.Where(circle => circle.Spam)
.Select(circle => circle.MakerId)
.Distinct()
.ToArrayAsync(cancellationToken);
}
}

7
JSMR.Infrastructure/Common/Locales/EnglishLocale.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Common.Locales;
public class EnglishLocale : ILocale
{
public string Abbreviation => "en";
public string Code => "en_US";
}

7
JSMR.Infrastructure/Common/Locales/ILocale.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Common.Locales;
public interface ILocale
{
string Abbreviation { get; }
string Code { get; }
}

7
JSMR.Infrastructure/Common/Locales/JapaneseLocale.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Common.Locales;
public class JapaneseLocale : ILocale
{
public string Abbreviation => "jp";
public string Code => "ja_JP";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/AlingualLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class AlingualLanguage : ISupportedLanguage
{
public string Code => "NM";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/ChineseLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class ChineseLanguage : ISupportedLanguage
{
public string Code => "CHI";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/DLSiteOfficialTranslationLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class DLSiteOfficialTranslationLanguage : ISupportedLanguage
{
public string Code => "DOT";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/EnglishLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class EnglishLanguage : ISupportedLanguage
{
public string Code => "ENG";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/ISupportedLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public interface ISupportedLanguage
{
string Code { get; }
}

6
JSMR.Infrastructure/Common/SupportedLanguages/JapaneseLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class JapaneseLanguage : ISupportedLanguage
{
public string Code => "JPN";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/KoreanLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class KoreanLanguage : ISupportedLanguage
{
public string Code => "KO_KR";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/SimplifiedChineseLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class SimplifiedChineseLanguage : ISupportedLanguage
{
public string Code => "CHI_HANS";
}

6
JSMR.Infrastructure/Common/SupportedLanguages/TraditionalChineseLanguage.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace JSMR.Infrastructure.Common.SupportedLanguages;
public class TraditionalChineseLanguage : ISupportedLanguage
{
public string Code => "CHI_HANT";
}

8
JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs
View File

@@ -7,12 +7,15 @@ using JSMR.Application.Creators.Queries.Search.Ports;
using JSMR.Application.Integrations.Ports;
using JSMR.Application.Tags.Ports;
using JSMR.Application.Tags.Queries.Search.Ports;
using JSMR.Application.VoiceWorks.Ports;
using JSMR.Application.VoiceWorks.Queries.Search;
using JSMR.Infrastructure.Caching;
using JSMR.Infrastructure.Caching.Adapters;
using JSMR.Infrastructure.Data.Repositories.Circles;
using JSMR.Infrastructure.Data.Repositories.Creators;
using JSMR.Infrastructure.Data.Repositories.Tags;
using JSMR.Infrastructure.Data.Repositories.VoiceWorks;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Integrations.DLSite;
using Microsoft.Extensions.DependencyInjection;
@@ -28,6 +31,7 @@ public static class InfrastructureServiceCollectionExtensions
services.AddScoped<IVoiceWorkSearchProvider, VoiceWorkSearchProvider>();
services.AddScoped<IVoiceWorkFullTextSearch, MySqlVoiceWorkFullTextSearch>();
services.AddScoped<IVoiceWorkWriter, VoiceWorkWriter>();
services.AddScoped<ITagSearchProvider, TagSearchProvider>();
services.AddScoped<ITagWriter, TagWriter>();
@@ -36,6 +40,10 @@ public static class InfrastructureServiceCollectionExtensions
services.AddScoped<ICreatorWriter, CreatorWriter>();
services.AddSingleton<ICache, MemoryCacheAdapter>();
services.AddSingleton<ISpamCircleCache, SpamCircleCache>();
services.AddScoped<IHttpService, HttpService>();
services.AddScoped<IHtmlLoader, HtmlLoader>();
return services;
}

13
JSMR.Infrastructure/Data/Repositories/VoiceWorks/MySqlBooleanQuery.cs
View File

@@ -4,10 +4,10 @@ namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public static class MySqlBooleanQuery
{
// Main entry
public static string Normalize(string input)
{
if (string.IsNullOrWhiteSpace(input)) return string.Empty;
if (string.IsNullOrWhiteSpace(input))
return string.Empty;
// Split into top-level tokens by spaces (not inside quotes/parentheses)
var tokens = SplitTopLevel(input.Trim(), ' ');
@@ -16,7 +16,12 @@ public static class MySqlBooleanQuery
foreach (var raw in tokens)
{
var t = raw.Trim();
if (t.Length == 0) continue;
if (t.Length == 0)
continue;
if (t is "|")
continue;
// Preserve explicit boolean operators user may already supply
if (t[0] == '-' || t[0] == '+')
@@ -63,7 +68,7 @@ public static class MySqlBooleanQuery
var orParts = SplitTopLevel(token, '|')
.Select(p => NormalizeOrAtom(p.Trim()))
.Where(p => p.Length > 0);
return "(" + string.Join("|", orParts) + ")";
return "(" + string.Join(" ", orParts) + ")";
}
// Plain atom -> as-is

25
JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs Normal file
View File

@@ -0,0 +1,25 @@
using JSMR.Application.VoiceWorks.Commands.SetFavorite;
using JSMR.Application.VoiceWorks.Ports;
using JSMR.Domain.Entities;
using Microsoft.EntityFrameworkCore;
namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks;
public class VoiceWorkWriter(AppDbContext context) : IVoiceWorkWriter
{
public async Task<SetVoiceWorkFavoriteResponse> SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken)
{
VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken);
voiceWork.Favorite = request.IsFavorite;
await context.SaveChangesAsync(cancellationToken);
return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite);
}
private async Task<VoiceWork> GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken)
{
return await context.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken)
?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found.");
}
}

117
JSMR.Infrastructure/Http/ApiClient.cs
View File

@@ -1,79 +1,88 @@
using Microsoft.Extensions.Logging;
using System.IO;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace JSMR.Infrastructure.Http;
public abstract class ApiClient(HttpClient http, ILogger logger, JsonSerializerOptions? json = null)
public abstract class ApiClient(IHttpService http, ILogger logger, JsonSerializerOptions? json = null)
{
protected async Task<T> GetJsonAsync<T>(
string url,
Action<HttpRequestHeaders>? configureHeaders = null,
CancellationToken ct = default
)
protected async Task<T> GetJsonAsync<T>(string url, CancellationToken cancellationToken = default)
{
using var req = new HttpRequestMessage(HttpMethod.Get, url);
configureHeaders?.Invoke(req.Headers);
string response = await http.GetStringAsync(url, cancellationToken);
LogRequest(req);
using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
await EnsureSuccess(res).ConfigureAwait(false);
var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
var model = await JsonSerializer.DeserializeAsync<T>(stream, json, ct).ConfigureAwait(false)
return JsonSerializer.Deserialize<T>(response, json)
?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}.");
return model;
}
protected async Task<TResponse> PostJsonAsync<TRequest, TResponse>(
string url,
TRequest payload,
Action<HttpRequestHeaders>? configureHeaders = null,
CancellationToken ct = default)
{
var content = new StringContent(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json");
using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
configureHeaders?.Invoke(req.Headers);
//protected async Task<T> GetJsonAsync<T>(
// string url,
// Action<HttpRequestHeaders>? configureHeaders = null,
// CancellationToken ct = default
// )
//{
// using var req = new HttpRequestMessage(HttpMethod.Get, url);
// configureHeaders?.Invoke(req.Headers);
LogRequest(req);
// LogRequest(req);
using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
await EnsureSuccess(res).ConfigureAwait(false);
// using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
// await EnsureSuccess(res).ConfigureAwait(false);
var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
// var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
var model = await JsonSerializer.DeserializeAsync<TResponse>(stream, json, ct).ConfigureAwait(false)
?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}.");
return model;
}
// var model = await JsonSerializer.DeserializeAsync<T>(stream, json, ct).ConfigureAwait(false)
// ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}.");
protected virtual void LogRequest(HttpRequestMessage req)
=> logger.LogDebug("HTTP {Method} {Uri}", req.Method, req.RequestUri);
// return model;
//}
protected virtual void LogFailure(HttpResponseMessage res, string body)
=> logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)res.StatusCode, res.RequestMessage?.RequestUri, Truncate(body, 500));
//protected async Task<TResponse> PostJsonAsync<TRequest, TResponse>(
// string url,
// TRequest payload,
// Action<HttpRequestHeaders>? configureHeaders = null,
// CancellationToken ct = default)
//{
// var content = new StringContent(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json");
// using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
// configureHeaders?.Invoke(req.Headers);
protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…";
// LogRequest(req);
protected async Task EnsureSuccess(HttpResponseMessage res)
{
if (res.IsSuccessStatusCode) return;
// using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
// await EnsureSuccess(res).ConfigureAwait(false);
string body;
try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); }
catch { body = "<unable to read body>"; }
// var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
LogFailure(res, body);
// var model = await JsonSerializer.DeserializeAsync<TResponse>(stream, json, ct).ConfigureAwait(false)
// ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}.");
// Throw a richer exception (you can customize per API)
throw new HttpRequestException(
$"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}",
null,
res.StatusCode);
}
// return model;
//}
//protected virtual void LogRequest(HttpRequestMessage req)
// => logger.LogDebug("HTTP {Method} {Uri}", req.Method, req.RequestUri);
//protected virtual void LogFailure(HttpResponseMessage res, string body)
// => logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)res.StatusCode, res.RequestMessage?.RequestUri, Truncate(body, 500));
//protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…";
//protected async Task EnsureSuccess(HttpResponseMessage res)
//{
// if (res.IsSuccessStatusCode) return;
// string body;
// try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); }
// catch { body = "<unable to read body>"; }
// LogFailure(res, body);
// Throw a richer exception(you can customize per API)
// throw new HttpRequestException(
// $"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}",
// null,
// res.StatusCode);
//}
}

16
JSMR.Infrastructure/Http/HtmlLoader.cs Normal file
View File

@@ -0,0 +1,16 @@
using HtmlAgilityPack;
namespace JSMR.Infrastructure.Http;
public class HtmlLoader(IHttpService httpService) : IHtmlLoader
{
public async Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken)
{
string html = await httpService.GetStringAsync(url, cancellationToken);
HtmlDocument document = new();
document.LoadHtml(html);
return document;
}
}

24
JSMR.Infrastructure/Http/HttpService.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace JSMR.Infrastructure.Http;
public class HttpService(HttpClient httpClient) : IHttpService
{
public Task<string> GetStringAsync(string url, CancellationToken cancellationToken)
=> GetStringAsync(url, new Dictionary<string, string>(), cancellationToken);
public async Task<string> GetStringAsync(string url, IDictionary<string, string> headers, CancellationToken cancellationToken)
{
using HttpRequestMessage request = new(HttpMethod.Get, url);
foreach (KeyValuePair<string, string> header in headers)
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
using HttpResponseMessage response = await httpClient.SendAsync(request, cancellationToken);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStringAsync(cancellationToken);
}
}

8
JSMR.Infrastructure/Http/IHtmlLoader.cs Normal file
View File

@@ -0,0 +1,8 @@
using HtmlAgilityPack;
namespace JSMR.Infrastructure.Http;
public interface IHtmlLoader
{
Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken);
}

7
JSMR.Infrastructure/Http/IHttpService.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Http;
public interface IHttpService
{
Task<string> GetStringAsync(string url, CancellationToken cancellationToken);
Task<string> GetStringAsync(string url, IDictionary<string, string> headers, CancellationToken cancellationToken);
}

4
JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs
View File

@@ -5,11 +5,11 @@ using Microsoft.Extensions.Logging;
namespace JSMR.Infrastructure.Integrations.Chobit;
public class ChobitClient(HttpClient http, ILogger logger) : ApiClient(http, logger), IChobitClient
public class ChobitClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IChobitClient
{
public Task<ChobitWorkResult> GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default)
{
var url = $"api/v2/dlsite/embed?workno_list=${string.Join(",", productIds)}";
return GetJsonAsync<ChobitWorkResult>(url, ct: cancellationToken);
return GetJsonAsync<ChobitWorkResult>(url, cancellationToken);
}
}

8
JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs
View File

@@ -7,21 +7,21 @@ using Microsoft.Extensions.Logging;
namespace JSMR.Infrastructure.Integrations.DLSite;
public class DLSiteClient(HttpClient http, ILogger logger) : ApiClient(http, logger), IDLSiteClient
public class DLSiteClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IDLSiteClient
{
public async Task<VoiceWorkDetailCollection> GetVoiceWorkDetailsAsync(string[] productIds, CancellationToken cancellationToken = default)
{
if (productIds.Length == 0)
throw new Exception("No products to get product information.");
return [];
string productIdCollection = string.Join(",", productIds.Where(x => !string.IsNullOrWhiteSpace(x)));
if (string.IsNullOrWhiteSpace(productIdCollection))
throw new Exception("Invalid product id(s).");
return [];
string url = $"maniax/product/info/ajax?product_id={productIdCollection}&cdn_cache_min=1";
var productInfoCollection = await GetJsonAsync<ProductInfoCollection>(url, ct: cancellationToken);
var productInfoCollection = await GetJsonAsync<ProductInfoCollection>(url, cancellationToken);
return DLSiteToDomainMapper.Map(productInfoCollection);
}

1
JSMR.Infrastructure/JSMR.Infrastructure.csproj
View File

@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.12.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.8" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.8" />

127
JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilder.cs Normal file
View File

@@ -0,0 +1,127 @@
using JSMR.Infrastructure.Common.Locales;
using JSMR.Infrastructure.Common.SupportedLanguages;
namespace JSMR.Infrastructure.Scanning;
public class DLSiteSearchFilterBuilder
{
private readonly List<string> _optionsAnd = [];
private readonly List<string> _optionsNot = [];
private readonly List<string> _excludedMakers = [];
private ILocale _locale = new JapaneseLocale();
private void AddToOptionsAnd(string value)
{
if (_optionsAnd.Contains(value))
return;
_optionsAnd.Add(value);
}
private void AddToOptionsNot(string value)
{
if (_optionsNot.Contains(value))
return;
_optionsNot.Add(value);
}
public DLSiteSearchFilterBuilder UseLocale(ILocale locale)
{
_locale = locale;
return this;
}
public DLSiteSearchFilterBuilder IncludeSupportedLanguage(ISupportedLanguage language)
{
AddToOptionsAnd(language.Code);
return this;
}
public DLSiteSearchFilterBuilder ExcludeMakers(string[] makerIds)
{
foreach (var makerId in makerIds)
ExcludeMaker(makerId);
return this;
}
public DLSiteSearchFilterBuilder ExcludeMaker(string makerId)
{
if (string.IsNullOrWhiteSpace(makerId))
return this;
string trimmedMakerId = makerId.Trim();
if (_excludedMakers.Contains(trimmedMakerId))
return this;
_excludedMakers.Add(trimmedMakerId);
return this;
}
public DLSiteSearchFilterBuilder ExcludePartiallyAIGeneratedWorks()
{
AddToOptionsNot("AIP");
return this;
}
public DLSiteSearchFilterBuilder ExcludeAIGeneratedWorks()
{
AddToOptionsNot("AIG");
return this;
}
public string BuildSearchQuery(int pageNumber, int pageSize)
{
ILocale locale = _locale ?? new JapaneseLocale();
using (var writer = new StringWriter())
{
writer.Write($"https://www.dlsite.com/maniax/");
writer.Write($"fsr/=/language/{locale.Abbreviation}/");
writer.Write("sex_category[0]/male/");
writer.Write("ana_flg/all/");
writer.Write("work_category[0]/doujin/");
writer.Write("order[0]/release_d/");
writer.Write("work_type_category[0]/audio/");
writer.Write("work_type_category_name[0]/ボイス・ASMR/");
if (_optionsAnd.Count > 0)
{
writer.Write("options_and_or/and/");
for (int index = 0; index < _optionsAnd.Count; index++)
{
writer.Write($"options[{index}]/{_optionsAnd[index]}/");
}
}
if (_excludedMakers.Count > 0)
{
List<string> spamMakers = [.. _excludedMakers.Select(x => "-" + x)];
string makerFilterValue = string.Join("+", spamMakers).Trim();
writer.Write($"keyword_maker_name/{makerFilterValue}/");
}
for (int index = 0; index < _optionsNot.Count; index++)
{
writer.Write($"options_not[{index}]/{_optionsNot[index]}/");
}
writer.Write($"per_page/{pageSize}/");
writer.Write($"page/{pageNumber}/");
writer.Write("show_type/1/");
writer.Write($"?locale={locale.Code}");
return writer.ToString();
}
}
}

52
JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilderExtensions.cs Normal file
View File

@@ -0,0 +1,52 @@
using JSMR.Infrastructure.Common.Locales;
using JSMR.Infrastructure.Common.SupportedLanguages;
namespace JSMR.Infrastructure.Scanning;
public static class DLSiteSearchFilterBuilderExtensions
{
public static DLSiteSearchFilterBuilder UseDefaultLocale(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.UseLocale(new JapaneseLocale());
}
public static DLSiteSearchFilterBuilder UseEnglishLocale(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.UseLocale(new EnglishLocale());
}
public static DLSiteSearchFilterBuilder IncludeJapaneseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new JapaneseLanguage());
}
public static DLSiteSearchFilterBuilder IncludeEnglishSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new EnglishLanguage());
}
public static DLSiteSearchFilterBuilder IncludeChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new ChineseLanguage());
}
public static DLSiteSearchFilterBuilder IncludeSimplifiedChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new SimplifiedChineseLanguage());
}
public static DLSiteSearchFilterBuilder IncludeTraditionalChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new TraditionalChineseLanguage());
}
public static DLSiteSearchFilterBuilder IncludeKoreanSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new KoreanLanguage());
}
public static DLSiteSearchFilterBuilder IncludeAlingualSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder)
{
return searchFilterBuilder.IncludeSupportedLanguage(new AlingualLanguage());
}
}

164
JSMR.Infrastructure/Scanning/EnglishVoiceWorksScanner.cs Normal file
View File

@@ -0,0 +1,164 @@
using JSMR.Application.Scanning;
using JSMR.Infrastructure.Caching;
using JSMR.Infrastructure.Common.Locales;
using JSMR.Infrastructure.Common.SupportedLanguages;
using JSMR.Infrastructure.Http;
using System.Text.RegularExpressions;
namespace JSMR.Infrastructure.Scanning;
public partial class EnglishVoiceWorksScanner(IHtmlLoader loader, ISpamCircleCache spamCircleCache)
: VoiceWorksScanner(loader, spamCircleCache)
{
[GeneratedRegex(@"Release:&nbsp;(.*?)[/](\d{2})[/](\d{4})", RegexOptions.IgnoreCase, "en-US")]
private static partial Regex SalesDateRegex();
[GeneratedRegex(@"^(Early|Middle|Late)\s(.*?)\s(\d{4})", RegexOptions.IgnoreCase, "en-US")]
private static partial Regex EstimatedDateRegex();
protected override ILocale Locale => new EnglishLocale();
protected override ISupportedLanguage[] SupportedLanguages =>
[
new JapaneseLanguage(),
new EnglishLanguage(),
new AlingualLanguage()
];
protected override DateTime? GetEstimatedReleaseDate(string expectedDate)
{
if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定"))
return null;
Regex textRegex = EstimatedDateRegex();
MatchCollection textMatches = textRegex.Matches(expectedDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
return null;
GroupCollection groups = textMatches[0].Groups;
int releaseYear = Convert.ToInt32(groups[3].Value);
int releaseMonth = 1;
int releaseDay = 1;
string releaseTime = groups[1].Value;
string releaseMonthText = groups[2].Value;
switch (releaseTime)
{
case "Early":
releaseDay = 1;
break;
case "Middle":
releaseDay = 11;
break;
case "Late":
releaseDay = 21;
break;
}
switch (releaseMonthText)
{
case "Jan.":
releaseMonth = 1;
break;
case "Feb.":
releaseMonth = 2;
break;
case "Mar.":
releaseMonth = 3;
break;
case "Apr.":
releaseMonth = 4;
break;
case "May.":
releaseMonth = 5;
break;
case "Jun.":
releaseMonth = 6;
break;
case "Jul.":
releaseMonth = 7;
break;
case "Aug.":
releaseMonth = 8;
break;
case "Sep.":
releaseMonth = 9;
break;
case "Oct.":
releaseMonth = 10;
break;
case "Nov.":
releaseMonth = 11;
break;
case "Dec.":
releaseMonth = 12;
break;
}
return new DateTime(releaseYear, releaseMonth, releaseDay);
}
protected override DateTime? GetSalesDate(string salesDate)
{
Regex textRegex = SalesDateRegex();
MatchCollection textMatches = textRegex.Matches(salesDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
return null;
string month = textMatches[0].Groups[1].Value;
int releaseMonth = -1;
switch (month)
{
case "Jan":
releaseMonth = 1;
break;
case "Feb":
releaseMonth = 2;
break;
case "Mar":
releaseMonth = 3;
break;
case "Apr":
releaseMonth = 4;
break;
case "May":
releaseMonth = 5;
break;
case "Jun":
releaseMonth = 6;
break;
case "Jul":
releaseMonth = 7;
break;
case "Aug":
releaseMonth = 8;
break;
case "Sep":
releaseMonth = 9;
break;
case "Oct":
releaseMonth = 10;
break;
case "Nov":
releaseMonth = 11;
break;
case "Dec":
releaseMonth = 12;
break;
}
if (releaseMonth == -1)
return null;
int releaseYear = Convert.ToInt32(textMatches[0].Groups[3].Value);
int releaseDay = Convert.ToInt32(textMatches[0].Groups[2].Value);
return new DateTime(releaseYear, releaseMonth, releaseDay);
}
}

74
JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs Normal file
View File

@@ -0,0 +1,74 @@
using JSMR.Infrastructure.Caching;
using JSMR.Infrastructure.Common.Locales;
using JSMR.Infrastructure.Common.SupportedLanguages;
using JSMR.Infrastructure.Http;
using System.Text.RegularExpressions;
namespace JSMR.Infrastructure.Scanning;
public class JapaneseVoiceWorksScanner(IHtmlLoader loader, ISpamCircleCache spamCircleCache)
: VoiceWorksScanner(loader, spamCircleCache)
{
protected override ILocale Locale => new JapaneseLocale();
protected override ISupportedLanguage[] SupportedLanguages =>
[
new JapaneseLanguage(),
new EnglishLanguage(),
new TraditionalChineseLanguage(),
new SimplifiedChineseLanguage(),
new KoreanLanguage(),
new AlingualLanguage()
];
protected override DateTime? GetEstimatedReleaseDate(string expectedDate)
{
if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定"))
return null;
Regex textRegex = new Regex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase);
MatchCollection textMatches = textRegex.Matches(expectedDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
return null;
int releaseYear = Convert.ToInt32(textMatches[0].Groups[1].Value);
int releaseMonth = Convert.ToInt32(textMatches[0].Groups[2].Value);
int releaseDay = 1;
string releaseTime = textMatches[0].Groups[3].Value;
switch (releaseTime)
{
case "上旬発売予定":
case "上旬 発売予定":
releaseDay = 1;
break;
case "中旬発売予定":
case "中旬 発売予定":
releaseDay = 11;
break;
case "下旬発売予定":
case "下旬 発売予定":
releaseDay = 21;
break;
}
return new DateTime(releaseYear, releaseMonth, releaseDay);
}
protected override DateTime? GetSalesDate(string salesDate)
{
Regex textRegex = new Regex("販売日:&nbsp;(.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase);
MatchCollection textMatches = textRegex.Matches(salesDate);
if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4)
return null;
int releaseYear = Convert.ToInt32(textMatches[0].Groups[1].Value);
int releaseMonth = Convert.ToInt32(textMatches[0].Groups[2].Value);
int releaseDay = Convert.ToInt32(textMatches[0].Groups[3].Value);
return new DateTime(releaseYear, releaseMonth, releaseDay);
}
}

37
JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs Normal file
View File

@@ -0,0 +1,37 @@
using HtmlAgilityPack;
namespace JSMR.Infrastructure.Scanning.Models;
public class DLSiteHtmlDocument
{
private readonly HtmlNodeCollection _workColumns;
private readonly HtmlNodeCollection _workColumnRights;
private readonly HtmlNodeCollection _workThumbs;
public HtmlNode PageTotalNode { get; }
public DLSiteHtmlDocument(HtmlDocument document)
{
_workColumns = document.DocumentNode.SelectNodes("//dl[@class='work_1col']");
_workColumnRights = document.DocumentNode.SelectNodes("//td[@class='work_1col_right']");
_workThumbs = document.DocumentNode.SelectNodes("//div[@class='work_thumb']");
PageTotalNode = document.DocumentNode.SelectNodes("//div[@class='page_total']/strong")[0];
}
public List<DLSiteHtmlNode> GetDLSiteNodes()
{
var nodes = new List<DLSiteHtmlNode>();
if (_workColumns.Count != _workColumnRights.Count || _workColumns.Count != _workThumbs.Count)
throw new Exception("Work column node counts do not match!");
for (int i = 0; i < _workColumns.Count; i++)
{
var node = new DLSiteHtmlNode(_workColumns[i], _workColumnRights[i], _workThumbs[i]);
nodes.Add(node);
}
return nodes;
}
}

156
JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs Normal file
View File

@@ -0,0 +1,156 @@
using HtmlAgilityPack;
namespace JSMR.Infrastructure.Scanning.Models;
public class DLSiteHtmlNode
{
public HtmlNode LeftNode { get; }
public HtmlNode RightNode { get; }
public HtmlNode ThumbNode { get; }
public HtmlNode ProductNode { get; private set; }
public HtmlNode ProductLinkNode { get; private set; }
public HtmlNode ProductTextNode { get; private set; }
public HtmlNode DescriptionNode { get; private set; }
public HtmlNode MakerNode { get; private set; }
public HtmlNode MakerLinkNode { get; private set; }
public HtmlNode SalesDateNode { get; private set; }
public HtmlNode ExpectedDateNode { get; private set; }
public HtmlNode DownloadsNode { get; private set; }
public HtmlNode StarRatingNode { get; private set; }
public HtmlNode ImageNode { get; private set; }
public List<HtmlNode> GenreNodes { get; private set; }
public List<HtmlNode> SearchTagNodes { get; private set; }
public List<HtmlNode> CreatorNodes { get; private set; }
public DLSiteHtmlNode(HtmlNode leftNode, HtmlNode rightNode, HtmlNode thumbNode)
{
LeftNode = leftNode;
RightNode = rightNode;
ThumbNode = thumbNode;
ProductNode = LeftNode.SelectNodes(".//dt[@class='work_name']")[0];
ProductLinkNode = ProductNode.SelectNodes(".//a")[0];
ProductTextNode = GetProductTextNode();
DescriptionNode = LeftNode.SelectNodes(".//dd[@class='work_text']")[0];
MakerNode = LeftNode.SelectNodes(".//dd[@class='maker_name']")[0];
MakerLinkNode = MakerNode.SelectNodes(".//a[contains(@href, 'maker_id')]")[0];
ExpectedDateNode = GetExpectedDateNode();
InitializeGenreNodes();
InitializeSearchTagNodes();
InitializeCreatorNodes();
InitializeSalesAndDownloadsNodes();
InitializeStarRatingNode();
InitializeImageNode();
}
private void InitializeGenreNodes()
{
HtmlNode genreNode = LeftNode.SelectNodes(".//dd[@class='work_genre']")[0];
GenreNodes = [.. genreNode.SelectNodes(".//span")];
}
private void InitializeSearchTagNodes()
{
HtmlNodeCollection searchTagNodes = LeftNode.SelectNodes(".//dd[@class='search_tag']");
if (searchTagNodes == null || searchTagNodes.Count == 0)
{
SearchTagNodes = [];
}
else
{
HtmlNodeCollection searchTagNodesLinks = searchTagNodes[0].SelectNodes(".//a");
if (searchTagNodesLinks == null || searchTagNodesLinks.Count == 0)
{
SearchTagNodes = [];
}
else
{
SearchTagNodes = [.. searchTagNodesLinks];
}
}
}
private void InitializeCreatorNodes()
{
HtmlNodeCollection creatorNodes = MakerNode.SelectNodes(".//a[contains(@href, 'keyword_creater')]");
if (creatorNodes == null || creatorNodes.Count == 0)
{
CreatorNodes = [];
}
else
{
CreatorNodes = [.. creatorNodes];
}
}
private void InitializeSalesAndDownloadsNodes()
{
HtmlNodeCollection workInfoBox = RightNode.SelectNodes(".//ul[@class='work_info_box']");
if (workInfoBox != null)
{
HtmlNodeCollection salesDateNodes = workInfoBox[0].SelectNodes(".//li[@class='sales_date']");
if (salesDateNodes != null && salesDateNodes.Count > 0)
{
SalesDateNode = salesDateNodes[0];
}
// TODO: Fix!
//DownloadsNode = RightNode.SelectSingleNode(".//span[@class='_dl_count_" + works[rightsIndex].ProductId + "']");
DownloadsNode = RightNode.SelectSingleNode(".//span[contains(@class, '_dl_count_')]");
}
}
private void InitializeStarRatingNode()
{
var ratingsNode = RightNode.SelectSingleNode(".//li[@class='work_rating']");
if (ratingsNode == null)
return;
StarRatingNode = ratingsNode.SelectSingleNode(".//div[contains(@class, 'star_rating')]");
}
private HtmlNode GetProductTextNode()
{
if (ProductLinkNode.ChildNodes.Count > 1 && ProductLinkNode.ChildNodes[0].Name == "#text")
{
return ProductLinkNode.ChildNodes[0];
}
else
{
return ProductLinkNode;
}
}
private HtmlNode GetExpectedDateNode()
{
HtmlNodeCollection expectedDateNodes = ProductNode.SelectNodes(".//p[@class='expected_date']");
if (expectedDateNodes != null && expectedDateNodes.Count > 0)
{
return expectedDateNodes[0];
}
else
{
return null;
}
}
private void InitializeImageNode()
{
HtmlNode linkNode = ThumbNode.SelectNodes(".//a")[0];
ImageNode = linkNode.SelectNodes(".//img")[0];
}
}

7
JSMR.Infrastructure/Scanning/Models/ScannedRating.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace JSMR.Infrastructure.Scanning.Models;
public class ScannedRating
{
public byte Score { get; set; }
public int Votes { get; set; }
}

49
JSMR.Infrastructure/Scanning/ScannerUtilities.cs Normal file
View File

@@ -0,0 +1,49 @@
using HtmlAgilityPack;
using System.Web;
namespace JSMR.Infrastructure.Scanning;
public static class ScannerUtilities
{
public static List<string> GetStringListFromNodes(List<HtmlNode> nodes)
{
return nodes
.Where(node => string.IsNullOrEmpty(node.InnerHtml) == false)
.Select(node => HttpUtility.HtmlDecode(node.InnerHtml))
.ToList();
}
public static string GetDecodedText(HtmlNode node)
{
if (node == null)
return string.Empty;
if (string.IsNullOrWhiteSpace(node.InnerHtml))
return string.Empty;
return HttpUtility.HtmlDecode(node.InnerHtml.Replace("\n", "")).Trim();
}
public static string GetTextBetween(string text, string startText, string endText)
{
int startIndex = text.IndexOf(startText) + startText.Length;
int endIndex = text.IndexOf(endText);
int length = endIndex - startIndex;
if (length <= 0)
return "";
return text.Substring(startIndex, length);
}
public static string GetImageSource(HtmlNode imageNode)
{
string imageSource = imageNode.GetAttributeValue("src", "");
if (string.IsNullOrEmpty(imageSource))
imageSource = imageNode.GetAttributeValue("data-src", "");
return imageSource;
}
}

164
JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs Normal file
View File

@@ -0,0 +1,164 @@
using HtmlAgilityPack;
using JSMR.Application.Scanning;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports;
using JSMR.Infrastructure.Caching;
using JSMR.Infrastructure.Common.Locales;
using JSMR.Infrastructure.Common.SupportedLanguages;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Scanning.Models;
using System.Globalization;
using System.Text.RegularExpressions;
namespace JSMR.Infrastructure.Scanning;
public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader, ISpamCircleCache spamCircleCache) : IVoiceWorksScanner
{
protected abstract ILocale Locale { get; }
protected abstract ISupportedLanguage[] SupportedLanguages { get; }
protected abstract DateTime? GetEstimatedReleaseDate(string expectedDate);
protected abstract DateTime? GetSalesDate(string salesDate);
protected virtual bool ExcludeSpamCircles => true;
protected virtual bool ExcludePartiallyAIGeneratedWorks => true;
protected virtual bool ExcludeAIGeneratedWorks => true;
public async Task<IReadOnlyList<DLSiteWork>> ScanPageAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken = default)
{
DLSiteHtmlDocument document = await GetDLSiteHtmlCollectionAsync(request, cancellationToken);
List<DLSiteHtmlNode> nodes = document.GetDLSiteNodes();
return GetDLSiteWorks(nodes);
}
private async Task<DLSiteHtmlDocument> GetDLSiteHtmlCollectionAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
{
string url = await GetUrlAsync(request, cancellationToken);
HtmlDocument document = await htmlLoader.GetHtmlDocumentAsync(url, cancellationToken);
return new DLSiteHtmlDocument(document);
}
protected virtual async ValueTask<string> GetUrlAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
{
DLSiteSearchFilterBuilder filterBuilder = new();
foreach (ISupportedLanguage supprotedLanguage in SupportedLanguages)
{
filterBuilder.IncludeSupportedLanguage(supprotedLanguage);
}
if (ExcludeSpamCircles)
{
string[] makerIds = await spamCircleCache.GetAsync(cancellationToken);
foreach (string makerId in makerIds)
filterBuilder.ExcludeMaker(makerId);
}
if (ExcludePartiallyAIGeneratedWorks)
filterBuilder.ExcludePartiallyAIGeneratedWorks();
if (ExcludeAIGeneratedWorks)
filterBuilder.ExcludeAIGeneratedWorks();
return filterBuilder.BuildSearchQuery(request.PageNumber, request.PageSize);
}
private List<DLSiteWork> GetDLSiteWorks(List<DLSiteHtmlNode> nodes)
{
var works = new List<DLSiteWork>();
//var spamCircles = SpamCircleCache.Get();
foreach (DLSiteHtmlNode node in nodes)
{
DLSiteWork work = GetDLSiteWork(node);
//if (spamCircles.Any(circle => circle.MakerId == work.MakerId))
// continue;
works.Add(work);
}
return works;
}
private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node)
{
DLSiteWork work = new();
work.ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode);
work.ProductUrl = node.ProductLinkNode.Attributes["href"].Value;
work.ProductId = ScannerUtilities.GetTextBetween(work.ProductUrl, "product_id/", ".html");
work.Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode);
string makerUrl = node.MakerLinkNode.Attributes["href"].Value;
work.MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html");
work.Description = ScannerUtilities.GetDecodedText(node.DescriptionNode);
if (node.ExpectedDateNode != null)
{
work.ExpectedDate = GetEstimatedReleaseDate(node.ExpectedDateNode.InnerHtml.Trim());
}
if (node.SalesDateNode != null)
{
work.SalesDate = GetSalesDate(node.SalesDateNode.InnerHtml);
}
if (node.DownloadsNode != null)
{
work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands);
}
var rating = GetScannedRating(node.StarRatingNode);
if (rating != null)
{
work.StarRating = rating.Score;
work.Votes = rating.Votes;
}
work.Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes);
work.Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes);
work.Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes);
string imageSource = ScannerUtilities.GetImageSource(node.ImageNode);
string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif");
work.SmallImageUrl = imageSource;
work.ImageUrl = imageUrl;
work.Type = imageUrl.Contains("ana/doujin") ? "Ana" : "Work";
return work;
}
private static ScannedRating? GetScannedRating(HtmlNode starRatingNode)
{
if (starRatingNode == null)
return null;
string voteText = starRatingNode.InnerText;
string? ratingClass = starRatingNode.GetClasses().FirstOrDefault(classNames =>
classNames.Contains("star_") && classNames != "star_rating");
if (string.IsNullOrEmpty(ratingClass))
return null;
Regex votesRegex = new Regex(@"\((.*?)\)", RegexOptions.IgnoreCase);
MatchCollection voteMatches = votesRegex.Matches(voteText);
if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2)
return null;
ScannedRating rating = new ScannedRating();
rating.Score = Convert.ToByte(ratingClass.Replace("star_", ""));
rating.Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands);
return rating;
}
}

33
JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs
View File

@@ -1,15 +1,13 @@
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Common;
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Integrations.DLSite;
using JSMR.Infrastructure.Integrations.DLSite.Mapping;
using JSMR.Infrastructure.Integrations.DLSite.Models;
using JSMR.Infrastructure.Integrations.DLSite.Serialization;
using JSMR.Tests.Utilities;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Shouldly;
using System.Net;
using System.Net.Http;
using System.Text.Json;
namespace JSMR.Tests.Integrations.DLSite;
@@ -25,26 +23,25 @@ public class DLSiteClientTests
{
string productInfoJson = await ReadJsonResourceAsync("Product-Info.json");
HttpResponseMessage response = new()
{
Content = new StringContent(productInfoJson),
StatusCode = HttpStatusCode.OK
};
IHttpService httpService = Substitute.For<IHttpService>();
HttpClient httpClient = Substitute.For<HttpClient>();
httpClient.BaseAddress = new Uri("https://fake.site.com/");
//{ BaseAddress = new Uri("https://www.dlsite.com/") };
httpClient.SendAsync(Arg.Any<HttpRequestMessage>(), Arg.Any<HttpCompletionOption>(), CancellationToken.None)
.Returns(Task.FromResult(response));
httpService.GetStringAsync(Arg.Any<string>(), CancellationToken.None)
.Returns(Task.FromResult(productInfoJson));
var logger = Substitute.For<ILogger<DLSiteClient>>();
var client = new DLSiteClient(httpClient, logger);
var client = new DLSiteClient(httpService, logger);
var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163"], CancellationToken.None);
result.Count.ShouldBe(1);
result.ShouldContainKey("RJ01230163");
result["RJ01230163"].HasTrial.ShouldBeTrue();
result["RJ01230163"].HasDLPlay.ShouldBeTrue();
result["RJ01230163"].HasReviews.ShouldBeTrue();
result["RJ01230163"].SupportedLanguages.ShouldBe([Language.English]);
result["RJ01230163"].DownloadCount.ShouldBe(659);
result["RJ01230163"].WishlistCount.ShouldBe(380);
}
[Fact]

79
JSMR.Tests/Unit/DLSiteSearchFilterBuilderTests.cs Normal file
View File

@@ -0,0 +1,79 @@
using JSMR.Infrastructure.Scanning;
using Shouldly;
namespace JSMR.Tests.Unit;
public class DLSiteSearchFilterBuilderTests
{
[Fact]
public void Build_Simple_Query()
{
var filterBuilder = new DLSiteSearchFilterBuilder()
.UseDefaultLocale()
.IncludeJapaneseSupportedLanguage();
string url = filterBuilder.BuildSearchQuery(1, 100);
using var writer = new StringWriter();
writer.Write($"https://www.dlsite.com/maniax/");
writer.Write($"fsr/=/language/jp/");
writer.Write("sex_category[0]/male/");
writer.Write("ana_flg/all/");
writer.Write("work_category[0]/doujin/");
writer.Write("order[0]/release_d/");
writer.Write("work_type_category[0]/audio/");
writer.Write("work_type_category_name[0]/ボイス・ASMR/");
writer.Write("options_and_or/and/");
writer.Write($"options[0]/JPN/");
writer.Write($"per_page/100/");
writer.Write($"page/1/");
writer.Write("show_type/1/");
writer.Write($"?locale=ja_JP");
url.ShouldBe(writer.ToString());
}
[Fact]
public void Build_Advanced_Query()
{
var filterBuilder = new DLSiteSearchFilterBuilder()
.UseEnglishLocale()
.IncludeJapaneseSupportedLanguage()
.IncludeEnglishSupportedLanguage()
.ExcludeMakers(["RG0000001", "RG0000002", "", "RG0000001"])
.ExcludePartiallyAIGeneratedWorks()
.ExcludeAIGeneratedWorks();
string url = filterBuilder.BuildSearchQuery(1, 100);
using var writer = new StringWriter();
writer.Write($"https://www.dlsite.com/maniax/");
writer.Write($"fsr/=/language/en/");
writer.Write("sex_category[0]/male/");
writer.Write("ana_flg/all/");
writer.Write("work_category[0]/doujin/");
writer.Write("order[0]/release_d/");
writer.Write("work_type_category[0]/audio/");
writer.Write("work_type_category_name[0]/ボイス・ASMR/");
writer.Write("options_and_or/and/");
writer.Write($"options[0]/JPN/");
writer.Write($"options[1]/ENG/");
writer.Write($"keyword_maker_name/-RG0000001+-RG0000002/");
writer.Write($"options_not[0]/AIP/");
writer.Write($"options_not[1]/AIG/");
writer.Write($"per_page/100/");
writer.Write($"page/1/");
writer.Write("show_type/1/");
writer.Write($"?locale=en_US");
url.ShouldBe(writer.ToString());
}
}

14
JSMR.Tests/Unit/MySqlBooleanQueryTests.cs
View File

@@ -10,15 +10,15 @@ public class MySqlBooleanQueryTests
{
string normalizedValue = MySqlBooleanQuery.Normalize("value1 value2|value3 -value4 \"value 5\"");
normalizedValue.ShouldBe("+value1 +(value2|value3) -value4 +\"value 5\"");
normalizedValue.ShouldBe("+value1 +(value2 value3) -value4 +\"value 5\"");
}
[Fact]
public void Normalize_Unusual_Usage()
{
string normalizedValue = MySqlBooleanQuery.Normalize("+value1 +(value2|value3) -value4 +\"value 5\"");
string normalizedValue = MySqlBooleanQuery.Normalize("+value1 +(value2 value3) -value4 +\"value 5\"");
normalizedValue.ShouldBe("+value1 +(value2|value3) -value4 +\"value 5\"");
normalizedValue.ShouldBe("+value1 +(value2 value3) -value4 +\"value 5\"");
}
[Fact]
@@ -28,4 +28,12 @@ public class MySqlBooleanQueryTests
normalizedValue.ShouldBe("+value1 +value2 +value3");
}
[Fact]
public void Normalize_Bad_Or_Data()
{
string normalizedValue = MySqlBooleanQuery.Normalize("value1 | value2");
normalizedValue.ShouldBe("+value1 +value2");
}
}