From 3c0a39b32495964a0308248d6e0dd853b6c3fd30 Mon Sep 17 00:00:00 2001 From: Brian Bicknell Date: Thu, 11 Sep 2025 00:07:49 -0400 Subject: [PATCH] Initial implementation of voice works scanning. --- JSMR.Api/JSMR.Api.http | 2 +- .../Scanning/Contracts/DLSiteWork.cs | 25 +++ .../Scanning/Contracts/DLSiteWorkCategory.cs | 13 ++ .../Scanning/Contracts/DLSiteWorkType.cs | 7 + .../Scanning/Ports/IVoiceWorksScanner.cs | 8 + .../Scanning/ScanVoiceWorksHandler.cs | 26 +++ .../Scanning/ScanVoiceWorksRequest.cs | 5 + .../Scanning/ScanVoiceWorksResponse.cs | 7 + .../SetVoiceWorkFavoriteHandler.cs | 11 ++ .../SetVoiceWorkFavoriteRequest.cs | 3 + .../SetVoiceWorkFavoriteResponse.cs | 3 + .../VoiceWorks/Ports/IVoiceWorkWriter.cs | 8 + .../{ => Adapters}/DistributedCacheAdapter.cs | 2 +- .../{ => Adapters}/MemoryCacheAdapter.cs | 2 +- JSMR.Infrastructure/Caching/ICacheObject.cs | 55 ++++++ .../Common/Locales/EnglishLocale.cs | 7 + JSMR.Infrastructure/Common/Locales/ILocale.cs | 7 + .../Common/Locales/JapaneseLocale.cs | 7 + .../SupportedLanguages/AlingualLanguage.cs | 6 + .../SupportedLanguages/ChineseLanguage.cs | 6 + .../DLSiteOfficialTranslationLanguage.cs | 6 + .../SupportedLanguages/EnglishLanguage.cs | 6 + .../SupportedLanguages/ISupportedLanguage.cs | 6 + .../SupportedLanguages/JapaneseLanguage.cs | 6 + .../SupportedLanguages/KoreanLanguage.cs | 6 + .../SimplifiedChineseLanguage.cs | 6 + .../TraditionalChineseLanguage.cs | 6 + ...frastructureServiceCollectionExtensions.cs | 8 + .../VoiceWorks/MySqlBooleanQuery.cs | 13 +- .../VoiceWorks/VoiceWorkWriter.cs | 25 +++ JSMR.Infrastructure/Http/ApiClient.cs | 117 +++++++------ JSMR.Infrastructure/Http/HtmlLoader.cs | 16 ++ JSMR.Infrastructure/Http/HttpService.cs | 24 +++ JSMR.Infrastructure/Http/IHtmlLoader.cs | 8 + JSMR.Infrastructure/Http/IHttpService.cs | 7 + .../Integrations/Chobit/ChobitClient.cs | 4 +- .../Integrations/DLSite/DLSiteClient.cs | 8 +- .../JSMR.Infrastructure.csproj | 1 + .../Scanning/DLSiteSearchFilterBuilder.cs | 127 ++++++++++++++ .../DLSiteSearchFilterBuilderExtensions.cs | 52 ++++++ .../Scanning/EnglishVoiceWorksScanner.cs | 164 ++++++++++++++++++ .../Scanning/JapaneseVoiceWorksScanner.cs | 74 ++++++++ .../Scanning/Models/DLSiteHtmlDocument.cs | 37 ++++ .../Scanning/Models/DLSiteHtmlNode.cs | 156 +++++++++++++++++ .../Scanning/Models/ScannedRating.cs | 7 + .../Scanning/ScannerUtilities.cs | 49 ++++++ .../Scanning/VoiceWorksScanner.cs | 164 ++++++++++++++++++ .../Integrations/DLSite/DLSiteClientTests.cs | 33 ++-- .../Unit/DLSiteSearchFilterBuilderTests.cs | 79 +++++++++ JSMR.Tests/Unit/MySqlBooleanQueryTests.cs | 14 +- 50 files changed, 1351 insertions(+), 88 deletions(-) create mode 100644 JSMR.Application/Scanning/Contracts/DLSiteWork.cs create mode 100644 JSMR.Application/Scanning/Contracts/DLSiteWorkCategory.cs create mode 100644 JSMR.Application/Scanning/Contracts/DLSiteWorkType.cs create mode 100644 JSMR.Application/Scanning/Ports/IVoiceWorksScanner.cs create mode 100644 JSMR.Application/Scanning/ScanVoiceWorksHandler.cs create mode 100644 JSMR.Application/Scanning/ScanVoiceWorksRequest.cs create mode 100644 JSMR.Application/Scanning/ScanVoiceWorksResponse.cs create mode 100644 JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteHandler.cs create mode 100644 JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteRequest.cs create mode 100644 JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteResponse.cs create mode 100644 JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs rename JSMR.Infrastructure/Caching/{ => Adapters}/DistributedCacheAdapter.cs (96%) rename JSMR.Infrastructure/Caching/{ => Adapters}/MemoryCacheAdapter.cs (95%) create mode 100644 JSMR.Infrastructure/Caching/ICacheObject.cs create mode 100644 JSMR.Infrastructure/Common/Locales/EnglishLocale.cs create mode 100644 JSMR.Infrastructure/Common/Locales/ILocale.cs create mode 100644 JSMR.Infrastructure/Common/Locales/JapaneseLocale.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/AlingualLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/ChineseLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/DLSiteOfficialTranslationLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/EnglishLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/ISupportedLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/JapaneseLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/KoreanLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/SimplifiedChineseLanguage.cs create mode 100644 JSMR.Infrastructure/Common/SupportedLanguages/TraditionalChineseLanguage.cs create mode 100644 JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs create mode 100644 JSMR.Infrastructure/Http/HtmlLoader.cs create mode 100644 JSMR.Infrastructure/Http/HttpService.cs create mode 100644 JSMR.Infrastructure/Http/IHtmlLoader.cs create mode 100644 JSMR.Infrastructure/Http/IHttpService.cs create mode 100644 JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilder.cs create mode 100644 JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilderExtensions.cs create mode 100644 JSMR.Infrastructure/Scanning/EnglishVoiceWorksScanner.cs create mode 100644 JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs create mode 100644 JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs create mode 100644 JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs create mode 100644 JSMR.Infrastructure/Scanning/Models/ScannedRating.cs create mode 100644 JSMR.Infrastructure/Scanning/ScannerUtilities.cs create mode 100644 JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs create mode 100644 JSMR.Tests/Unit/DLSiteSearchFilterBuilderTests.cs diff --git a/JSMR.Api/JSMR.Api.http b/JSMR.Api/JSMR.Api.http index cb49ff2..081663e 100644 --- a/JSMR.Api/JSMR.Api.http +++ b/JSMR.Api/JSMR.Api.http @@ -45,7 +45,7 @@ Content-Type: {{contentType}} { "options": { "criteria": { - "keywords": "tsundere" + "keywords": "maid harem" }, "pageNumber": 1, "pageSize": 100, diff --git a/JSMR.Application/Scanning/Contracts/DLSiteWork.cs b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs new file mode 100644 index 0000000..b1867a8 --- /dev/null +++ b/JSMR.Application/Scanning/Contracts/DLSiteWork.cs @@ -0,0 +1,25 @@ +namespace JSMR.Application.Scanning.Contracts; + +public class DLSiteWork +{ + public DLSiteWorkType WorkType { get; set; } + public DLSiteWorkCategory Category { get; set; } + public string? ProductName { get; set; } + public string? ProductUrl { get; set; } + public string? ProductId { get; set; } + public DateOnly? AnnouncedDate { get; set; } + public DateTime? ExpectedDate { get; set; } + public DateTime? SalesDate { get; set; } + public int? Downloads { get; set; } + public byte? StarRating { get; set; } + public int? Votes { get; set; } + public string? Maker { get; set; } + public string? MakerId { get; set; } + public string? Description { get; set; } + public ICollection Genres { get; set; } = []; + public ICollection Tags { get; set; } = []; + public ICollection Creators { get; set; } = []; + public string? ImageUrl { get; set; } + public string? SmallImageUrl { get; set; } + public string? Type { get; set; } +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/Contracts/DLSiteWorkCategory.cs b/JSMR.Application/Scanning/Contracts/DLSiteWorkCategory.cs new file mode 100644 index 0000000..b97b577 --- /dev/null +++ b/JSMR.Application/Scanning/Contracts/DLSiteWorkCategory.cs @@ -0,0 +1,13 @@ +namespace JSMR.Application.Scanning.Contracts; + +public enum DLSiteWorkCategory +{ + Unknown, + VoiceASMR, + Manga, + RPG, + Video, + Adventure, + Simulation, + CG +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/Contracts/DLSiteWorkType.cs b/JSMR.Application/Scanning/Contracts/DLSiteWorkType.cs new file mode 100644 index 0000000..c00fc86 --- /dev/null +++ b/JSMR.Application/Scanning/Contracts/DLSiteWorkType.cs @@ -0,0 +1,7 @@ +namespace JSMR.Application.Scanning.Contracts; + +public enum DLSiteWorkType +{ + Released, + Announced +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/Ports/IVoiceWorksScanner.cs b/JSMR.Application/Scanning/Ports/IVoiceWorksScanner.cs new file mode 100644 index 0000000..ea12fb3 --- /dev/null +++ b/JSMR.Application/Scanning/Ports/IVoiceWorksScanner.cs @@ -0,0 +1,8 @@ +using JSMR.Application.Scanning.Contracts; + +namespace JSMR.Application.Scanning.Ports; + +public interface IVoiceWorksScanner +{ + Task> ScanPageAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken = default); +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs new file mode 100644 index 0000000..0554d6c --- /dev/null +++ b/JSMR.Application/Scanning/ScanVoiceWorksHandler.cs @@ -0,0 +1,26 @@ +using JSMR.Application.Scanning.Ports; + +namespace JSMR.Application.Scanning; + +public sealed class ScanVoiceWorksHandler(IVoiceWorksScanner scanner) +{ + //public async Task HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken) + //{ + // var works = await scanner.ScanPageAsync(request, cancellationToken); + + // if (works.Count == 0) + // return new ScanVoiceWorksResponse(); + + // var ingests = works.Select(VoiceWorkIngest.From).ToList(); + // var upsert = await _writer.UpsertAsync(ingests, ct); + + // // only update search text for affected rows + // await _search.UpdateAsync(upsert.AffectedVoiceWorkIds, ct); + + // return new ScanVoiceWorksResponse + // { + // Inserted = upsert.Inserted, + // Updated = upsert.Updated + // }; + //} +} \ No newline at end of file diff --git a/JSMR.Application/Scanning/ScanVoiceWorksRequest.cs b/JSMR.Application/Scanning/ScanVoiceWorksRequest.cs new file mode 100644 index 0000000..226b7e7 --- /dev/null +++ b/JSMR.Application/Scanning/ScanVoiceWorksRequest.cs @@ -0,0 +1,5 @@ +using JSMR.Application.Common; + +namespace JSMR.Application.Scanning; + +public sealed record ScanVoiceWorksRequest(int PageNumber, int PageSize, Locale Locale); \ No newline at end of file diff --git a/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs b/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs new file mode 100644 index 0000000..da872a5 --- /dev/null +++ b/JSMR.Application/Scanning/ScanVoiceWorksResponse.cs @@ -0,0 +1,7 @@ +namespace JSMR.Application.Scanning; + +public sealed class ScanVoiceWorksResponse +{ + public int Inserted { get; init; } + public int Updated { get; init; } +} \ No newline at end of file diff --git a/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteHandler.cs b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteHandler.cs new file mode 100644 index 0000000..7e05cdc --- /dev/null +++ b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteHandler.cs @@ -0,0 +1,11 @@ +using JSMR.Application.VoiceWorks.Ports; + +namespace JSMR.Application.VoiceWorks.Commands.SetFavorite; + +public sealed class SetVoiceWorkFavoriteHandler(IVoiceWorkWriter writer) +{ + public async Task HandleAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken = default) + { + return await writer.SetFavoriteAsync(request, cancellationToken); + } +} \ No newline at end of file diff --git a/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteRequest.cs b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteRequest.cs new file mode 100644 index 0000000..bffaca1 --- /dev/null +++ b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteRequest.cs @@ -0,0 +1,3 @@ +namespace JSMR.Application.VoiceWorks.Commands.SetFavorite; + +public sealed record SetVoiceWorkFavoriteRequest(int VoiceWorkId, bool IsFavorite); \ No newline at end of file diff --git a/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteResponse.cs b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteResponse.cs new file mode 100644 index 0000000..1565339 --- /dev/null +++ b/JSMR.Application/VoiceWorks/Commands/SetFavorite/SetVoiceWorkFavoriteResponse.cs @@ -0,0 +1,3 @@ +namespace JSMR.Application.VoiceWorks.Commands.SetFavorite; + +public sealed record SetVoiceWorkFavoriteResponse(int VoiceWorkId, bool IsFavorite); \ No newline at end of file diff --git a/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs b/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs new file mode 100644 index 0000000..50c70dd --- /dev/null +++ b/JSMR.Application/VoiceWorks/Ports/IVoiceWorkWriter.cs @@ -0,0 +1,8 @@ +using JSMR.Application.VoiceWorks.Commands.SetFavorite; + +namespace JSMR.Application.VoiceWorks.Ports; + +public interface IVoiceWorkWriter +{ + Task SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Caching/DistributedCacheAdapter.cs b/JSMR.Infrastructure/Caching/Adapters/DistributedCacheAdapter.cs similarity index 96% rename from JSMR.Infrastructure/Caching/DistributedCacheAdapter.cs rename to JSMR.Infrastructure/Caching/Adapters/DistributedCacheAdapter.cs index 94388da..c6e5e3b 100644 --- a/JSMR.Infrastructure/Caching/DistributedCacheAdapter.cs +++ b/JSMR.Infrastructure/Caching/Adapters/DistributedCacheAdapter.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Caching.Distributed; using System.Text; using System.Text.Json; -namespace JSMR.Infrastructure.Caching; +namespace JSMR.Infrastructure.Caching.Adapters; public class DistributedCacheAdapter(IDistributedCache distributedCache) : ICache { diff --git a/JSMR.Infrastructure/Caching/MemoryCacheAdapter.cs b/JSMR.Infrastructure/Caching/Adapters/MemoryCacheAdapter.cs similarity index 95% rename from JSMR.Infrastructure/Caching/MemoryCacheAdapter.cs rename to JSMR.Infrastructure/Caching/Adapters/MemoryCacheAdapter.cs index 7bd7bcf..fe00e05 100644 --- a/JSMR.Infrastructure/Caching/MemoryCacheAdapter.cs +++ b/JSMR.Infrastructure/Caching/Adapters/MemoryCacheAdapter.cs @@ -1,7 +1,7 @@ using JSMR.Application.Common.Caching; using Microsoft.Extensions.Caching.Memory; -namespace JSMR.Infrastructure.Caching; +namespace JSMR.Infrastructure.Caching.Adapters; public class MemoryCacheAdapter(IMemoryCache memoryCache) : ICache { diff --git a/JSMR.Infrastructure/Caching/ICacheObject.cs b/JSMR.Infrastructure/Caching/ICacheObject.cs new file mode 100644 index 0000000..60bbb63 --- /dev/null +++ b/JSMR.Infrastructure/Caching/ICacheObject.cs @@ -0,0 +1,55 @@ +using JSMR.Application.Common.Caching; +using JSMR.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; + +namespace JSMR.Infrastructure.Caching; + +public interface ICacheObject +{ + Task GetAsync(CancellationToken cancellationToken = default); + Task RefreshAsync(CancellationToken cancellationToken = default); +} + +public abstract class CacheObject(ICache cache) : ICacheObject +{ + protected abstract string Key { get; } + protected abstract Task FetchAsync(CancellationToken cancellationToken = default); + + protected virtual CacheEntryOptions Options => new() { SlidingExpiration = TimeSpan.FromHours(1) }; + + public async virtual Task GetAsync(CancellationToken cancellationToken = default) + { + return await cache.GetAsync(Key, cancellationToken) ?? await RefreshAsync(cancellationToken); + } + + public async virtual Task RefreshAsync(CancellationToken cancellationToken = default) + { + T cacheObject = await FetchAsync(cancellationToken); + + await cache.SetAsync(Key, cacheObject, Options, cancellationToken); + + return cacheObject; + } +} + +public interface ISpamCircleCache : ICacheObject +{ + +} + +public class SpamCircleCache(IDbContextFactory contextFactory, ICache cache) : CacheObject(cache), ISpamCircleCache +{ + protected override string Key => "SpamCircles"; + + protected override async Task FetchAsync(CancellationToken cancellationToken = default) + { + using var context = contextFactory.CreateDbContext(); + + return await context.Circles + .AsNoTracking() + .Where(circle => circle.Spam) + .Select(circle => circle.MakerId) + .Distinct() + .ToArrayAsync(cancellationToken); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/Locales/EnglishLocale.cs b/JSMR.Infrastructure/Common/Locales/EnglishLocale.cs new file mode 100644 index 0000000..6f1805a --- /dev/null +++ b/JSMR.Infrastructure/Common/Locales/EnglishLocale.cs @@ -0,0 +1,7 @@ +namespace JSMR.Infrastructure.Common.Locales; + +public class EnglishLocale : ILocale +{ + public string Abbreviation => "en"; + public string Code => "en_US"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/Locales/ILocale.cs b/JSMR.Infrastructure/Common/Locales/ILocale.cs new file mode 100644 index 0000000..64928b7 --- /dev/null +++ b/JSMR.Infrastructure/Common/Locales/ILocale.cs @@ -0,0 +1,7 @@ +namespace JSMR.Infrastructure.Common.Locales; + +public interface ILocale +{ + string Abbreviation { get; } + string Code { get; } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/Locales/JapaneseLocale.cs b/JSMR.Infrastructure/Common/Locales/JapaneseLocale.cs new file mode 100644 index 0000000..0585b40 --- /dev/null +++ b/JSMR.Infrastructure/Common/Locales/JapaneseLocale.cs @@ -0,0 +1,7 @@ +namespace JSMR.Infrastructure.Common.Locales; + +public class JapaneseLocale : ILocale +{ + public string Abbreviation => "jp"; + public string Code => "ja_JP"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/AlingualLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/AlingualLanguage.cs new file mode 100644 index 0000000..5c50001 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/AlingualLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class AlingualLanguage : ISupportedLanguage +{ + public string Code => "NM"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/ChineseLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/ChineseLanguage.cs new file mode 100644 index 0000000..ee95422 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/ChineseLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class ChineseLanguage : ISupportedLanguage +{ + public string Code => "CHI"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/DLSiteOfficialTranslationLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/DLSiteOfficialTranslationLanguage.cs new file mode 100644 index 0000000..39207ca --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/DLSiteOfficialTranslationLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class DLSiteOfficialTranslationLanguage : ISupportedLanguage +{ + public string Code => "DOT"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/EnglishLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/EnglishLanguage.cs new file mode 100644 index 0000000..095f217 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/EnglishLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class EnglishLanguage : ISupportedLanguage +{ + public string Code => "ENG"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/ISupportedLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/ISupportedLanguage.cs new file mode 100644 index 0000000..b50add6 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/ISupportedLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public interface ISupportedLanguage +{ + string Code { get; } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/JapaneseLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/JapaneseLanguage.cs new file mode 100644 index 0000000..440e52e --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/JapaneseLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class JapaneseLanguage : ISupportedLanguage +{ + public string Code => "JPN"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/KoreanLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/KoreanLanguage.cs new file mode 100644 index 0000000..49715b3 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/KoreanLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class KoreanLanguage : ISupportedLanguage +{ + public string Code => "KO_KR"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/SimplifiedChineseLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/SimplifiedChineseLanguage.cs new file mode 100644 index 0000000..28438d7 --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/SimplifiedChineseLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class SimplifiedChineseLanguage : ISupportedLanguage +{ + public string Code => "CHI_HANS"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Common/SupportedLanguages/TraditionalChineseLanguage.cs b/JSMR.Infrastructure/Common/SupportedLanguages/TraditionalChineseLanguage.cs new file mode 100644 index 0000000..7be0acc --- /dev/null +++ b/JSMR.Infrastructure/Common/SupportedLanguages/TraditionalChineseLanguage.cs @@ -0,0 +1,6 @@ +namespace JSMR.Infrastructure.Common.SupportedLanguages; + +public class TraditionalChineseLanguage : ISupportedLanguage +{ + public string Code => "CHI_HANT"; +} \ No newline at end of file diff --git a/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs b/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs index 2752e1d..a86f366 100644 --- a/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs +++ b/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs @@ -7,12 +7,15 @@ using JSMR.Application.Creators.Queries.Search.Ports; using JSMR.Application.Integrations.Ports; using JSMR.Application.Tags.Ports; using JSMR.Application.Tags.Queries.Search.Ports; +using JSMR.Application.VoiceWorks.Ports; using JSMR.Application.VoiceWorks.Queries.Search; using JSMR.Infrastructure.Caching; +using JSMR.Infrastructure.Caching.Adapters; using JSMR.Infrastructure.Data.Repositories.Circles; using JSMR.Infrastructure.Data.Repositories.Creators; using JSMR.Infrastructure.Data.Repositories.Tags; using JSMR.Infrastructure.Data.Repositories.VoiceWorks; +using JSMR.Infrastructure.Http; using JSMR.Infrastructure.Integrations.DLSite; using Microsoft.Extensions.DependencyInjection; @@ -28,6 +31,7 @@ public static class InfrastructureServiceCollectionExtensions services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -36,6 +40,10 @@ public static class InfrastructureServiceCollectionExtensions services.AddScoped(); services.AddSingleton(); + services.AddSingleton(); + + services.AddScoped(); + services.AddScoped(); return services; } diff --git a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/MySqlBooleanQuery.cs b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/MySqlBooleanQuery.cs index d68ea86..4df346d 100644 --- a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/MySqlBooleanQuery.cs +++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/MySqlBooleanQuery.cs @@ -4,10 +4,10 @@ namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; public static class MySqlBooleanQuery { - // Main entry public static string Normalize(string input) { - if (string.IsNullOrWhiteSpace(input)) return string.Empty; + if (string.IsNullOrWhiteSpace(input)) + return string.Empty; // Split into top-level tokens by spaces (not inside quotes/parentheses) var tokens = SplitTopLevel(input.Trim(), ' '); @@ -16,7 +16,12 @@ public static class MySqlBooleanQuery foreach (var raw in tokens) { var t = raw.Trim(); - if (t.Length == 0) continue; + + if (t.Length == 0) + continue; + + if (t is "|") + continue; // Preserve explicit boolean operators user may already supply if (t[0] == '-' || t[0] == '+') @@ -63,7 +68,7 @@ public static class MySqlBooleanQuery var orParts = SplitTopLevel(token, '|') .Select(p => NormalizeOrAtom(p.Trim())) .Where(p => p.Length > 0); - return "(" + string.Join("|", orParts) + ")"; + return "(" + string.Join(" ", orParts) + ")"; } // Plain atom -> as-is diff --git a/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs new file mode 100644 index 0000000..32eaa3b --- /dev/null +++ b/JSMR.Infrastructure/Data/Repositories/VoiceWorks/VoiceWorkWriter.cs @@ -0,0 +1,25 @@ +using JSMR.Application.VoiceWorks.Commands.SetFavorite; +using JSMR.Application.VoiceWorks.Ports; +using JSMR.Domain.Entities; +using Microsoft.EntityFrameworkCore; + +namespace JSMR.Infrastructure.Data.Repositories.VoiceWorks; + +public class VoiceWorkWriter(AppDbContext context) : IVoiceWorkWriter +{ + public async Task SetFavoriteAsync(SetVoiceWorkFavoriteRequest request, CancellationToken cancellationToken) + { + VoiceWork voiceWork = await GetVoiceWorkAsync(request.VoiceWorkId, cancellationToken); + voiceWork.Favorite = request.IsFavorite; + + await context.SaveChangesAsync(cancellationToken); + + return new SetVoiceWorkFavoriteResponse(request.VoiceWorkId, request.IsFavorite); + } + + private async Task GetVoiceWorkAsync(int voiceWorkId, CancellationToken cancellationToken) + { + return await context.VoiceWorks.FirstOrDefaultAsync(voiceWork => voiceWork.VoiceWorkId == voiceWorkId, cancellationToken) + ?? throw new KeyNotFoundException($"Voice Work {voiceWorkId} not found."); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Http/ApiClient.cs b/JSMR.Infrastructure/Http/ApiClient.cs index ff8b485..5365e1e 100644 --- a/JSMR.Infrastructure/Http/ApiClient.cs +++ b/JSMR.Infrastructure/Http/ApiClient.cs @@ -1,79 +1,88 @@ using Microsoft.Extensions.Logging; +using System.IO; using System.Net.Http.Headers; using System.Text; using System.Text.Json; namespace JSMR.Infrastructure.Http; -public abstract class ApiClient(HttpClient http, ILogger logger, JsonSerializerOptions? json = null) +public abstract class ApiClient(IHttpService http, ILogger logger, JsonSerializerOptions? json = null) { - protected async Task GetJsonAsync( - string url, - Action? configureHeaders = null, - CancellationToken ct = default - ) + protected async Task GetJsonAsync(string url, CancellationToken cancellationToken = default) { - using var req = new HttpRequestMessage(HttpMethod.Get, url); - configureHeaders?.Invoke(req.Headers); + string response = await http.GetStringAsync(url, cancellationToken); - LogRequest(req); - - using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); - await EnsureSuccess(res).ConfigureAwait(false); - - var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); - - var model = await JsonSerializer.DeserializeAsync(stream, json, ct).ConfigureAwait(false) + return JsonSerializer.Deserialize(response, json) ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}."); - - return model; } - protected async Task PostJsonAsync( - string url, - TRequest payload, - Action? configureHeaders = null, - CancellationToken ct = default) - { - var content = new StringContent(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json"); - using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; - configureHeaders?.Invoke(req.Headers); + //protected async Task GetJsonAsync( + // string url, + // Action? configureHeaders = null, + // CancellationToken ct = default + // ) + //{ + // using var req = new HttpRequestMessage(HttpMethod.Get, url); + // configureHeaders?.Invoke(req.Headers); - LogRequest(req); + // LogRequest(req); - using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); - await EnsureSuccess(res).ConfigureAwait(false); + // using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + // await EnsureSuccess(res).ConfigureAwait(false); - var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + // var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); - var model = await JsonSerializer.DeserializeAsync(stream, json, ct).ConfigureAwait(false) - ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}."); - - return model; - } + // var model = await JsonSerializer.DeserializeAsync(stream, json, ct).ConfigureAwait(false) + // ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}."); - protected virtual void LogRequest(HttpRequestMessage req) - => logger.LogDebug("HTTP {Method} {Uri}", req.Method, req.RequestUri); + // return model; + //} - protected virtual void LogFailure(HttpResponseMessage res, string body) - => logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)res.StatusCode, res.RequestMessage?.RequestUri, Truncate(body, 500)); + //protected async Task PostJsonAsync( + // string url, + // TRequest payload, + // Action? configureHeaders = null, + // CancellationToken ct = default) + //{ + // var content = new StringContent(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json"); + // using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; + // configureHeaders?.Invoke(req.Headers); - protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…"; + // LogRequest(req); - protected async Task EnsureSuccess(HttpResponseMessage res) - { - if (res.IsSuccessStatusCode) return; + // using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + // await EnsureSuccess(res).ConfigureAwait(false); - string body; - try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); } - catch { body = ""; } + // var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); - LogFailure(res, body); + // var model = await JsonSerializer.DeserializeAsync(stream, json, ct).ConfigureAwait(false) + // ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}."); - // Throw a richer exception (you can customize per API) - throw new HttpRequestException( - $"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}", - null, - res.StatusCode); - } + // return model; + //} + + //protected virtual void LogRequest(HttpRequestMessage req) + // => logger.LogDebug("HTTP {Method} {Uri}", req.Method, req.RequestUri); + + //protected virtual void LogFailure(HttpResponseMessage res, string body) + // => logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)res.StatusCode, res.RequestMessage?.RequestUri, Truncate(body, 500)); + + //protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…"; + + //protected async Task EnsureSuccess(HttpResponseMessage res) + //{ + // if (res.IsSuccessStatusCode) return; + + // string body; + // try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); } + // catch { body = ""; } + + // LogFailure(res, body); + + // Throw a richer exception(you can customize per API) + // throw new HttpRequestException( + // $"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}", + // null, + // res.StatusCode); + //} } \ No newline at end of file diff --git a/JSMR.Infrastructure/Http/HtmlLoader.cs b/JSMR.Infrastructure/Http/HtmlLoader.cs new file mode 100644 index 0000000..fc9091a --- /dev/null +++ b/JSMR.Infrastructure/Http/HtmlLoader.cs @@ -0,0 +1,16 @@ +using HtmlAgilityPack; + +namespace JSMR.Infrastructure.Http; + +public class HtmlLoader(IHttpService httpService) : IHtmlLoader +{ + public async Task GetHtmlDocumentAsync(string url, CancellationToken cancellationToken) + { + string html = await httpService.GetStringAsync(url, cancellationToken); + + HtmlDocument document = new(); + document.LoadHtml(html); + + return document; + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Http/HttpService.cs b/JSMR.Infrastructure/Http/HttpService.cs new file mode 100644 index 0000000..d495618 --- /dev/null +++ b/JSMR.Infrastructure/Http/HttpService.cs @@ -0,0 +1,24 @@ +namespace JSMR.Infrastructure.Http; + +public class HttpService(HttpClient httpClient) : IHttpService +{ + public Task GetStringAsync(string url, CancellationToken cancellationToken) + => GetStringAsync(url, new Dictionary(), cancellationToken); + + public async Task GetStringAsync(string url, IDictionary headers, CancellationToken cancellationToken) + { + using HttpRequestMessage request = new(HttpMethod.Get, url); + + foreach (KeyValuePair header in headers) + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0"); + + using HttpResponseMessage response = await httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsStringAsync(cancellationToken); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Http/IHtmlLoader.cs b/JSMR.Infrastructure/Http/IHtmlLoader.cs new file mode 100644 index 0000000..c1753bf --- /dev/null +++ b/JSMR.Infrastructure/Http/IHtmlLoader.cs @@ -0,0 +1,8 @@ +using HtmlAgilityPack; + +namespace JSMR.Infrastructure.Http; + +public interface IHtmlLoader +{ + Task GetHtmlDocumentAsync(string url, CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Http/IHttpService.cs b/JSMR.Infrastructure/Http/IHttpService.cs new file mode 100644 index 0000000..1cb8d5e --- /dev/null +++ b/JSMR.Infrastructure/Http/IHttpService.cs @@ -0,0 +1,7 @@ +namespace JSMR.Infrastructure.Http; + +public interface IHttpService +{ + Task GetStringAsync(string url, CancellationToken cancellationToken); + Task GetStringAsync(string url, IDictionary headers, CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs b/JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs index ec8d943..37edbe8 100644 --- a/JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs +++ b/JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs @@ -5,11 +5,11 @@ using Microsoft.Extensions.Logging; namespace JSMR.Infrastructure.Integrations.Chobit; -public class ChobitClient(HttpClient http, ILogger logger) : ApiClient(http, logger), IChobitClient +public class ChobitClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IChobitClient { public Task GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default) { var url = $"api/v2/dlsite/embed?workno_list=${string.Join(",", productIds)}"; - return GetJsonAsync(url, ct: cancellationToken); + return GetJsonAsync(url, cancellationToken); } } \ No newline at end of file diff --git a/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs b/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs index 3524b0b..0406b9a 100644 --- a/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs +++ b/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs @@ -7,21 +7,21 @@ using Microsoft.Extensions.Logging; namespace JSMR.Infrastructure.Integrations.DLSite; -public class DLSiteClient(HttpClient http, ILogger logger) : ApiClient(http, logger), IDLSiteClient +public class DLSiteClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IDLSiteClient { public async Task GetVoiceWorkDetailsAsync(string[] productIds, CancellationToken cancellationToken = default) { if (productIds.Length == 0) - throw new Exception("No products to get product information."); + return []; string productIdCollection = string.Join(",", productIds.Where(x => !string.IsNullOrWhiteSpace(x))); if (string.IsNullOrWhiteSpace(productIdCollection)) - throw new Exception("Invalid product id(s)."); + return []; string url = $"maniax/product/info/ajax?product_id={productIdCollection}&cdn_cache_min=1"; - var productInfoCollection = await GetJsonAsync(url, ct: cancellationToken); + var productInfoCollection = await GetJsonAsync(url, cancellationToken); return DLSiteToDomainMapper.Map(productInfoCollection); } diff --git a/JSMR.Infrastructure/JSMR.Infrastructure.csproj b/JSMR.Infrastructure/JSMR.Infrastructure.csproj index 649c1bb..6b50e65 100644 --- a/JSMR.Infrastructure/JSMR.Infrastructure.csproj +++ b/JSMR.Infrastructure/JSMR.Infrastructure.csproj @@ -7,6 +7,7 @@ + diff --git a/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilder.cs b/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilder.cs new file mode 100644 index 0000000..b060383 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilder.cs @@ -0,0 +1,127 @@ +using JSMR.Infrastructure.Common.Locales; +using JSMR.Infrastructure.Common.SupportedLanguages; + +namespace JSMR.Infrastructure.Scanning; + +public class DLSiteSearchFilterBuilder +{ + private readonly List _optionsAnd = []; + private readonly List _optionsNot = []; + private readonly List _excludedMakers = []; + + private ILocale _locale = new JapaneseLocale(); + + private void AddToOptionsAnd(string value) + { + if (_optionsAnd.Contains(value)) + return; + + _optionsAnd.Add(value); + } + + private void AddToOptionsNot(string value) + { + if (_optionsNot.Contains(value)) + return; + + _optionsNot.Add(value); + } + + public DLSiteSearchFilterBuilder UseLocale(ILocale locale) + { + _locale = locale; + + return this; + } + + public DLSiteSearchFilterBuilder IncludeSupportedLanguage(ISupportedLanguage language) + { + AddToOptionsAnd(language.Code); + + return this; + } + + public DLSiteSearchFilterBuilder ExcludeMakers(string[] makerIds) + { + foreach (var makerId in makerIds) + ExcludeMaker(makerId); + + return this; + } + + public DLSiteSearchFilterBuilder ExcludeMaker(string makerId) + { + if (string.IsNullOrWhiteSpace(makerId)) + return this; + + string trimmedMakerId = makerId.Trim(); + + if (_excludedMakers.Contains(trimmedMakerId)) + return this; + + _excludedMakers.Add(trimmedMakerId); + + return this; + } + + public DLSiteSearchFilterBuilder ExcludePartiallyAIGeneratedWorks() + { + AddToOptionsNot("AIP"); + + return this; + } + + public DLSiteSearchFilterBuilder ExcludeAIGeneratedWorks() + { + AddToOptionsNot("AIG"); + + return this; + } + + public string BuildSearchQuery(int pageNumber, int pageSize) + { + ILocale locale = _locale ?? new JapaneseLocale(); + + using (var writer = new StringWriter()) + { + writer.Write($"https://www.dlsite.com/maniax/"); + writer.Write($"fsr/=/language/{locale.Abbreviation}/"); + + writer.Write("sex_category[0]/male/"); + writer.Write("ana_flg/all/"); + writer.Write("work_category[0]/doujin/"); + writer.Write("order[0]/release_d/"); + writer.Write("work_type_category[0]/audio/"); + writer.Write("work_type_category_name[0]/ボイス・ASMR/"); + + if (_optionsAnd.Count > 0) + { + writer.Write("options_and_or/and/"); + + for (int index = 0; index < _optionsAnd.Count; index++) + { + writer.Write($"options[{index}]/{_optionsAnd[index]}/"); + } + } + + if (_excludedMakers.Count > 0) + { + List spamMakers = [.. _excludedMakers.Select(x => "-" + x)]; + string makerFilterValue = string.Join("+", spamMakers).Trim(); + writer.Write($"keyword_maker_name/{makerFilterValue}/"); + } + + for (int index = 0; index < _optionsNot.Count; index++) + { + writer.Write($"options_not[{index}]/{_optionsNot[index]}/"); + } + + writer.Write($"per_page/{pageSize}/"); + writer.Write($"page/{pageNumber}/"); + writer.Write("show_type/1/"); + writer.Write($"?locale={locale.Code}"); + + return writer.ToString(); + } + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilderExtensions.cs b/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilderExtensions.cs new file mode 100644 index 0000000..3404f73 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/DLSiteSearchFilterBuilderExtensions.cs @@ -0,0 +1,52 @@ +using JSMR.Infrastructure.Common.Locales; +using JSMR.Infrastructure.Common.SupportedLanguages; + +namespace JSMR.Infrastructure.Scanning; + +public static class DLSiteSearchFilterBuilderExtensions +{ + public static DLSiteSearchFilterBuilder UseDefaultLocale(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.UseLocale(new JapaneseLocale()); + } + + public static DLSiteSearchFilterBuilder UseEnglishLocale(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.UseLocale(new EnglishLocale()); + } + + public static DLSiteSearchFilterBuilder IncludeJapaneseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new JapaneseLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeEnglishSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new EnglishLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new ChineseLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeSimplifiedChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new SimplifiedChineseLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeTraditionalChineseSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new TraditionalChineseLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeKoreanSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new KoreanLanguage()); + } + + public static DLSiteSearchFilterBuilder IncludeAlingualSupportedLanguage(this DLSiteSearchFilterBuilder searchFilterBuilder) + { + return searchFilterBuilder.IncludeSupportedLanguage(new AlingualLanguage()); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/EnglishVoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/EnglishVoiceWorksScanner.cs new file mode 100644 index 0000000..66a5e2e --- /dev/null +++ b/JSMR.Infrastructure/Scanning/EnglishVoiceWorksScanner.cs @@ -0,0 +1,164 @@ +using JSMR.Application.Scanning; +using JSMR.Infrastructure.Caching; +using JSMR.Infrastructure.Common.Locales; +using JSMR.Infrastructure.Common.SupportedLanguages; +using JSMR.Infrastructure.Http; +using System.Text.RegularExpressions; + +namespace JSMR.Infrastructure.Scanning; + +public partial class EnglishVoiceWorksScanner(IHtmlLoader loader, ISpamCircleCache spamCircleCache) + : VoiceWorksScanner(loader, spamCircleCache) +{ + [GeneratedRegex(@"Release: (.*?)[/](\d{2})[/](\d{4})", RegexOptions.IgnoreCase, "en-US")] + private static partial Regex SalesDateRegex(); + + [GeneratedRegex(@"^(Early|Middle|Late)\s(.*?)\s(\d{4})", RegexOptions.IgnoreCase, "en-US")] + private static partial Regex EstimatedDateRegex(); + + protected override ILocale Locale => new EnglishLocale(); + + protected override ISupportedLanguage[] SupportedLanguages => + [ + new JapaneseLanguage(), + new EnglishLanguage(), + new AlingualLanguage() + ]; + + protected override DateTime? GetEstimatedReleaseDate(string expectedDate) + { + if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定")) + return null; + + Regex textRegex = EstimatedDateRegex(); + MatchCollection textMatches = textRegex.Matches(expectedDate); + + if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) + return null; + + GroupCollection groups = textMatches[0].Groups; + + int releaseYear = Convert.ToInt32(groups[3].Value); + + int releaseMonth = 1; + int releaseDay = 1; + + string releaseTime = groups[1].Value; + string releaseMonthText = groups[2].Value; + + switch (releaseTime) + { + case "Early": + releaseDay = 1; + break; + case "Middle": + releaseDay = 11; + break; + case "Late": + releaseDay = 21; + break; + } + + switch (releaseMonthText) + { + case "Jan.": + releaseMonth = 1; + break; + case "Feb.": + releaseMonth = 2; + break; + case "Mar.": + releaseMonth = 3; + break; + case "Apr.": + releaseMonth = 4; + break; + case "May.": + releaseMonth = 5; + break; + case "Jun.": + releaseMonth = 6; + break; + case "Jul.": + releaseMonth = 7; + break; + case "Aug.": + releaseMonth = 8; + break; + case "Sep.": + releaseMonth = 9; + break; + case "Oct.": + releaseMonth = 10; + break; + case "Nov.": + releaseMonth = 11; + break; + case "Dec.": + releaseMonth = 12; + break; + } + + return new DateTime(releaseYear, releaseMonth, releaseDay); + } + + protected override DateTime? GetSalesDate(string salesDate) + { + Regex textRegex = SalesDateRegex(); + MatchCollection textMatches = textRegex.Matches(salesDate); + + if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) + return null; + + string month = textMatches[0].Groups[1].Value; + int releaseMonth = -1; + + switch (month) + { + case "Jan": + releaseMonth = 1; + break; + case "Feb": + releaseMonth = 2; + break; + case "Mar": + releaseMonth = 3; + break; + case "Apr": + releaseMonth = 4; + break; + case "May": + releaseMonth = 5; + break; + case "Jun": + releaseMonth = 6; + break; + case "Jul": + releaseMonth = 7; + break; + case "Aug": + releaseMonth = 8; + break; + case "Sep": + releaseMonth = 9; + break; + case "Oct": + releaseMonth = 10; + break; + case "Nov": + releaseMonth = 11; + break; + case "Dec": + releaseMonth = 12; + break; + } + + if (releaseMonth == -1) + return null; + + int releaseYear = Convert.ToInt32(textMatches[0].Groups[3].Value); + int releaseDay = Convert.ToInt32(textMatches[0].Groups[2].Value); + + return new DateTime(releaseYear, releaseMonth, releaseDay); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs new file mode 100644 index 0000000..35ece96 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/JapaneseVoiceWorksScanner.cs @@ -0,0 +1,74 @@ +using JSMR.Infrastructure.Caching; +using JSMR.Infrastructure.Common.Locales; +using JSMR.Infrastructure.Common.SupportedLanguages; +using JSMR.Infrastructure.Http; +using System.Text.RegularExpressions; + +namespace JSMR.Infrastructure.Scanning; + +public class JapaneseVoiceWorksScanner(IHtmlLoader loader, ISpamCircleCache spamCircleCache) + : VoiceWorksScanner(loader, spamCircleCache) +{ + protected override ILocale Locale => new JapaneseLocale(); + + protected override ISupportedLanguage[] SupportedLanguages => + [ + new JapaneseLanguage(), + new EnglishLanguage(), + new TraditionalChineseLanguage(), + new SimplifiedChineseLanguage(), + new KoreanLanguage(), + new AlingualLanguage() + ]; + + protected override DateTime? GetEstimatedReleaseDate(string expectedDate) + { + if (expectedDate.Contains("販売中") || expectedDate.Contains("発売予定未定")) + return null; + + Regex textRegex = new Regex("(.*?)年(.*?)月(.*)", RegexOptions.IgnoreCase); + MatchCollection textMatches = textRegex.Matches(expectedDate); + + if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) + return null; + + int releaseYear = Convert.ToInt32(textMatches[0].Groups[1].Value); + int releaseMonth = Convert.ToInt32(textMatches[0].Groups[2].Value); + int releaseDay = 1; + + string releaseTime = textMatches[0].Groups[3].Value; + + switch (releaseTime) + { + case "上旬発売予定": + case "上旬 発売予定": + releaseDay = 1; + break; + case "中旬発売予定": + case "中旬 発売予定": + releaseDay = 11; + break; + case "下旬発売予定": + case "下旬 発売予定": + releaseDay = 21; + break; + } + + return new DateTime(releaseYear, releaseMonth, releaseDay); + } + + protected override DateTime? GetSalesDate(string salesDate) + { + Regex textRegex = new Regex("販売日: (.*?)年(.*?)月(.*)日", RegexOptions.IgnoreCase); + MatchCollection textMatches = textRegex.Matches(salesDate); + + if (textMatches.Count == 0 || textMatches[0].Groups.Count < 4) + return null; + + int releaseYear = Convert.ToInt32(textMatches[0].Groups[1].Value); + int releaseMonth = Convert.ToInt32(textMatches[0].Groups[2].Value); + int releaseDay = Convert.ToInt32(textMatches[0].Groups[3].Value); + + return new DateTime(releaseYear, releaseMonth, releaseDay); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs new file mode 100644 index 0000000..1dbf75b --- /dev/null +++ b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs @@ -0,0 +1,37 @@ +using HtmlAgilityPack; + +namespace JSMR.Infrastructure.Scanning.Models; + +public class DLSiteHtmlDocument +{ + private readonly HtmlNodeCollection _workColumns; + private readonly HtmlNodeCollection _workColumnRights; + private readonly HtmlNodeCollection _workThumbs; + + public HtmlNode PageTotalNode { get; } + + public DLSiteHtmlDocument(HtmlDocument document) + { + _workColumns = document.DocumentNode.SelectNodes("//dl[@class='work_1col']"); + _workColumnRights = document.DocumentNode.SelectNodes("//td[@class='work_1col_right']"); + _workThumbs = document.DocumentNode.SelectNodes("//div[@class='work_thumb']"); + + PageTotalNode = document.DocumentNode.SelectNodes("//div[@class='page_total']/strong")[0]; + } + + public List GetDLSiteNodes() + { + var nodes = new List(); + + if (_workColumns.Count != _workColumnRights.Count || _workColumns.Count != _workThumbs.Count) + throw new Exception("Work column node counts do not match!"); + + for (int i = 0; i < _workColumns.Count; i++) + { + var node = new DLSiteHtmlNode(_workColumns[i], _workColumnRights[i], _workThumbs[i]); + nodes.Add(node); + } + + return nodes; + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs new file mode 100644 index 0000000..f6c5b25 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs @@ -0,0 +1,156 @@ +using HtmlAgilityPack; + +namespace JSMR.Infrastructure.Scanning.Models; + +public class DLSiteHtmlNode +{ + public HtmlNode LeftNode { get; } + public HtmlNode RightNode { get; } + public HtmlNode ThumbNode { get; } + + public HtmlNode ProductNode { get; private set; } + public HtmlNode ProductLinkNode { get; private set; } + public HtmlNode ProductTextNode { get; private set; } + public HtmlNode DescriptionNode { get; private set; } + public HtmlNode MakerNode { get; private set; } + public HtmlNode MakerLinkNode { get; private set; } + public HtmlNode SalesDateNode { get; private set; } + public HtmlNode ExpectedDateNode { get; private set; } + public HtmlNode DownloadsNode { get; private set; } + public HtmlNode StarRatingNode { get; private set; } + public HtmlNode ImageNode { get; private set; } + public List GenreNodes { get; private set; } + public List SearchTagNodes { get; private set; } + public List CreatorNodes { get; private set; } + + public DLSiteHtmlNode(HtmlNode leftNode, HtmlNode rightNode, HtmlNode thumbNode) + { + LeftNode = leftNode; + RightNode = rightNode; + ThumbNode = thumbNode; + + ProductNode = LeftNode.SelectNodes(".//dt[@class='work_name']")[0]; + ProductLinkNode = ProductNode.SelectNodes(".//a")[0]; + ProductTextNode = GetProductTextNode(); + + DescriptionNode = LeftNode.SelectNodes(".//dd[@class='work_text']")[0]; + + MakerNode = LeftNode.SelectNodes(".//dd[@class='maker_name']")[0]; + MakerLinkNode = MakerNode.SelectNodes(".//a[contains(@href, 'maker_id')]")[0]; + + ExpectedDateNode = GetExpectedDateNode(); + + InitializeGenreNodes(); + InitializeSearchTagNodes(); + InitializeCreatorNodes(); + InitializeSalesAndDownloadsNodes(); + InitializeStarRatingNode(); + InitializeImageNode(); + } + + private void InitializeGenreNodes() + { + HtmlNode genreNode = LeftNode.SelectNodes(".//dd[@class='work_genre']")[0]; + + GenreNodes = [.. genreNode.SelectNodes(".//span")]; + } + + private void InitializeSearchTagNodes() + { + HtmlNodeCollection searchTagNodes = LeftNode.SelectNodes(".//dd[@class='search_tag']"); + + if (searchTagNodes == null || searchTagNodes.Count == 0) + { + SearchTagNodes = []; + } + else + { + HtmlNodeCollection searchTagNodesLinks = searchTagNodes[0].SelectNodes(".//a"); + + if (searchTagNodesLinks == null || searchTagNodesLinks.Count == 0) + { + SearchTagNodes = []; + } + else + { + SearchTagNodes = [.. searchTagNodesLinks]; + } + } + } + + private void InitializeCreatorNodes() + { + HtmlNodeCollection creatorNodes = MakerNode.SelectNodes(".//a[contains(@href, 'keyword_creater')]"); + + if (creatorNodes == null || creatorNodes.Count == 0) + { + CreatorNodes = []; + } + else + { + CreatorNodes = [.. creatorNodes]; + } + } + + private void InitializeSalesAndDownloadsNodes() + { + HtmlNodeCollection workInfoBox = RightNode.SelectNodes(".//ul[@class='work_info_box']"); + + if (workInfoBox != null) + { + HtmlNodeCollection salesDateNodes = workInfoBox[0].SelectNodes(".//li[@class='sales_date']"); + + if (salesDateNodes != null && salesDateNodes.Count > 0) + { + SalesDateNode = salesDateNodes[0]; + } + + // TODO: Fix! + //DownloadsNode = RightNode.SelectSingleNode(".//span[@class='_dl_count_" + works[rightsIndex].ProductId + "']"); + DownloadsNode = RightNode.SelectSingleNode(".//span[contains(@class, '_dl_count_')]"); + } + } + + private void InitializeStarRatingNode() + { + var ratingsNode = RightNode.SelectSingleNode(".//li[@class='work_rating']"); + + if (ratingsNode == null) + return; + + StarRatingNode = ratingsNode.SelectSingleNode(".//div[contains(@class, 'star_rating')]"); + } + + private HtmlNode GetProductTextNode() + { + if (ProductLinkNode.ChildNodes.Count > 1 && ProductLinkNode.ChildNodes[0].Name == "#text") + { + return ProductLinkNode.ChildNodes[0]; + } + else + { + return ProductLinkNode; + } + } + + private HtmlNode GetExpectedDateNode() + { + HtmlNodeCollection expectedDateNodes = ProductNode.SelectNodes(".//p[@class='expected_date']"); + + if (expectedDateNodes != null && expectedDateNodes.Count > 0) + { + return expectedDateNodes[0]; + } + else + { + return null; + } + } + + private void InitializeImageNode() + { + HtmlNode linkNode = ThumbNode.SelectNodes(".//a")[0]; + + ImageNode = linkNode.SelectNodes(".//img")[0]; + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/Models/ScannedRating.cs b/JSMR.Infrastructure/Scanning/Models/ScannedRating.cs new file mode 100644 index 0000000..1e46537 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/Models/ScannedRating.cs @@ -0,0 +1,7 @@ +namespace JSMR.Infrastructure.Scanning.Models; + +public class ScannedRating +{ + public byte Score { get; set; } + public int Votes { get; set; } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/ScannerUtilities.cs b/JSMR.Infrastructure/Scanning/ScannerUtilities.cs new file mode 100644 index 0000000..3b2e118 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/ScannerUtilities.cs @@ -0,0 +1,49 @@ +using HtmlAgilityPack; +using System.Web; + +namespace JSMR.Infrastructure.Scanning; + +public static class ScannerUtilities +{ + public static List GetStringListFromNodes(List nodes) + { + return nodes + .Where(node => string.IsNullOrEmpty(node.InnerHtml) == false) + .Select(node => HttpUtility.HtmlDecode(node.InnerHtml)) + .ToList(); + } + + public static string GetDecodedText(HtmlNode node) + { + if (node == null) + return string.Empty; + + if (string.IsNullOrWhiteSpace(node.InnerHtml)) + return string.Empty; + + return HttpUtility.HtmlDecode(node.InnerHtml.Replace("\n", "")).Trim(); + } + + public static string GetTextBetween(string text, string startText, string endText) + { + int startIndex = text.IndexOf(startText) + startText.Length; + int endIndex = text.IndexOf(endText); + + int length = endIndex - startIndex; + + if (length <= 0) + return ""; + + return text.Substring(startIndex, length); + } + + public static string GetImageSource(HtmlNode imageNode) + { + string imageSource = imageNode.GetAttributeValue("src", ""); + + if (string.IsNullOrEmpty(imageSource)) + imageSource = imageNode.GetAttributeValue("data-src", ""); + + return imageSource; + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs new file mode 100644 index 0000000..4bf25a5 --- /dev/null +++ b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs @@ -0,0 +1,164 @@ +using HtmlAgilityPack; +using JSMR.Application.Scanning; +using JSMR.Application.Scanning.Contracts; +using JSMR.Application.Scanning.Ports; +using JSMR.Infrastructure.Caching; +using JSMR.Infrastructure.Common.Locales; +using JSMR.Infrastructure.Common.SupportedLanguages; +using JSMR.Infrastructure.Http; +using JSMR.Infrastructure.Scanning.Models; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace JSMR.Infrastructure.Scanning; + +public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader, ISpamCircleCache spamCircleCache) : IVoiceWorksScanner +{ + protected abstract ILocale Locale { get; } + protected abstract ISupportedLanguage[] SupportedLanguages { get; } + + protected abstract DateTime? GetEstimatedReleaseDate(string expectedDate); + protected abstract DateTime? GetSalesDate(string salesDate); + + protected virtual bool ExcludeSpamCircles => true; + protected virtual bool ExcludePartiallyAIGeneratedWorks => true; + protected virtual bool ExcludeAIGeneratedWorks => true; + + public async Task> ScanPageAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken = default) + { + DLSiteHtmlDocument document = await GetDLSiteHtmlCollectionAsync(request, cancellationToken); + List nodes = document.GetDLSiteNodes(); + + return GetDLSiteWorks(nodes); + } + + private async Task GetDLSiteHtmlCollectionAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken) + { + string url = await GetUrlAsync(request, cancellationToken); + + HtmlDocument document = await htmlLoader.GetHtmlDocumentAsync(url, cancellationToken); + + return new DLSiteHtmlDocument(document); + } + + protected virtual async ValueTask GetUrlAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken) + { + DLSiteSearchFilterBuilder filterBuilder = new(); + + foreach (ISupportedLanguage supprotedLanguage in SupportedLanguages) + { + filterBuilder.IncludeSupportedLanguage(supprotedLanguage); + } + + if (ExcludeSpamCircles) + { + string[] makerIds = await spamCircleCache.GetAsync(cancellationToken); + + foreach (string makerId in makerIds) + filterBuilder.ExcludeMaker(makerId); + } + + if (ExcludePartiallyAIGeneratedWorks) + filterBuilder.ExcludePartiallyAIGeneratedWorks(); + + if (ExcludeAIGeneratedWorks) + filterBuilder.ExcludeAIGeneratedWorks(); + + return filterBuilder.BuildSearchQuery(request.PageNumber, request.PageSize); + } + + private List GetDLSiteWorks(List nodes) + { + var works = new List(); + //var spamCircles = SpamCircleCache.Get(); + + foreach (DLSiteHtmlNode node in nodes) + { + DLSiteWork work = GetDLSiteWork(node); + + //if (spamCircles.Any(circle => circle.MakerId == work.MakerId)) + // continue; + + works.Add(work); + } + + return works; + } + + private DLSiteWork GetDLSiteWork(DLSiteHtmlNode node) + { + DLSiteWork work = new(); + + work.ProductName = ScannerUtilities.GetDecodedText(node.ProductTextNode); + work.ProductUrl = node.ProductLinkNode.Attributes["href"].Value; + work.ProductId = ScannerUtilities.GetTextBetween(work.ProductUrl, "product_id/", ".html"); + work.Maker = ScannerUtilities.GetDecodedText(node.MakerLinkNode); + + string makerUrl = node.MakerLinkNode.Attributes["href"].Value; + work.MakerId = ScannerUtilities.GetTextBetween(makerUrl, "maker_id/", ".html"); + + work.Description = ScannerUtilities.GetDecodedText(node.DescriptionNode); + + if (node.ExpectedDateNode != null) + { + work.ExpectedDate = GetEstimatedReleaseDate(node.ExpectedDateNode.InnerHtml.Trim()); + } + + if (node.SalesDateNode != null) + { + work.SalesDate = GetSalesDate(node.SalesDateNode.InnerHtml); + } + + if (node.DownloadsNode != null) + { + work.Downloads = int.Parse(node.DownloadsNode.InnerHtml, NumberStyles.AllowThousands); + } + + var rating = GetScannedRating(node.StarRatingNode); + + if (rating != null) + { + work.StarRating = rating.Score; + work.Votes = rating.Votes; + } + + work.Genres = ScannerUtilities.GetStringListFromNodes(node.GenreNodes); + work.Tags = ScannerUtilities.GetStringListFromNodes(node.SearchTagNodes); + work.Creators = ScannerUtilities.GetStringListFromNodes(node.CreatorNodes); + + string imageSource = ScannerUtilities.GetImageSource(node.ImageNode); + string imageUrl = imageSource.Replace("_sam.jpg", "_main.jpg").Replace("_sam.gif", "_main.gif"); + + work.SmallImageUrl = imageSource; + work.ImageUrl = imageUrl; + work.Type = imageUrl.Contains("ana/doujin") ? "Ana" : "Work"; + + return work; + } + + private static ScannedRating? GetScannedRating(HtmlNode starRatingNode) + { + if (starRatingNode == null) + return null; + + string voteText = starRatingNode.InnerText; + + string? ratingClass = starRatingNode.GetClasses().FirstOrDefault(classNames => + classNames.Contains("star_") && classNames != "star_rating"); + + if (string.IsNullOrEmpty(ratingClass)) + return null; + + Regex votesRegex = new Regex(@"\((.*?)\)", RegexOptions.IgnoreCase); + MatchCollection voteMatches = votesRegex.Matches(voteText); + + if (voteMatches.Count == 0 || voteMatches[0].Groups.Count < 2) + return null; + + ScannedRating rating = new ScannedRating(); + rating.Score = Convert.ToByte(ratingClass.Replace("star_", "")); + rating.Votes = int.Parse(voteMatches[0].Groups[1].Value, NumberStyles.AllowThousands); + + return rating; + } +} \ No newline at end of file diff --git a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs index 31e23a2..8ae113d 100644 --- a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs +++ b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs @@ -1,15 +1,13 @@ -using JSMR.Application.Integrations.DLSite.Models; +using JSMR.Application.Common; +using JSMR.Application.Integrations.DLSite.Models; +using JSMR.Infrastructure.Http; using JSMR.Infrastructure.Integrations.DLSite; using JSMR.Infrastructure.Integrations.DLSite.Mapping; using JSMR.Infrastructure.Integrations.DLSite.Models; -using JSMR.Infrastructure.Integrations.DLSite.Serialization; using JSMR.Tests.Utilities; using Microsoft.Extensions.Logging; using NSubstitute; using Shouldly; -using System.Net; -using System.Net.Http; -using System.Text.Json; namespace JSMR.Tests.Integrations.DLSite; @@ -25,26 +23,25 @@ public class DLSiteClientTests { string productInfoJson = await ReadJsonResourceAsync("Product-Info.json"); - HttpResponseMessage response = new() - { - Content = new StringContent(productInfoJson), - StatusCode = HttpStatusCode.OK - }; + IHttpService httpService = Substitute.For(); - HttpClient httpClient = Substitute.For(); - httpClient.BaseAddress = new Uri("https://fake.site.com/"); - - //{ BaseAddress = new Uri("https://www.dlsite.com/") }; - - httpClient.SendAsync(Arg.Any(), Arg.Any(), CancellationToken.None) - .Returns(Task.FromResult(response)); + httpService.GetStringAsync(Arg.Any(), CancellationToken.None) + .Returns(Task.FromResult(productInfoJson)); var logger = Substitute.For>(); - var client = new DLSiteClient(httpClient, logger); + var client = new DLSiteClient(httpService, logger); var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163"], CancellationToken.None); result.Count.ShouldBe(1); + + result.ShouldContainKey("RJ01230163"); + result["RJ01230163"].HasTrial.ShouldBeTrue(); + result["RJ01230163"].HasDLPlay.ShouldBeTrue(); + result["RJ01230163"].HasReviews.ShouldBeTrue(); + result["RJ01230163"].SupportedLanguages.ShouldBe([Language.English]); + result["RJ01230163"].DownloadCount.ShouldBe(659); + result["RJ01230163"].WishlistCount.ShouldBe(380); } [Fact] diff --git a/JSMR.Tests/Unit/DLSiteSearchFilterBuilderTests.cs b/JSMR.Tests/Unit/DLSiteSearchFilterBuilderTests.cs new file mode 100644 index 0000000..cecd717 --- /dev/null +++ b/JSMR.Tests/Unit/DLSiteSearchFilterBuilderTests.cs @@ -0,0 +1,79 @@ +using JSMR.Infrastructure.Scanning; +using Shouldly; + +namespace JSMR.Tests.Unit; + +public class DLSiteSearchFilterBuilderTests +{ + [Fact] + public void Build_Simple_Query() + { + var filterBuilder = new DLSiteSearchFilterBuilder() + .UseDefaultLocale() + .IncludeJapaneseSupportedLanguage(); + + string url = filterBuilder.BuildSearchQuery(1, 100); + + using var writer = new StringWriter(); + + writer.Write($"https://www.dlsite.com/maniax/"); + writer.Write($"fsr/=/language/jp/"); + writer.Write("sex_category[0]/male/"); + writer.Write("ana_flg/all/"); + writer.Write("work_category[0]/doujin/"); + writer.Write("order[0]/release_d/"); + writer.Write("work_type_category[0]/audio/"); + writer.Write("work_type_category_name[0]/ボイス・ASMR/"); + + writer.Write("options_and_or/and/"); + writer.Write($"options[0]/JPN/"); + + writer.Write($"per_page/100/"); + writer.Write($"page/1/"); + writer.Write("show_type/1/"); + writer.Write($"?locale=ja_JP"); + + url.ShouldBe(writer.ToString()); + } + + [Fact] + public void Build_Advanced_Query() + { + var filterBuilder = new DLSiteSearchFilterBuilder() + .UseEnglishLocale() + .IncludeJapaneseSupportedLanguage() + .IncludeEnglishSupportedLanguage() + .ExcludeMakers(["RG0000001", "RG0000002", "", "RG0000001"]) + .ExcludePartiallyAIGeneratedWorks() + .ExcludeAIGeneratedWorks(); + + string url = filterBuilder.BuildSearchQuery(1, 100); + + using var writer = new StringWriter(); + + writer.Write($"https://www.dlsite.com/maniax/"); + writer.Write($"fsr/=/language/en/"); + writer.Write("sex_category[0]/male/"); + writer.Write("ana_flg/all/"); + writer.Write("work_category[0]/doujin/"); + writer.Write("order[0]/release_d/"); + writer.Write("work_type_category[0]/audio/"); + writer.Write("work_type_category_name[0]/ボイス・ASMR/"); + + writer.Write("options_and_or/and/"); + writer.Write($"options[0]/JPN/"); + writer.Write($"options[1]/ENG/"); + + writer.Write($"keyword_maker_name/-RG0000001+-RG0000002/"); + + writer.Write($"options_not[0]/AIP/"); + writer.Write($"options_not[1]/AIG/"); + + writer.Write($"per_page/100/"); + writer.Write($"page/1/"); + writer.Write("show_type/1/"); + writer.Write($"?locale=en_US"); + + url.ShouldBe(writer.ToString()); + } +} \ No newline at end of file diff --git a/JSMR.Tests/Unit/MySqlBooleanQueryTests.cs b/JSMR.Tests/Unit/MySqlBooleanQueryTests.cs index 832ee2c..5c3604d 100644 --- a/JSMR.Tests/Unit/MySqlBooleanQueryTests.cs +++ b/JSMR.Tests/Unit/MySqlBooleanQueryTests.cs @@ -10,15 +10,15 @@ public class MySqlBooleanQueryTests { string normalizedValue = MySqlBooleanQuery.Normalize("value1 value2|value3 -value4 \"value 5\""); - normalizedValue.ShouldBe("+value1 +(value2|value3) -value4 +\"value 5\""); + normalizedValue.ShouldBe("+value1 +(value2 value3) -value4 +\"value 5\""); } [Fact] public void Normalize_Unusual_Usage() { - string normalizedValue = MySqlBooleanQuery.Normalize("+value1 +(value2|value3) -value4 +\"value 5\""); + string normalizedValue = MySqlBooleanQuery.Normalize("+value1 +(value2 value3) -value4 +\"value 5\""); - normalizedValue.ShouldBe("+value1 +(value2|value3) -value4 +\"value 5\""); + normalizedValue.ShouldBe("+value1 +(value2 value3) -value4 +\"value 5\""); } [Fact] @@ -28,4 +28,12 @@ public class MySqlBooleanQueryTests normalizedValue.ShouldBe("+value1 +value2 +value3"); } + + [Fact] + public void Normalize_Bad_Or_Data() + { + string normalizedValue = MySqlBooleanQuery.Normalize("value1 | value2"); + + normalizedValue.ShouldBe("+value1 +value2"); + } }