diff --git a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs index 1dbf75b..beda30b 100644 --- a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs +++ b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlDocument.cs @@ -13,7 +13,8 @@ public class DLSiteHtmlDocument public DLSiteHtmlDocument(HtmlDocument document) { _workColumns = document.DocumentNode.SelectNodes("//dl[@class='work_1col']"); - _workColumnRights = document.DocumentNode.SelectNodes("//td[@class='work_1col_right']"); + //_workColumnRights = document.DocumentNode.SelectNodes("//td[@class='work_1col_right']"); + _workColumnRights = document.DocumentNode.SelectNodes("//td[contains(@class, 'work_1col_right')]"); _workThumbs = document.DocumentNode.SelectNodes("//div[@class='work_thumb']"); PageTotalNode = document.DocumentNode.SelectNodes("//div[@class='page_total']/strong")[0]; diff --git a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs index f6c5b25..64990f0 100644 --- a/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs +++ b/JSMR.Infrastructure/Scanning/Models/DLSiteHtmlNode.cs @@ -11,17 +11,18 @@ public class DLSiteHtmlNode public HtmlNode ProductNode { get; private set; } public HtmlNode ProductLinkNode { get; private set; } public HtmlNode ProductTextNode { get; private set; } - public HtmlNode DescriptionNode { get; private set; } + public HtmlNode? DescriptionNode { get; private set; } public HtmlNode MakerNode { get; private set; } public HtmlNode MakerLinkNode { get; private set; } - public HtmlNode SalesDateNode { get; private set; } - public HtmlNode ExpectedDateNode { get; private set; } + public HtmlNode? ExpectedDateNode { get; private set; } + public HtmlNode? WorkInfoBox { get; private set; } + public HtmlNode? SalesDateNode { get; private set; } public HtmlNode DownloadsNode { get; private set; } - public HtmlNode StarRatingNode { get; private set; } + public HtmlNode? StarRatingNode { get; private set; } public HtmlNode ImageNode { get; private set; } - public List GenreNodes { get; private set; } - public List SearchTagNodes { get; private set; } - public List CreatorNodes { get; private set; } + public HtmlNode[] GenreNodes { get; private set; } + public HtmlNode[] SearchTagNodes { get; private set; } + public HtmlNode[] CreatorNodes { get; private set; } public DLSiteHtmlNode(HtmlNode leftNode, HtmlNode rightNode, HtmlNode thumbNode) { @@ -33,35 +34,44 @@ public class DLSiteHtmlNode ProductLinkNode = ProductNode.SelectNodes(".//a")[0]; ProductTextNode = GetProductTextNode(); - DescriptionNode = LeftNode.SelectNodes(".//dd[@class='work_text']")[0]; + //DescriptionNode = LeftNode.SelectNodes(".//dd[@class='work_text']")[0]; + DescriptionNode = LeftNode.SelectNodes(".//dd[@class='work_text']")?.FirstOrDefault(); MakerNode = LeftNode.SelectNodes(".//dd[@class='maker_name']")[0]; MakerLinkNode = MakerNode.SelectNodes(".//a[contains(@href, 'maker_id')]")[0]; - ExpectedDateNode = GetExpectedDateNode(); + //ExpectedDateNode = GetExpectedDateNode(); + ExpectedDateNode = ProductNode.SelectNodes(".//p[@class='expected_date']")?.FirstOrDefault(); - InitializeGenreNodes(); - InitializeSearchTagNodes(); - InitializeCreatorNodes(); - InitializeSalesAndDownloadsNodes(); - InitializeStarRatingNode(); - InitializeImageNode(); + GenreNodes = GetGenreNodes(); + SearchTagNodes = GetSearchTagNodes(); + CreatorNodes = GetCreatorNodes(); + WorkInfoBox = RightNode.SelectNodes(".//ul[@class='work_info_box']")?.FirstOrDefault(); + SalesDateNode = WorkInfoBox?.SelectNodes(".//li[@class='sales_date']")?.FirstOrDefault(); + + // TODO: Fix! + //DownloadsNode = RightNode.SelectSingleNode(".//span[@class='_dl_count_" + works[rightsIndex].ProductId + "']"); + DownloadsNode = RightNode.SelectSingleNode(".//span[contains(@class, '_dl_count_')]"); + + //InitializeSalesAndDownloadsNodes(); + StarRatingNode = GetStarRatingNode(); + ImageNode = GetImageNode(); } - private void InitializeGenreNodes() + private HtmlNode[] GetGenreNodes() { HtmlNode genreNode = LeftNode.SelectNodes(".//dd[@class='work_genre']")[0]; - GenreNodes = [.. genreNode.SelectNodes(".//span")]; + return [.. genreNode.SelectNodes(".//span")]; } - private void InitializeSearchTagNodes() + private HtmlNode[] GetSearchTagNodes() { HtmlNodeCollection searchTagNodes = LeftNode.SelectNodes(".//dd[@class='search_tag']"); if (searchTagNodes == null || searchTagNodes.Count == 0) { - SearchTagNodes = []; + return []; } else { @@ -69,56 +79,64 @@ public class DLSiteHtmlNode if (searchTagNodesLinks == null || searchTagNodesLinks.Count == 0) { - SearchTagNodes = []; + return []; } else { - SearchTagNodes = [.. searchTagNodesLinks]; + return [.. searchTagNodesLinks]; } } } - private void InitializeCreatorNodes() + private HtmlNode[] GetCreatorNodes() { HtmlNodeCollection creatorNodes = MakerNode.SelectNodes(".//a[contains(@href, 'keyword_creater')]"); if (creatorNodes == null || creatorNodes.Count == 0) { - CreatorNodes = []; + return []; } else { - CreatorNodes = [.. creatorNodes]; + return [.. creatorNodes]; } } - private void InitializeSalesAndDownloadsNodes() - { - HtmlNodeCollection workInfoBox = RightNode.SelectNodes(".//ul[@class='work_info_box']"); + //private void InitializeSalesAndDownloadsNodes() + //{ + // HtmlNodeCollection workInfoBox = RightNode.SelectNodes(".//ul[@class='work_info_box']"); - if (workInfoBox != null) - { - HtmlNodeCollection salesDateNodes = workInfoBox[0].SelectNodes(".//li[@class='sales_date']"); + // if (workInfoBox != null) + // { + // HtmlNodeCollection salesDateNodes = workInfoBox[0].SelectNodes(".//li[@class='sales_date']"); - if (salesDateNodes != null && salesDateNodes.Count > 0) - { - SalesDateNode = salesDateNodes[0]; - } + // if (salesDateNodes != null && salesDateNodes.Count > 0) + // { + // SalesDateNode = salesDateNodes[0]; + // } - // TODO: Fix! - //DownloadsNode = RightNode.SelectSingleNode(".//span[@class='_dl_count_" + works[rightsIndex].ProductId + "']"); - DownloadsNode = RightNode.SelectSingleNode(".//span[contains(@class, '_dl_count_')]"); - } - } + // // TODO: Fix! + // //DownloadsNode = RightNode.SelectSingleNode(".//span[@class='_dl_count_" + works[rightsIndex].ProductId + "']"); + // DownloadsNode = RightNode.SelectSingleNode(".//span[contains(@class, '_dl_count_')]"); + // } + //} - private void InitializeStarRatingNode() + //private HtmlNode? GetSalesDateNode() + //{ + // if (WorkInfoBox is null) + // return null; + + // return WorkInfoBox.SelectNodes(".//li[@class='sales_date']").FirstOrDefault(); + //} + + private HtmlNode? GetStarRatingNode() { var ratingsNode = RightNode.SelectSingleNode(".//li[@class='work_rating']"); if (ratingsNode == null) - return; + return null; - StarRatingNode = ratingsNode.SelectSingleNode(".//div[contains(@class, 'star_rating')]"); + return ratingsNode.SelectSingleNode(".//div[contains(@class, 'star_rating')]"); } private HtmlNode GetProductTextNode() @@ -133,24 +151,24 @@ public class DLSiteHtmlNode } } - private HtmlNode GetExpectedDateNode() - { - HtmlNodeCollection expectedDateNodes = ProductNode.SelectNodes(".//p[@class='expected_date']"); + //private HtmlNode? GetExpectedDateNode() + //{ + // HtmlNodeCollection expectedDateNodes = ProductNode.SelectNodes(".//p[@class='expected_date']").FirstOrDefault(); - if (expectedDateNodes != null && expectedDateNodes.Count > 0) - { - return expectedDateNodes[0]; - } - else - { - return null; - } - } + // if (expectedDateNodes != null && expectedDateNodes.Count > 0) + // { + // return expectedDateNodes[0]; + // } + // else + // { + // return null; + // } + //} - private void InitializeImageNode() + private HtmlNode GetImageNode() { HtmlNode linkNode = ThumbNode.SelectNodes(".//a")[0]; - ImageNode = linkNode.SelectNodes(".//img")[0]; + return linkNode.SelectNodes(".//img")[0]; } } \ No newline at end of file diff --git a/JSMR.Infrastructure/Scanning/ScannerUtilities.cs b/JSMR.Infrastructure/Scanning/ScannerUtilities.cs index 3b2e118..93884bf 100644 --- a/JSMR.Infrastructure/Scanning/ScannerUtilities.cs +++ b/JSMR.Infrastructure/Scanning/ScannerUtilities.cs @@ -5,7 +5,7 @@ namespace JSMR.Infrastructure.Scanning; public static class ScannerUtilities { - public static List GetStringListFromNodes(List nodes) + public static List GetStringListFromNodes(HtmlNode[] nodes) { return nodes .Where(node => string.IsNullOrEmpty(node.InnerHtml) == false) diff --git a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs index 15e50f5..e9125a4 100644 --- a/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs +++ b/JSMR.Infrastructure/Scanning/VoiceWorksScanner.cs @@ -114,7 +114,7 @@ public abstract class VoiceWorksScanner(IHtmlLoader htmlLoader) : IVoiceWorksSca return work; } - private static AgeRating GetAgeRating(List genreNodes) + private static AgeRating GetAgeRating(HtmlNode[] genreNodes) { List genres = ScannerUtilities.GetStringListFromNodes(genreNodes); diff --git a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs index 8ae113d..af58717 100644 --- a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs +++ b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs @@ -39,7 +39,8 @@ public class DLSiteClientTests result["RJ01230163"].HasTrial.ShouldBeTrue(); result["RJ01230163"].HasDLPlay.ShouldBeTrue(); result["RJ01230163"].HasReviews.ShouldBeTrue(); - result["RJ01230163"].SupportedLanguages.ShouldBe([Language.English]); + result["RJ01230163"].SupportedLanguages.Length.ShouldBe(1); + result["RJ01230163"].SupportedLanguages.Select(x => x.Language).ShouldContain(Language.English); result["RJ01230163"].DownloadCount.ShouldBe(659); result["RJ01230163"].WishlistCount.ShouldBe(380); } @@ -71,13 +72,13 @@ public class DLSiteClientTests voiceWorkDetails.DownloadCount.ShouldBe(100); voiceWorkDetails.HasTrial.ShouldBe(true); voiceWorkDetails.HasDLPlay.ShouldBe(true); - voiceWorkDetails.AI.ShouldBe(Application.Common.AIGeneration.None); + voiceWorkDetails.AI.ShouldBe(AIGeneration.None); voiceWorkDetails.Series.ShouldNotBeNull(); voiceWorkDetails.Series.Identifier.ShouldBe("SE0001"); voiceWorkDetails.Series.Name.ShouldBe("Series 1"); voiceWorkDetails.SupportedLanguages.Length.ShouldBe(1); - voiceWorkDetails.SupportedLanguages[0].ShouldBe(Application.Common.Language.Japanese); + voiceWorkDetails.SupportedLanguages[0].Language.ShouldBe(Language.Japanese); } } \ No newline at end of file diff --git a/JSMR.Tests/JSMR.Tests.csproj b/JSMR.Tests/JSMR.Tests.csproj index d566af0..74dd7a5 100644 --- a/JSMR.Tests/JSMR.Tests.csproj +++ b/JSMR.Tests/JSMR.Tests.csproj @@ -14,6 +14,8 @@ + + diff --git a/JSMR.Tests/Scanning/English-Page-Updated.html b/JSMR.Tests/Scanning/English-Page-Updated.html new file mode 100644 index 0000000..95dfa09 --- /dev/null +++ b/JSMR.Tests/Scanning/English-Page-Updated.html @@ -0,0 +1,150 @@ + + +
+
+ Reorder : + +
+
+ 626609 + total. Showing: + 1～30 +
+
+ Display ： + +
+
+ + Items per page ： +
+
+
+ + + + + + + + + + + + + + + + + + + +
+
+ + [ENG Sub] Welcome to Soleil! [Translators Unite]
[ENG Sub] Welcome to Soleil! [Translators Unite]
 + +
Voice / ASMR
+
+ 		+ +
+
+ Oct. 30, 13:59 (JST) Discounted for a limited time. + +
+ + + + + + + +
+ + [ENG Sub] Welcome to Soleil! + +
+ +
+ Translators Unite + + / + 沼倉愛美 +
+ + +
+ + + + $ 5.90 + + + + + $ 13.10 + + + 55%OFF + +
+ + + +
+ All AgesTrial version + + + +
+ +
+ + Moe + Healing + Binaural + ASMR + Ear Cleaning + Slice of Life / Daily Living + Heartwarming + Whispering + +
+
+ 		+ +
    +
  • Release date: Oct/16/2025
    • +
  • +
    Purchased: 1
    +
    • +
  • +
    • +
  • (3,420)
    • +
+ + +
+ +
+
+ + \ No newline at end of file diff --git a/JSMR.Tests/Scanning/Japanese-Page.html b/JSMR.Tests/Scanning/Japanese-Page.html new file mode 100644 index 0000000..538dd2c --- /dev/null +++ b/JSMR.Tests/Scanning/Japanese-Page.html @@ -0,0 +1,157 @@ + + +
+
+ Reorder : + +
+
+ 626609 + total. Showing: + 1～30 +
+
+ Display ： + +
+
+ + Items per page ： +
+
+
+ + + + + + + + + + + + + + + + + + + + +
+
+ + 小悪魔ドSメンズエステ嬢の暴発解禁キワキワ施術 [シルトクレーテ]
小悪魔ドSメンズエステ嬢の暴発解禁キワキワ施術 [シルトクレーテ]
 + +
ボイス・ASMR
+
+ 		+ +
+
+ 2025年11月11日 23時59分 割引終了 + +
+ + + + 専売 + + + +
+ + 小悪魔ドSメンズエステ嬢の暴発解禁キワキワ施術 + +
+ +
+ シルトクレーテ + + / + 柚木つばめ +
+ + +
+ + + + $ 8.74 + + + + + $ 10.92 + + + 20%OFF + +
+ + +
リピート指名していたオキニ嬢の在籍店が閉店し途方に暮れていた貴方だったが、ある日彼女の個人SNSで別のお店のリンクを発見する。大慌てで当日夜の予約を確保し、今度こそ抜いてくれないかと淡い期待を胸に駅近くのマンションの一室へ…。
+ +
+ 体験版 + + + +
+ +
+ + バイノーラル/ダミヘ + 手コキ + 足コキ + パイズリ + 言葉責め + 焦らし + 乳首責め + 本番なし + +
+
+ 		+ +
    +
  • 販売日: 2025年10月15日
    • +
  • +
    販売数: 1,220
    +
    • +
  • + +
    + (2) +
    +     +
    • +
  • (31)
    • +
+ + +
+ +
+
+ + \ No newline at end of file diff --git a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs index d2f7ac7..c97bd77 100644 --- a/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs +++ b/JSMR.Tests/Scanning/VoiceWorkScannerTests.cs @@ -5,7 +5,7 @@ using JSMR.Tests.Utilities; using NSubstitute; using Shouldly; -namespace JSMR.Tests.Integrations.DLSite; +namespace JSMR.Tests.Scanning; public class VoiceWorkScannerTests { @@ -14,6 +14,45 @@ public class VoiceWorkScannerTests return await ResourceHelper.ReadAsync($"JSMR.Tests.Scanning.{resourceName}"); } + [Fact] + public async Task Scan_With_Japanese_Locale() + { + string html = await ReadResourceAsync("Japanese-Page.html"); + + IHttpService httpService = Substitute.For(); + + httpService.GetStringAsync(Arg.Any(), CancellationToken.None) + .Returns(Task.FromResult(html)); + + HtmlLoader loader = new(httpService); + JapaneseVoiceWorksScanner scanner = new(loader); + + VoiceWorkScanOptions options = new( + PageNumber: 1, + PageSize: 100, + ExcludeAIGeneratedWorks: true, + ExcludePartiallyAIGeneratedWorks: true, + ExcludedMakerIds: [] + ); + + var result = await scanner.ScanPageAsync(options, CancellationToken.None); + + result.Count.ShouldBe(1); + + result[0].ExpectedDate.ShouldBeNull(); + result[0].SalesDate.ShouldBe(new DateOnly(2025, 10, 15)); + result[0].ProductId.ShouldBe("RJ01462066"); + result[0].ProductName.ShouldBe("小悪魔ドSメンズエステ嬢の暴発解禁キワキワ施術"); + result[0].Description.ShouldBe("リピート指名していたオキニ嬢の在籍店が閉店し途方に暮れていた貴方だったが、ある日彼女の個人SNSで別のお店のリンクを発見する。大慌てで当日夜の予約を確保し、今度こそ抜いてくれないかと淡い期待を胸に駅近くのマンションの一室へ…。"); + result[0].Maker.ShouldBe("シルトクレーテ"); + result[0].MakerId.ShouldBe("RG40741"); + result[0].Creators.ShouldBe(["柚木つばめ"]); + result[0].Genres.ShouldBe(["体験版"]); + result[0].Tags.ShouldBe(["バイノーラル/ダミヘ", "手コキ", "足コキ", "パイズリ", "言葉責め", "焦らし", "乳首責め", "本番なし"]); + result[0].Type.ShouldBe(DLSiteWorkType.Released); + result[0].Downloads.ShouldBe(1220); + } + [Fact] public async Task Scan_With_English_Locale() { @@ -57,4 +96,43 @@ public class VoiceWorkScannerTests result[1].ProductId.ShouldBe("RJ00000002"); result[1].Type.ShouldBe(DLSiteWorkType.Announced); } + + [Fact] + public async Task Scan_With_Updated_English_Locale() + { + string html = await ReadResourceAsync("English-Page-Updated.html"); + + IHttpService httpService = Substitute.For(); + + httpService.GetStringAsync(Arg.Any(), CancellationToken.None) + .Returns(Task.FromResult(html)); + + HtmlLoader loader = new(httpService); + EnglishVoiceWorksScanner scanner = new(loader); + + VoiceWorkScanOptions options = new( + PageNumber: 1, + PageSize: 100, + ExcludeAIGeneratedWorks: true, + ExcludePartiallyAIGeneratedWorks: true, + ExcludedMakerIds: [] + ); + + var result = await scanner.ScanPageAsync(options, CancellationToken.None); + + result.Count.ShouldBe(1); + + result[0].ExpectedDate.ShouldBeNull(); + result[0].SalesDate.ShouldBe(new DateOnly(2025, 10, 16)); + result[0].ProductId.ShouldBe("RJ01455722"); + result[0].ProductName.ShouldBe("[ENG Sub] Welcome to Soleil!"); + result[0].Description.ShouldBe(string.Empty); // Waiting on this to get fixed on the site + result[0].Maker.ShouldBe("Translators Unite"); + result[0].MakerId.ShouldBe("RG60289"); + result[0].Creators.ShouldBe(["沼倉愛美"]); + result[0].Genres.ShouldBe(["All Ages", "Trial version"]); + result[0].Tags.ShouldBe(["Moe", "Healing", "Binaural", "ASMR", "Ear Cleaning", "Slice of Life / Daily Living", "Heartwarming", "Whispering"]); + result[0].Type.ShouldBe(DLSiteWorkType.Released); + result[0].Downloads.ShouldBe(1); + } } \ No newline at end of file