brister/jsmr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Chobit Integration #2

Merged
brister merged 2 commits from feature/issue-1-chobit-integration into master 2026-03-15 02:34:12 +00:00
Conversation 0 Commits 2 Files Changed 34 +529 -109
34 changed files with 529 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
JSMR.Application/Integrations/Chobit/Models/ChobitResult.cs
View File

@@ -1,7 +1,12 @@
namespace JSMR.Application.Integrations.Chobit.Models;
using System.Text.Json.Serialization;
namespace JSMR.Application.Integrations.Chobit.Models;
public class ChobitResult
{
[JsonPropertyName("count")]
public int Count { get; set; }
[JsonPropertyName("works")]
public ChobitWork[] Works { get; set; } = [];
}

3
JSMR.Application/Integrations/Chobit/Models/ChobitResultCollection.cs Normal file
View File

@@ -0,0 +1,3 @@
namespace JSMR.Application.Integrations.Chobit.Models;
public class ChobitResultCollection : Dictionary<string, ChobitResult> { }

29
JSMR.Application/Integrations/Chobit/Models/ChobitWork.cs
View File

@@ -1,16 +1,39 @@
namespace JSMR.Application.Integrations.Chobit.Models;
using System.Text.Json.Serialization;
namespace JSMR.Application.Integrations.Chobit.Models;
public class ChobitWork
{
[JsonPropertyName("work_id")]
public string? WorkId { get; set; }
[JsonPropertyName("dlsite_work_id")]
public string? DLSiteWorkId { get; set; }
[JsonPropertyName("work_name")]
public string? WorkName { get; set; }
[JsonPropertyName("work_name_kana")]
public string? WorkNameKana { get; set; }
public string? URL { get; set; }
public string? EmbedURL { get; set; }
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("embed_url")]
public string? EmbedUrl { get; set; }
[JsonPropertyName("thumb")]
public string? Thumb { get; set; }
[JsonPropertyName("mini_thumb")]
public string? MiniThumb { get; set; }
[JsonPropertyName("file_type")]
public string? FileType { get; set; }
[JsonPropertyName("embed_width")]
public decimal EmbedWidth { get; set; }
[JsonPropertyName("embed_height")]
public decimal EmbedHeight { get; set; }
}

3
JSMR.Application/Integrations/Chobit/Models/ChobitWorkResult.cs
View File

@@ -1,3 +0,0 @@
namespace JSMR.Application.Integrations.Chobit.Models;
public class ChobitWorkResult : Dictionary<string, ChobitResult> { }

3
JSMR.Application/Integrations/Ports/IChobitClient.cs
View File

@@ -4,5 +4,6 @@ namespace JSMR.Application.Integrations.Ports;
public interface IChobitClient
{
Task<ChobitWorkResult> GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default);
Task<ChobitResult> GetSampleInfoAsync(string productId, CancellationToken cancellationToken = default);
Task<ChobitResultCollection> GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default);
}

9
JSMR.Application/Scanning/Contracts/VoiceWorkIngest.cs
View File

@@ -1,4 +1,5 @@
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Chobit.Models;
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Domain.Enums;
using JSMR.Domain.ValueObjects;
@@ -16,7 +17,7 @@ public sealed record VoiceWorkIngest
public int WishlistCount { get; init; }
public int Downloads { get; init; }
public bool HasTrial { get; init; }
public bool HasDLPlay { get; init; }
public bool HasChobit { get; init; }
public byte? StarRating { get; init; }
public int? Votes { get; init; }
public AgeRating AgeRating { get; init; }
@@ -29,7 +30,7 @@ public sealed record VoiceWorkIngest
public VoiceWorkSeries? Series { get; init; }
public VoiceWorkTranslation? Translation { get; init; }
public static VoiceWorkIngest From(DLSiteWork work, VoiceWorkDetails? details)
public static VoiceWorkIngest From(DLSiteWork work, VoiceWorkDetails? details, ChobitResult? chobit)
{
return new VoiceWorkIngest()
{
@@ -43,7 +44,7 @@ public sealed record VoiceWorkIngest
WishlistCount = details?.WishlistCount ?? 0,
Downloads = Math.Max(work.Downloads, details?.DownloadCount ?? 0),
HasTrial = work.HasTrial || (details?.HasTrial ?? false),
HasDLPlay = details?.HasDLPlay ?? false,
HasChobit = chobit?.Count > 0,
StarRating = work.StarRating,
Votes = work.Votes,
AgeRating = details?.AgeRating ?? work.AgeRating,

6
JSMR.Application/Scanning/ScanVoiceWorksHandler.cs
View File

@@ -1,4 +1,5 @@
using JSMR.Application.Common.Caching;
using JSMR.Application.Integrations.Chobit.Models;
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Ports;
using JSMR.Application.Scanning.Contracts;
@@ -10,6 +11,7 @@ public sealed class ScanVoiceWorksHandler(
IVoiceWorkScannerRepository scannerRepository,
IVoiceWorkUpdaterRepository updaterRepository,
IDLSiteClient dlsiteClient,
IChobitClient chobitClient,
ISpamCircleCache spamCircleCache,
IVoiceWorkSearchUpdater searchUpdater)
{
@@ -44,11 +46,13 @@ public sealed class ScanVoiceWorksHandler(
string[] productIds = [.. scanResult.Works.Where(x => !string.IsNullOrWhiteSpace(x.ProductId)).Select(x => x.ProductId!)];
VoiceWorkDetailCollection voiceWorkDetails = await dlsiteClient.GetVoiceWorkDetailsAsync(productIds, cancellationToken);
ChobitResultCollection chobitResults = await chobitClient.GetSampleInfoAsync(productIds, cancellationToken);
VoiceWorkIngest[] ingests = [.. scanResult.Works.Select(work =>
{
voiceWorkDetails.TryGetValue(work.ProductId!, out VoiceWorkDetails? value);
return VoiceWorkIngest.From(work, value);
chobitResults.TryGetValue(work.ProductId, out ChobitResult? chobit);
return VoiceWorkIngest.From(work, value, chobit);
})];
VoiceWorkUpsertResult[] upsertResults = await updater.UpsertAsync(ingests, cancellationToken);

49
JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs
View File

@@ -24,6 +24,7 @@ using JSMR.Infrastructure.Data.Repositories.Users;
using JSMR.Infrastructure.Data.Repositories.VoiceWorks;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Ingestion;
using JSMR.Infrastructure.Integrations.Chobit;
using JSMR.Infrastructure.Integrations.DLSite;
using JSMR.Infrastructure.Scanning;
using Microsoft.Extensions.DependencyInjection;
@@ -76,6 +77,9 @@ public static class InfrastructureServiceCollectionExtensions
services.AddSingleton<ITimeProvider, TokyoTimeProvider>();
services.AddHttpServices();
services.AddNewHttpServices();
services.AddScoped<IUserRepository, UserRepository>();
return services;
}
@@ -113,7 +117,50 @@ public static class InfrastructureServiceCollectionExtensions
// Register DLSiteClient as a normal scoped service
services.AddScoped<IDLSiteClient, DLSiteClient>();
services.AddScoped<IUserRepository, UserRepository>();
return services;
}
private static IServiceCollection AddNewHttpServices(this IServiceCollection services)
{
services.AddHttpClient<IDLSiteClient, DLSiteClient>((sp, http) =>
{
http.BaseAddress = new Uri("https://www.dlsite.com/");
http.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
http.Timeout = TimeSpan.FromSeconds(15);
})
.AddResilienceHandler("dlsite", builder =>
{
builder.AddRetry(new HttpRetryStrategyOptions
{
MaxRetryAttempts = 3,
UseJitter = true,
Delay = TimeSpan.FromMilliseconds(200),
BackoffType = DelayBackoffType.Exponential,
ShouldHandle = new PredicateBuilder<HttpResponseMessage>()
.Handle<HttpRequestException>()
.HandleResult(r => (int)r.StatusCode >= 500 || (int)r.StatusCode == 429)
});
});
services.AddHttpClient<IChobitClient, ChobitClient>((sp, http) =>
{
http.BaseAddress = new Uri("https://chobit.cc/");
http.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
http.Timeout = TimeSpan.FromSeconds(15);
})
.AddResilienceHandler("chobit", builder =>
{
builder.AddRetry(new HttpRetryStrategyOptions
{
MaxRetryAttempts = 3,
UseJitter = true,
Delay = TimeSpan.FromMilliseconds(200),
BackoffType = DelayBackoffType.Exponential,
ShouldHandle = new PredicateBuilder<HttpResponseMessage>()
.Handle<HttpRequestException>()
.HandleResult(r => (int)r.StatusCode >= 500 || (int)r.StatusCode == 429)
});
});
return services;
}

146
JSMR.Infrastructure/Http/ApiClient.cs
View File

@@ -1,91 +1,121 @@
using Microsoft.Extensions.Logging;
using System.IO;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace JSMR.Infrastructure.Http;
public abstract class ApiClient(IHttpService http, ILogger logger, JsonSerializerOptions? json = null)
public abstract class ApiClient(HttpClient http, ILogger logger, JsonSerializerOptions? json = null)
{
protected async Task<T> GetJsonAsync<T>(string url, CancellationToken cancellationToken = default)
protected async Task<TResponse> GetJsonAsync<TResponse>(
string url,
Action<HttpRequestHeaders>? configureHeaders = null,
CancellationToken cancellationToken = default)
{
HttpStringResponse response = await http.GetAsync(url, cancellationToken);
using HttpRequestMessage request = new(HttpMethod.Get, url);
configureHeaders?.Invoke(request.Headers);
if (response.Content is null)
throw new Exception("No content to deserialize");
LogRequest(request);
return JsonSerializer.Deserialize<T>(response.Content, json)
?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}.");
using HttpResponseMessage response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
await EnsureSuccess(response).ConfigureAwait(false);
Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
return await JsonSerializer.DeserializeAsync<TResponse>(stream, json, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}.");
}
//protected async Task<T> GetJsonAsync<T>(
// string url,
// Action<HttpRequestHeaders>? configureHeaders = null,
// CancellationToken ct = default
// )
//{
// using var req = new HttpRequestMessage(HttpMethod.Get, url);
// configureHeaders?.Invoke(req.Headers);
protected async Task<TResponse> GetJsonpAsync<TResponse>(
string url,
Action<HttpRequestHeaders>? configureHeaders = null,
CancellationToken cancellationToken = default)
{
using HttpRequestMessage request = new(HttpMethod.Get, url);
configureHeaders?.Invoke(request.Headers);
// LogRequest(req);
LogRequest(request);
// using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
// await EnsureSuccess(res).ConfigureAwait(false);
using HttpResponseMessage response = await http.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken).ConfigureAwait(false);
// var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
await EnsureSuccess(response).ConfigureAwait(false);
// var model = await JsonSerializer.DeserializeAsync<T>(stream, json, ct).ConfigureAwait(false)
// ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(T).Name} from {url}.");
string body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
// return model;
//}
string jsonBody = ExtractJsonFromJsonp(body);
//protected async Task<TResponse> PostJsonAsync<TRequest, TResponse>(
// string url,
// TRequest payload,
// Action<HttpRequestHeaders>? configureHeaders = null,
// CancellationToken ct = default)
//{
// var content = new StringContent(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json");
// using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
// configureHeaders?.Invoke(req.Headers);
return JsonSerializer.Deserialize<TResponse>(jsonBody, json)
?? throw new InvalidOperationException($"Failed to deserialize JSONP payload to {typeof(TResponse).Name} from {url}.");
}
// LogRequest(req);
private static string ExtractJsonFromJsonp(string body)
{
if (string.IsNullOrWhiteSpace(body))
throw new InvalidOperationException("Response body was empty.");
// using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
// await EnsureSuccess(res).ConfigureAwait(false);
body = body.Trim();
// var stream = await res.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
int firstParen = body.IndexOf('(');
int lastParen = body.LastIndexOf(')');
// var model = await JsonSerializer.DeserializeAsync<TResponse>(stream, json, ct).ConfigureAwait(false)
// ?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}.");
if (firstParen < 0 || lastParen <= firstParen)
throw new InvalidOperationException("Response was not valid JSONP.");
// return model;
//}
return body[(firstParen + 1)..lastParen].Trim();
}
//protected virtual void LogRequest(HttpRequestMessage req)
// => logger.LogDebug("HTTP {Method} {Uri}", req.Method, req.RequestUri);
protected async Task<TResponse> PostJsonAsync<TRequest, TResponse>(
string url,
TRequest payload,
Action<HttpRequestHeaders>? configureHeaders = null,
CancellationToken cancellationToken = default)
{
StringContent content = new(JsonSerializer.Serialize(payload, json), Encoding.UTF8, "application/json");
//protected virtual void LogFailure(HttpResponseMessage res, string body)
// => logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)res.StatusCode, res.RequestMessage?.RequestUri, Truncate(body, 500));
using HttpRequestMessage request = new(HttpMethod.Post, url)
{
Content = content
};
//protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…";
configureHeaders?.Invoke(request.Headers);
//protected async Task EnsureSuccess(HttpResponseMessage res)
//{
// if (res.IsSuccessStatusCode) return;
LogRequest(request);
// string body;
// try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); }
// catch { body = "<unable to read body>"; }
using HttpResponseMessage response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
await EnsureSuccess(response).ConfigureAwait(false);
// LogFailure(res, body);
Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
return await JsonSerializer.DeserializeAsync<TResponse>(stream, json, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Failed to deserialize JSON to {typeof(TResponse).Name} from {url}.");
}
protected virtual void LogRequest(HttpRequestMessage request)
=> logger.LogDebug("HTTP {Method} {Uri}", request.Method, request.RequestUri);
protected virtual void LogFailure(HttpResponseMessage response, string body)
=> logger.LogWarning("HTTP {Status} for {Uri}. Body: {Body}", (int)response.StatusCode, response.RequestMessage?.RequestUri, Truncate(body, 500));
protected static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "…";
protected async Task EnsureSuccess(HttpResponseMessage res)
{
if (res.IsSuccessStatusCode)
return;
string body;
try { body = await res.Content.ReadAsStringAsync().ConfigureAwait(false); }
catch { body = "<unable to read body>"; }
LogFailure(res, body);
//Throw a richer exception(you can customize per API)
// throw new HttpRequestException(
// $"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}",
// null,
// res.StatusCode);
//}
throw new HttpRequestException(
$"Request to {res.RequestMessage?.RequestUri} failed: {(int)res.StatusCode} {res.ReasonPhrase}. Body: {Truncate(body, 1000)}",
null,
res.StatusCode);
}
}

2
JSMR.Infrastructure/Ingestion/VoiceWorkUpdater.cs
View File

@@ -206,7 +206,7 @@ public class VoiceWorkUpdater(AppDbContext dbContext, ITimeProvider timeProvider
voiceWork.Downloads = ingest.Downloads;
voiceWork.WishlistCount = ingest.WishlistCount;
voiceWork.HasTrial = ingest.HasTrial;
voiceWork.HasChobit = ingest.HasDLPlay;
voiceWork.HasChobit = ingest.HasChobit;
voiceWork.StarRating = ingest.StarRating;
voiceWork.Votes = ingest.Votes;
voiceWork.OriginalProductId = ingest.Translation?.OriginalProductId;

14
JSMR.Infrastructure/Integrations/Chobit/ChobitClient.cs
View File

@@ -5,11 +5,17 @@ using Microsoft.Extensions.Logging;
namespace JSMR.Infrastructure.Integrations.Chobit;
public class ChobitClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IChobitClient
public class ChobitClient(HttpClient http, ILogger<ChobitClient> logger) : ApiClient(http, logger), IChobitClient
{
public Task<ChobitWorkResult> GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default)
public Task<ChobitResult> GetSampleInfoAsync(string productId, CancellationToken cancellationToken = default)
{
var url = $"api/v2/dlsite/embed?workno_list=${string.Join(",", productIds)}";
return GetJsonAsync<ChobitWorkResult>(url, cancellationToken);
var url = $"api/v1/dlsite/embed?workno={productId}";
return GetJsonpAsync<ChobitResult>(url, cancellationToken: cancellationToken);
}
public Task<ChobitResultCollection> GetSampleInfoAsync(string[] productIds, CancellationToken cancellationToken = default)
{
var url = $"api/v2/dlsite/embed?workno_list={string.Join(",", productIds)}";
return GetJsonpAsync<ChobitResultCollection>(url, cancellationToken: cancellationToken);
}
}

14
JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs
View File

@@ -7,21 +7,19 @@ using Microsoft.Extensions.Logging;
namespace JSMR.Infrastructure.Integrations.DLSite;
public class DLSiteClient(IHttpService http, ILogger<DLSiteClient> logger) : ApiClient(http, logger), IDLSiteClient
public class DLSiteClient(HttpClient http, ILogger<DLSiteClient> logger) : ApiClient(http, logger), IDLSiteClient
{
public async Task<VoiceWorkDetailCollection> GetVoiceWorkDetailsAsync(string[] productIds, CancellationToken cancellationToken = default)
{
if (productIds.Length == 0)
return [];
string productIdCollection = string.Join(",", productIds.Where(x => !string.IsNullOrWhiteSpace(x)));
if (string.IsNullOrWhiteSpace(productIdCollection))
string[] validProductIds = [.. productIds.Where(x => !string.IsNullOrWhiteSpace(x)).Distinct()];
if (validProductIds.Length == 0)
return [];
string productIdCollection = string.Join(",", validProductIds);
string url = $"maniax/product/info/ajax?product_id={productIdCollection}&cdn_cache_min=1";
var productInfoCollection = await GetJsonAsync<ProductInfoCollection>(url, cancellationToken);
ProductInfoCollection productInfoCollection = await GetJsonAsync<ProductInfoCollection>(url, cancellationToken: cancellationToken);
return DLSiteToDomainMapper.Map(productInfoCollection);
}

9
JSMR.Tests/Http/FakeHttpMessageHandler.cs Normal file
View File

@@ -0,0 +1,9 @@
namespace JSMR.Tests.Http;
internal sealed class FakeHttpMessageHandler(Func<HttpRequestMessage, HttpResponseMessage> handler) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
return Task.FromResult(handler(request));
}
}

2
JSMR.Tests/Ingestion/IngestTestFactory.cs
View File

@@ -21,7 +21,7 @@ internal class IngestTestFactory
WishlistCount = 100,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
AgeRating = AgeRating.AllAges,
HasImage = false,
SupportedLanguages = [SupportedLanguage.Japanese],

154
JSMR.Tests/Ingestion/Japanese/Basic_Insert_And_Update_Tests.cs Normal file
View File

@@ -0,0 +1,154 @@
using JSMR.Application.Scanning.Contracts;
using JSMR.Domain.Entities;
using JSMR.Domain.Enums;
using JSMR.Domain.ValueObjects;
using JSMR.Infrastructure.Data;
using JSMR.Tests.Fixtures;
using Microsoft.EntityFrameworkCore;
using Shouldly;
namespace JSMR.Tests.Ingestion.Japanese;
public class Basic_Insert_And_Update_Tests(MariaDbContainerFixture fixture) : JapaneseIngestionTestsBase(fixture)
{
[Fact]
public async Task Basic_Insert_And_Update_Test()
{
await using AppDbContext dbContext = await GetAppDbContextAsync();
VoiceWorkIngest ingest = new()
{
MakerId = "RG1",
MakerName = "My Maker",
ProductId = "RJ1",
Title = "My Product",
Description = "My description",
Tags = ["Tag 1", "Tag 2"],
Creators = ["Creator 1"],
WishlistCount = 100,
Downloads = 0,
HasTrial = true,
HasChobit = false,
AgeRating = AgeRating.AllAges,
HasImage = false,
SupportedLanguages = [SupportedLanguage.Japanese],
SalesDate = null,
ExpectedDate = new DateOnly(2025, 2, 1)
};
DateTime dateTime = TokyoLocalToUtc(2025, 01, 15, 00, 00, 00);
await UpsertAsync(dbContext, dateTime, [ingest]);
Circle? circle = await dbContext.Circles.FirstOrDefaultAsync(v => v.MakerId == ingest.MakerId, TestContext.Current.CancellationToken);
circle.ShouldNotBeNull();
circle.Name.ShouldBe(ingest.MakerName);
VoiceWork? voiceWork = await dbContext.VoiceWorks.FirstOrDefaultAsync(v => v.ProductId == ingest.ProductId, TestContext.Current.CancellationToken);
voiceWork.ShouldNotBeNull();
voiceWork.ProductName.ShouldBe(ingest.Title);
voiceWork.Description.ShouldBe(ingest.Description);
voiceWork.WishlistCount.ShouldBe(ingest.WishlistCount);
voiceWork.Downloads.ShouldBe(ingest.Downloads);
voiceWork.HasTrial.ShouldBe(ingest.HasTrial);
voiceWork.HasChobit.ShouldBe(ingest.HasChobit);
voiceWork.Rating.ShouldBe((int)ingest.AgeRating);
voiceWork.HasImage.ShouldBe(ingest.HasImage);
voiceWork.SubtitleLanguage.ShouldBe((byte)Language.Japanese);
voiceWork.SalesDate.ShouldBeNull();
voiceWork.ExpectedDate.ShouldBe(ingest.ExpectedDate!.Value.ToDateTime(TimeOnly.MinValue));
foreach (string tagName in ingest.Tags)
{
Tag? tag = await dbContext.Tags.FirstOrDefaultAsync(t => t.Name == tagName, TestContext.Current.CancellationToken);
tag.ShouldNotBeNull();
VoiceWorkTag? voiceWorkTag = await dbContext.VoiceWorkTags.FirstOrDefaultAsync(vwt =>
vwt.VoiceWorkId == voiceWork.VoiceWorkId && vwt.TagId == tag.TagId, TestContext.Current.CancellationToken);
voiceWorkTag.ShouldNotBeNull();
}
foreach (string creatorName in ingest.Creators)
{
Creator? creator = await dbContext.Creators.FirstOrDefaultAsync(c => c.Name == creatorName, TestContext.Current.CancellationToken);
creator.ShouldNotBeNull();
VoiceWorkCreator? voiceWorkCreator = await dbContext.VoiceWorkCreators.FirstOrDefaultAsync(vwc =>
vwc.VoiceWorkId == voiceWork.VoiceWorkId && vwc.CreatorId == creator.CreatorId, TestContext.Current.CancellationToken);
voiceWorkCreator.ShouldNotBeNull();
}
foreach (SupportedLanguage supportedLanguage in ingest.SupportedLanguages)
{
VoiceWorkSupportedLanguage? voiceWorkSupportedLanauge = await dbContext.VoiceWorkSupportedLanguages.FirstOrDefaultAsync(x =>
x.VoiceWorkId == voiceWork.VoiceWorkId && x.Language == supportedLanguage.Code, TestContext.Current.CancellationToken);
voiceWorkSupportedLanauge.ShouldNotBeNull();
}
VoiceWorkIngest updatedIngest = ingest with
{
MakerName = "My Maker (Updated)",
Tags = ["Tag 1", "Not Tag 2"],
Creators = ["Not Creator 1"],
Downloads = 50,
HasChobit = true,
SupportedLanguages = [SupportedLanguage.Japanese, SupportedLanguage.English],
HasImage = true,
SalesDate = new DateOnly(2025, 2, 5),
ExpectedDate = null
};
DateTime updateDateTime = TokyoLocalToUtc(2025, 02, 03, 00, 00, 00);
await UpsertAsync(dbContext, dateTime, [updatedIngest]);
circle = await dbContext.Circles.FirstOrDefaultAsync(v => v.MakerId == updatedIngest.MakerId, TestContext.Current.CancellationToken);
circle.ShouldNotBeNull();
circle.Name.ShouldBe(updatedIngest.MakerName);
voiceWork = await dbContext.VoiceWorks.FirstOrDefaultAsync(v => v.ProductId == updatedIngest.ProductId, TestContext.Current.CancellationToken);
voiceWork.ShouldNotBeNull();
voiceWork.ProductName.ShouldBe(updatedIngest.Title);
voiceWork.Description.ShouldBe(updatedIngest.Description);
voiceWork.WishlistCount.ShouldBe(updatedIngest.WishlistCount);
voiceWork.Downloads.ShouldBe(updatedIngest.Downloads);
voiceWork.HasTrial.ShouldBe(updatedIngest.HasTrial);
voiceWork.HasChobit.ShouldBe(updatedIngest.HasChobit);
voiceWork.Rating.ShouldBe((int)updatedIngest.AgeRating);
voiceWork.HasImage.ShouldBe(updatedIngest.HasImage);
voiceWork.SubtitleLanguage.ShouldBe((byte)Language.English);
voiceWork.SalesDate.ShouldBe(updatedIngest.SalesDate!.Value.ToDateTime(TimeOnly.MinValue));
voiceWork.ExpectedDate.ShouldBeNull();
foreach (string tagName in updatedIngest.Tags.Union(ingest.Tags))
{
Tag? tag = await dbContext.Tags.FirstOrDefaultAsync(t => t.Name == tagName, TestContext.Current.CancellationToken);
tag.ShouldNotBeNull();
VoiceWorkTag? voiceWorkTag = await dbContext.VoiceWorkTags.FirstOrDefaultAsync(vwt =>
vwt.VoiceWorkId == voiceWork.VoiceWorkId && vwt.TagId == tag.TagId, TestContext.Current.CancellationToken);
voiceWorkTag.ShouldNotBeNull();
}
foreach (string creatorName in updatedIngest.Creators.Union(ingest.Creators))
{
Creator? creator = await dbContext.Creators.FirstOrDefaultAsync(c => c.Name == creatorName, TestContext.Current.CancellationToken);
creator.ShouldNotBeNull();
VoiceWorkCreator? voiceWorkCreator = await dbContext.VoiceWorkCreators.FirstOrDefaultAsync(vwc =>
vwc.VoiceWorkId == voiceWork.VoiceWorkId && vwc.CreatorId == creator.CreatorId, TestContext.Current.CancellationToken);
voiceWorkCreator.ShouldNotBeNull();
}
foreach (SupportedLanguage supportedLanguage in updatedIngest.SupportedLanguages.Union(ingest.SupportedLanguages))
{
VoiceWorkSupportedLanguage? voiceWorkSupportedLanauge = await dbContext.VoiceWorkSupportedLanguages.FirstOrDefaultAsync(x =>
x.VoiceWorkId == voiceWork.VoiceWorkId && x.Language == supportedLanguage.Code, TestContext.Current.CancellationToken);
voiceWorkSupportedLanauge.ShouldNotBeNull();
}
}
}

2
JSMR.Tests/Ingestion/Japanese/Fail_Attempted_Insert_With_Spam_Circle_Tests.cs
View File

@@ -29,7 +29,7 @@ public class Fail_Attempted_Insert_With_Spam_Circle_Tests(MariaDbContainerFixtur
WishlistCount = 100,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
AgeRating = AgeRating.R18,
HasImage = false,
SupportedLanguages = [SupportedLanguage.Japanese],

2
JSMR.Tests/Ingestion/Japanese/Fail_Attempted_Update_With_Decreased_Downloads_Tests.cs
View File

@@ -29,7 +29,7 @@ public class Fail_Attempted_Update_With_Decreased_Downloads_Tests(MariaDbContain
WishlistCount = 50,
Downloads = 10,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.AllAges,

2
JSMR.Tests/Ingestion/Japanese/Fail_Attempted_Update_With_Sales_Date_Reversal_Tests.cs
View File

@@ -29,7 +29,7 @@ public class Fail_Attempted_Update_With_Sales_Date_Reversal_Tests(MariaDbContain
WishlistCount = 50,
Downloads = 10,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.AllAges,

2
JSMR.Tests/Ingestion/Japanese/Insert_New_Release_And_Scan_Again_Later_Tests.cs
View File

@@ -28,7 +28,7 @@ public class Insert_New_Release_And_Scan_Again_Later_Tests(MariaDbContainerFixtu
WishlistCount = 50,
Downloads = 10,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.AllAges,

2
JSMR.Tests/Ingestion/Japanese/Insert_New_Release_With_New_Tags_And_Creators_Tests.cs
View File

@@ -29,7 +29,7 @@ public class Insert_New_Release_With_New_Tags_And_Creators_Tests(MariaDbContaine
WishlistCount = 50,
Downloads = 10,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.AllAges,

2
JSMR.Tests/Ingestion/Japanese/Insert_New_Upcoming_And_Scan_Again_Later_Tests.cs
View File

@@ -28,7 +28,7 @@ public class Insert_New_Upcoming_And_Scan_Again_Later_Tests(MariaDbContainerFixt
WishlistCount = 100,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
AgeRating = AgeRating.AllAges,
HasImage = false,
SupportedLanguages = [SupportedLanguage.Japanese],

2
JSMR.Tests/Ingestion/Japanese/Insert_New_Upcoming_Release_Same_Day_Tests.cs
View File

@@ -28,7 +28,7 @@ public class Insert_New_Upcoming_Release_Same_Day_Tests(MariaDbContainerFixture
WishlistCount = 100,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
AgeRating = AgeRating.AllAges,
HasImage = false,
SupportedLanguages = [SupportedLanguage.Japanese],

2
JSMR.Tests/Ingestion/Japanese/Insert_New_Upcoming_With_Existing_Tags_And_Creators_Tests.cs
View File

@@ -29,7 +29,7 @@ public class Insert_New_Upcoming_With_Existing_Tags_And_Creators_Tests(MariaDbCo
WishlistCount = 250,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.R15,

2
JSMR.Tests/Ingestion/Japanese/Update_Upcoming_With_No_Expected_Date_Tests.cs
View File

@@ -26,7 +26,7 @@ public class Update_Upcoming_With_No_Expected_Date_Tests(MariaDbContainerFixture
WishlistCount = 250,
Downloads = 0,
HasTrial = false,
HasDLPlay = false,
HasChobit = false,
StarRating = null,
Votes = null,
AgeRating = AgeRating.R15,

98
JSMR.Tests/Integrations/Chobit/ChobitClientTests.cs Normal file
View File

@@ -0,0 +1,98 @@
using JSMR.Application.Integrations.Chobit.Models;
using JSMR.Infrastructure.Integrations.Chobit;
using JSMR.Tests.Http;
using JSMR.Tests.Utilities;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Shouldly;
using System.Net;
using System.Text;
namespace JSMR.Tests.Integrations.Chobit;
public class ChobitClientTests
{
private static async Task<string> ReadJsonResourceAsync(string resourceName)
{
return await ResourceHelper.ReadAsync($"JSMR.Tests.Integrations.Chobit.{resourceName}");
}
private static async Task<ChobitClient> GetChobitClientThatReturns(string resourceName)
{
string content = await ReadJsonResourceAsync(resourceName);
FakeHttpMessageHandler handler = new(request =>
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(content, Encoding.UTF8, "application/json")
};
});
HttpClient httpClient = new(handler)
{
BaseAddress = new Uri("https://fake-chobit.cc/")
};
var logger = Substitute.For<ILogger<ChobitClient>>();
var client = new ChobitClient(httpClient, logger);
return client;
}
[Fact]
public async Task Deserialize_Chobit_Sample_Info()
{
var client = await GetChobitClientThatReturns("Sample-Chobit-Result.jsonp");
var result = await client.GetSampleInfoAsync("RJ01430276", CancellationToken.None);
result.Count.ShouldBe(1);
result.Works.Length.ShouldBe(1);
ChobitWork work = result.Works[0];
work.WorkId.ShouldBe("70nh8");
work.DLSiteWorkId.ShouldBe("RJ01430276");
work.WorkName.ShouldBe("【ブルーアーカイブ】シグレ(温泉)ASMR～溶けていく温度を交わして～");
}
[Fact]
public async Task Deserialize_Chobit_Sample_Info_Collection()
{
var client = await GetChobitClientThatReturns("Sample-Chobit-Result-Collection.jsonp");
var result = await client.GetSampleInfoAsync(["RJ01430276"], CancellationToken.None);
result.Count.ShouldBe(1);
result.ShouldContainKey("RJ01430276");
result["RJ01430276"].Count.ShouldBe(1);
result["RJ01430276"].Works.Length.ShouldBe(1);
ChobitWork work = result["RJ01430276"].Works[0];
work.WorkId.ShouldBe("70nh8");
work.DLSiteWorkId.ShouldBe("RJ01430276");
work.WorkName.ShouldBe("【ブルーアーカイブ】シグレ(温泉)ASMR～溶けていく温度を交わして～");
}
[Fact]
public async Task Deserialize_Chobit_Sample_Info_No_Data()
{
var client = await GetChobitClientThatReturns("Sample-Chobit-Result-No-Data.jsonp");
var result = await client.GetSampleInfoAsync("RJ01585659", CancellationToken.None);
result.Count.ShouldBe(0);
result.Works.Length.ShouldBe(0);
}
[Fact]
public async Task Deserialize_Chobit_Sample_Info_Collection_No_Data()
{
var client = await GetChobitClientThatReturns("Sample-Chobit-Result-Collection-No-Data.jsonp");
var result = await client.GetSampleInfoAsync(["RJ01585659"], CancellationToken.None);
result.Count.ShouldBe(1);
result.ShouldContainKey("RJ01585659");
result["RJ01585659"].Count.ShouldBe(0);
result["RJ01585659"].Works.Length.ShouldBe(0);
}
}

1
JSMR.Tests/Integrations/Chobit/Sample-Chobit-Result-Collection-No-Data.jsonp Normal file
View File

@@ -0,0 +1 @@
response({"RJ01585659":{"count":0,"works":[]}})

1
JSMR.Tests/Integrations/Chobit/Sample-Chobit-Result-Collection.jsonp Normal file
View File

@@ -0,0 +1 @@
response({"RJ01430276":{"count":1,"works":[{"work_id":"70nh8","dlsite_work_id":"RJ01430276","work_name":"\u3010\u30d6\u30eb\u30fc\u30a2\u30fc\u30ab\u30a4\u30d6\u3011\u30b7\u30b0\u30ec(\u6e29\u6cc9)ASMR\uff5e\u6eb6\u3051\u3066\u3044\u304f\u6e29\u5ea6\u3092\u4ea4\u308f\u3057\u3066\uff5e","work_name_kana":"\u30d6\u30eb\u30fc\u30a2\u30fc\u30ab\u30a4\u30d6\u30b7\u30b0\u30ec\u30aa\u30f3\u30bb\u30f3\u30a8\u30fc\u30a8\u30b9\u30a8\u30e0\u30a2\u30fc\u30eb\u30c8\u30b1\u30c6\u30a4\u30af\u30aa\u30f3\u30c9\u30f2\u30ab\u30ef\u30b7\u30c6","url":"https:\/\/chobit.cc\/70nh8\/vgj6safb","embed_url":"https:\/\/chobit.cc\/embed\/70nh8\/vgj6safb?dlsite=1","thumb":"https:\/\/media.dlsite.com\/chobit\/contents\/2507\/f1kkssulligowgkkcc0k80soo\/f1kkssulligowgkkcc0k80soo_cover.jpg?w=560\u0026h=420","mini_thumb":"https:\/\/media.dlsite.com\/chobit\/contents\/2507\/f1kkssulligowgkkcc0k80soo\/f1kkssulligowgkkcc0k80soo_cover.jpg?w=100\u0026h=100","file_type":"audio","embed_width":740,"embed_height":215}]}})

1
JSMR.Tests/Integrations/Chobit/Sample-Chobit-Result-No-Data.jsonp Normal file
View File

@@ -0,0 +1 @@
response({"count":0,"works":[]})

1
JSMR.Tests/Integrations/Chobit/Sample-Chobit-Result.jsonp Normal file
View File

@@ -0,0 +1 @@
response({"count":1,"works":[{"work_id":"70nh8","dlsite_work_id":"RJ01430276","work_name":"\u3010\u30d6\u30eb\u30fc\u30a2\u30fc\u30ab\u30a4\u30d6\u3011\u30b7\u30b0\u30ec(\u6e29\u6cc9)ASMR\uff5e\u6eb6\u3051\u3066\u3044\u304f\u6e29\u5ea6\u3092\u4ea4\u308f\u3057\u3066\uff5e","work_name_kana":"\u30d6\u30eb\u30fc\u30a2\u30fc\u30ab\u30a4\u30d6\u30b7\u30b0\u30ec\u30aa\u30f3\u30bb\u30f3\u30a8\u30fc\u30a8\u30b9\u30a8\u30e0\u30a2\u30fc\u30eb\u30c8\u30b1\u30c6\u30a4\u30af\u30aa\u30f3\u30c9\u30f2\u30ab\u30ef\u30b7\u30c6","url":"https:\/\/chobit.cc\/70nh8\/vgj6safb","embed_url":"https:\/\/chobit.cc\/embed\/70nh8\/vgj6safb?dlsite=1","thumb":"https:\/\/media.dlsite.com\/chobit\/contents\/2507\/f1kkssulligowgkkcc0k80soo\/f1kkssulligowgkkcc0k80soo_cover.jpg?w=560\u0026h=420","mini_thumb":"https:\/\/media.dlsite.com\/chobit\/contents\/2507\/f1kkssulligowgkkcc0k80soo\/f1kkssulligowgkkcc0k80soo_cover.jpg?w=100\u0026h=100","file_type":"audio","embed_width":740,"embed_height":215}]})

37
JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs
View File

@@ -1,15 +1,16 @@
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Domain.Enums;
using JSMR.Domain.ValueObjects;
using JSMR.Infrastructure.Http;
using JSMR.Infrastructure.Integrations.DLSite;
using JSMR.Infrastructure.Integrations.DLSite.Mapping;
using JSMR.Infrastructure.Integrations.DLSite.Models;
using JSMR.Tests.Extensions;
using JSMR.Tests.Http;
using JSMR.Tests.Utilities;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Shouldly;
using System.Net;
using System.Text;
namespace JSMR.Tests.Integrations.DLSite;
@@ -20,17 +21,33 @@ public class DLSiteClientTests
return await ResourceHelper.ReadAsync($"JSMR.Tests.Integrations.DLSite.{resourceName}");
}
private static async Task<DLSiteClient> GetDLSiteClientThatReturns(string resourceName)
{
string content = await ReadJsonResourceAsync(resourceName);
FakeHttpMessageHandler handler = new(request =>
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(content, Encoding.UTF8, "application/json")
};
});
HttpClient httpClient = new(handler)
{
BaseAddress = new Uri("https://www.fake-dlsite.com/")
};
var logger = Substitute.For<ILogger<DLSiteClient>>();
var client = new DLSiteClient(httpClient, logger);
return client;
}
[Fact]
public async Task Deserialize_Product_Info_Collection()
{
string productInfoJson = await ReadJsonResourceAsync("Product-Info.json");
IHttpService httpService = Substitute.For<IHttpService>();
httpService.ReturnsContent(productInfoJson);
var logger = Substitute.For<ILogger<DLSiteClient>>();
var client = new DLSiteClient(httpService, logger);
var client = await GetDLSiteClientThatReturns("Product-Info.json");
var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163", "RJ01536422"], CancellationToken.None);
result.Count.ShouldBe(2);

4
JSMR.Tests/JSMR.Tests.csproj
View File

@@ -13,6 +13,10 @@
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Integrations\Chobit\Sample-Chobit-Result-Collection-No-Data.jsonp" />
<EmbeddedResource Include="Integrations\Chobit\Sample-Chobit-Result-No-Data.jsonp" />
<EmbeddedResource Include="Integrations\Chobit\Sample-Chobit-Result-Collection.jsonp" />
<EmbeddedResource Include="Integrations\Chobit\Sample-Chobit-Result.jsonp" />
<EmbeddedResource Include="Integrations\DLSite\Product-Info.json" />
<EmbeddedResource Include="Scanning\English-Page-Updated.html" />
<EmbeddedResource Include="Scanning\Japanese-Page-Updated.html" />

16
JSMR.Tests/Scanning/VoiceWorkScannerTests.cs
View File

@@ -1,4 +1,5 @@
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Chobit.Models;
using JSMR.Application.Integrations.DLSite.Models;
using JSMR.Application.Integrations.Ports;
using JSMR.Application.Scanning.Contracts;
using JSMR.Application.Scanning.Ports;
@@ -141,10 +142,21 @@ public class VoiceWorkScannerTests
}
};
ChobitResultCollection chobitResultCollection = new()
{
{
"RJ1",
new ChobitResult()
{
Count = 0
}
}
};
dlsiteClient.GetVoiceWorkDetailsAsync(Arg.Any<string[]>(), CancellationToken.None)
.Returns(Task.FromResult(detailCollection));
VoiceWorkIngest ingest = VoiceWorkIngest.From(scannedWorks[0], detailCollection["RJ1"]);
VoiceWorkIngest ingest = VoiceWorkIngest.From(scannedWorks[0], detailCollection["RJ1"], chobitResultCollection["RJ1"]);
// TODO: Test other fields
ingest.Title.ShouldBe("Product Title");

1
JSMR.Worker/JSMR.Worker.csproj
View File

@@ -34,7 +34,6 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\JSMR.Application\JSMR.Application.csproj" />
<ProjectReference Include="..\JSMR.Infrastructure\JSMR.Infrastructure.csproj" />
</ItemGroup>

8
JSMR.Worker/Properties/launchSettings.json
View File

@@ -8,6 +8,14 @@
},
"workingDirectory": ""
},
"Scan (JP, 5 pages)": {
"commandName": "Project",
"commandLineArgs": "scan --locale Japanese --start 1 --end 5 --pageSize 100",
"environmentVariables": {
"DOTNET_ENVIRONMENT": "Development"
},
"workingDirectory": ""
},
"Scan (EN, 3 pages)": {
"commandName": "Project",
"commandLineArgs": "scan --locale English --start 1 --end 3 --pageSize 100",