diff --git a/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs b/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs index 20b7502..2f11c0c 100644 --- a/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs +++ b/JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs @@ -48,6 +48,7 @@ public static class InfrastructureServiceCollectionExtensions services.AddKeyedScoped(Locale.Japanese); services.AddKeyedScoped(Locale.English); + services.AddScoped(); //services.AddKeyedScoped(Locale.Japanese); //services.AddKeyedScoped(Locale.English); @@ -64,27 +65,34 @@ public static class InfrastructureServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); - services.AddHttpClient(client => - { - client.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0"); - }); - - services.AddScoped(); - services.AddScoped(); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddHttpClient(httpClient => + services.AddHttpServices(); + + return services; + } + + private static IServiceCollection AddHttpServices(this IServiceCollection services) + { + //services.AddHttpClient(client => + //{ + // client.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0"); + //}); + + //services.AddScoped(); + services.AddScoped(); + + // ONE registration for IHttpService as a typed client: + services.AddHttpClient((sp, http) => { - httpClient.BaseAddress = new Uri("https://www.dlsite.com/"); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0 (+contact@example.com)"); - httpClient.Timeout = TimeSpan.FromSeconds(15); + http.BaseAddress = new Uri("https://www.dlsite.com/"); + http.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0"); + http.Timeout = TimeSpan.FromSeconds(15); }) - .AddResilienceHandler("dlsite", builder => - { + .AddResilienceHandler("dlsite", builder => { builder.AddRetry(new HttpRetryStrategyOptions { MaxRetryAttempts = 3, @@ -93,21 +101,13 @@ public static class InfrastructureServiceCollectionExtensions BackoffType = DelayBackoffType.Exponential, ShouldHandle = new PredicateBuilder() .Handle() - .HandleResult(msg => - msg.StatusCode == (HttpStatusCode)429 || - (int)msg.StatusCode >= 500) + .HandleResult(r => (int)r.StatusCode >= 500 || (int)r.StatusCode == 429) }); - - // (Optional) add a circuit breaker: - // builder.AddCircuitBreaker(new HttpCircuitBreakerStrategyOptions - // { - // FailureRatio = 0.2, - // SamplingDuration = TimeSpan.FromSeconds(30), - // MinimumThroughput = 20, - // BreakDuration = TimeSpan.FromSeconds(15) - // }); }); + // Register DLSiteClient as a normal scoped service + services.AddScoped(); + return services; } } \ No newline at end of file diff --git a/JSMR.Infrastructure/Data/AppDbContextFactory.cs b/JSMR.Infrastructure/Data/AppDbContextFactory.cs new file mode 100644 index 0000000..420dba4 --- /dev/null +++ b/JSMR.Infrastructure/Data/AppDbContextFactory.cs @@ -0,0 +1,30 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Configuration; + +namespace JSMR.Infrastructure.Data; + +public sealed class AppDbContextFactory : IDesignTimeDbContextFactory +{ + public AppDbContext CreateDbContext(string[] args) + { + // adjust base path if needed (points to the worker for secrets/env) + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddJsonFile("appsettings.Development.json", optional: true) + .AddUserSecrets(typeof(AppDbContextFactory).Assembly, optional: true) + .AddEnvironmentVariables() + .Build(); + + var conn = config.GetConnectionString("AppDb") + ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb"); + + var options = new DbContextOptionsBuilder() + .UseMySql(conn, ServerVersion.AutoDetect(conn)) + .EnableSensitiveDataLogging(false) + .Options; + + return new AppDbContext(options); + } +} \ No newline at end of file diff --git a/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs b/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs index 0406b9a..fb81d46 100644 --- a/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs +++ b/JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging; namespace JSMR.Infrastructure.Integrations.DLSite; -public class DLSiteClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IDLSiteClient +public class DLSiteClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IDLSiteClient { public async Task GetVoiceWorkDetailsAsync(string[] productIds, CancellationToken cancellationToken = default) { diff --git a/JSMR.Infrastructure/Integrations/DLSite/Serialization/DictionaryConverter.cs b/JSMR.Infrastructure/Integrations/DLSite/Serialization/DictionaryConverter.cs index 6a510d8..ebd18cb 100644 --- a/JSMR.Infrastructure/Integrations/DLSite/Serialization/DictionaryConverter.cs +++ b/JSMR.Infrastructure/Integrations/DLSite/Serialization/DictionaryConverter.cs @@ -6,8 +6,31 @@ namespace JSMR.Infrastructure.Integrations.DLSite.Serialization; public sealed class DictionaryConverter : JsonConverter> where TKey : notnull { public override Dictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - => JsonSerializer.Deserialize>(ref reader, options); + { + if (reader.TokenType == JsonTokenType.Null) + return null; + + if (reader.TokenType == JsonTokenType.StartArray) + { + if (!reader.Read()) + throw new JsonException("Unexpected end while reading array."); + + if (reader.TokenType != JsonTokenType.EndArray) + throw new JsonException("Non-empty JSON array cannot be converted to Dictionary."); + + return []; + } + + if (reader.TokenType == JsonTokenType.StartObject) + { + return JsonSerializer.Deserialize>(ref reader, options) ?? []; + } + + throw new JsonException($"Unexpected token {reader.TokenType} when reading Dictionary."); + } public override void Write(Utf8JsonWriter writer, Dictionary value, JsonSerializerOptions options) - => JsonSerializer.Serialize(writer, value, options); + { + JsonSerializer.Serialize(writer, (IDictionary)value, options); + } } \ No newline at end of file diff --git a/JSMR.Infrastructure/JSMR.Infrastructure.csproj b/JSMR.Infrastructure/JSMR.Infrastructure.csproj index 89b1ce2..d61d50c 100644 --- a/JSMR.Infrastructure/JSMR.Infrastructure.csproj +++ b/JSMR.Infrastructure/JSMR.Infrastructure.csproj @@ -19,6 +19,9 @@ + + + diff --git a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs index a525f3d..b90bf15 100644 --- a/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs +++ b/JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs @@ -32,9 +32,9 @@ public class DLSiteClientTests var logger = Substitute.For>(); var client = new DLSiteClient(httpService, logger); - var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163"], CancellationToken.None); + var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163", "RJ01536422"], CancellationToken.None); - result.Count.ShouldBe(1); + result.Count.ShouldBe(2); result.ShouldContainKey("RJ01230163"); result["RJ01230163"].HasTrial.ShouldBeTrue(); @@ -44,6 +44,9 @@ public class DLSiteClientTests result["RJ01230163"].SupportedLanguages.Select(x => x.Language).ShouldContain(Language.English); result["RJ01230163"].DownloadCount.ShouldBe(659); result["RJ01230163"].WishlistCount.ShouldBe(380); + + // Testing this one for empty array of prices + result.ShouldContainKey("RJ01536422"); } [Fact] diff --git a/JSMR.Tests/Integrations/DLSite/Product-Info.json b/JSMR.Tests/Integrations/DLSite/Product-Info.json index 3dc9cbb..9dd873b 100644 --- a/JSMR.Tests/Integrations/DLSite/Product-Info.json +++ b/JSMR.Tests/Integrations/DLSite/Product-Info.json @@ -219,5 +219,94 @@ } ], "default_point_str": "150" + }, + "RJ01536422": { + "site_id": "home", + "site_id_touch": "hometouch", + "maker_id": "RG36156", + "age_category": 1, + "affiliate_deny": 0, + "dl_count": null, + "wishlist_count": 94, + "dl_format": 0, + "rank": [], + "rate_average": null, + "rate_average_2dp": null, + "rate_average_star": null, + "rate_count": null, + "rate_count_detail": [], + "review_count": 0, + "price": null, + "price_without_tax": null, + "price_str": "0", + "default_point_rate": 5, + "default_point": 0, + "product_point_rate": null, + "dlsiteplay_work": false, + "is_ana": true, + "is_sale": true, + "on_sale": 0, + "is_discount": false, + "is_pointup": false, + "gift": [], + "is_rental": false, + "work_rentals": [], + "upgrade_min_price": 110, + "down_url": "https:\/\/www.dlsite.com\/maniax\/download\/=\/product_id\/RJ01536422.html", + "is_tartget": null, + "title_id": null, + "title_name": null, + "title_name_masked": null, + "title_volumn": null, + "title_work_count": null, + "is_title_completed": false, + "bulkbuy_key": null, + "bonuses": [], + "is_limit_work": false, + "is_sold_out": false, + "limit_stock": 0, + "is_reserve_work": false, + "is_reservable": false, + "is_timesale": false, + "timesale_stock": 0, + "is_free": false, + "is_oly": false, + "is_led": false, + "is_noreduction": false, + "is_wcc": false, + "translation_info": { + "is_translation_agree": false, + "is_volunteer": false, + "is_original": true, + "is_parent": false, + "is_child": false, + "is_translation_bonus_child": false, + "original_workno": null, + "parent_workno": null, + "child_worknos": [], + "lang": null, + "production_trade_price_rate": 0, + "translation_bonus_langs": [], + "translation_status_for_translator": [] + }, + "work_name": "\u73c8\u7432\u5c4b \u7db4 \/ \u3044\u3064\u3082\u3044\u3064\u3067\u3082\u301cAlone with you\u301c", + "work_name_masked": "\u73c8\u7432\u5c4b \u7db4 \/ \u3044\u3064\u3082\u3044\u3064\u3067\u3082\u301cAlone with you\u301c", + "work_image": "\/\/img.dlsite.jp\/modpub\/images2\/ana\/doujin\/RJ01537000\/RJ01536422_ana_img_main.jpg", + "sales_end_info": null, + "voice_pack": null, + "regist_date": "2026-01-01 00:00:00", + "locale_price": [], + "locale_price_str": [], + "currency_price": null, + "work_type": "SOU", + "book_type": null, + "discount_calc_type": null, + "is_garumani_general": false, + "is_pack_work": false, + "limited_free_terms": [], + "official_price": null, + "options": "JPN#SND", + "custom_genres": [], + "default_point_str": "0" } } \ No newline at end of file diff --git a/JSMR.Worker/JSMR.Worker.csproj b/JSMR.Worker/JSMR.Worker.csproj index b22fb16..d04094f 100644 --- a/JSMR.Worker/JSMR.Worker.csproj +++ b/JSMR.Worker/JSMR.Worker.csproj @@ -5,11 +5,30 @@ net9.0 enable enable + f4ef1bd4-0cc5-4663-8108-fbb9c9eef5ae + + + + + + PreserveNewest + + + PreserveNewest + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/JSMR.Worker/Options/ScanOptions.cs b/JSMR.Worker/Options/ScanOptions.cs index 6b29067..7814359 100644 --- a/JSMR.Worker/Options/ScanOptions.cs +++ b/JSMR.Worker/Options/ScanOptions.cs @@ -2,7 +2,7 @@ public sealed class ScanOptions { - public string? Locale { get; init; } = "Japanese"; // maps to your Locale enum + public string Locale { get; init; } = "Japanese"; // maps to your Locale enum public int? StartPage { get; init; } // if null, resume from checkpoint or 1 public int? EndPage { get; init; } // optional cap public int? PageSize { get; init; } // override default diff --git a/JSMR.Worker/Program.cs b/JSMR.Worker/Program.cs index 5bef40b..b18e2fb 100644 --- a/JSMR.Worker/Program.cs +++ b/JSMR.Worker/Program.cs @@ -11,16 +11,30 @@ using System.CommandLine; HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); +// Build a single configuration pipeline +builder.Configuration + .SetBasePath(builder.Environment.ContentRootPath) + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName}.json", optional: true, reloadOnChange: true) + // Add user secrets (works regardless of environment when optional: true) + .AddUserSecrets(typeof(Program).Assembly, optional: true) + .AddEnvironmentVariables(); + +// Pull the connection string from config (appsettings or secrets or env) +string connectionString = builder.Configuration.GetConnectionString("AppDb") + ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb"); + //builder.Services.AddSerilog(o => o // .WriteTo.Console() // .MinimumLevel.Information()); builder.Services .AddApplication() - .AddInfrastructure(); + .AddInfrastructure() + .AddMemoryCache(); -string connectionString = builder.Configuration.GetConnectionString("AppDb") - ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb2"); +//string connectionString = builder.Configuration.GetConnectionString("AppDb") +// ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb"); builder.Services.AddDbContextFactory(optionsBuilder => optionsBuilder @@ -106,4 +120,21 @@ rootCommand.Add(scan); //rootCommand.SetAction(async (parseResult, cancellationToken) => await rootCommand.InvokeAsync("scan")); +Command schemaDumpCommand = new("schema-dump", "Emit EF model as a full create script (desired.sql)"); + +schemaDumpCommand.SetAction(async (parseResult, cancellationToken) => +{ + using var host = builder.Build(); + await using var scope = host.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var sql = db.Database.GenerateCreateScript(); + var outPath = Path.GetFullPath("desired.sql"); + await File.WriteAllTextAsync(outPath, sql); + + Console.WriteLine($"[OK] Wrote EF model create script to: {outPath}"); +}); + +rootCommand.Add(schemaDumpCommand); + return await rootCommand.Parse(args).InvokeAsync(); \ No newline at end of file diff --git a/JSMR.Worker/Properties/launchSettings.json b/JSMR.Worker/Properties/launchSettings.json new file mode 100644 index 0000000..5e2372d --- /dev/null +++ b/JSMR.Worker/Properties/launchSettings.json @@ -0,0 +1,26 @@ +{ + "profiles": { + "Scan (JP, 1 page)": { + "commandName": "Project", + "commandLineArgs": "scan --locale Japanese --start 1 --end 1 --pageSize 100", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + }, + "workingDirectory": "" + }, + "Scan (EN, 3 pages)": { + "commandName": "Project", + "commandLineArgs": "scan --locale English --start 1 --end 3 --pageSize 100", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + }, + "Watch (JP, every 5m)": { + "commandName": "Project", + "commandLineArgs": "scan --locale Japanese --start 1 --watch --every 00:05:00 --pageSize 100", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} \ No newline at end of file diff --git a/JSMR.Worker/Services/ScanRunner.cs b/JSMR.Worker/Services/ScanRunner.cs index 074cfe8..84e846c 100644 --- a/JSMR.Worker/Services/ScanRunner.cs +++ b/JSMR.Worker/Services/ScanRunner.cs @@ -1,8 +1,10 @@ using JSMR.Application.Enums; using JSMR.Application.Scanning; +using JSMR.Infrastructure.Common.Time; using JSMR.Worker.Options; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using System.Globalization; namespace JSMR.Worker.Services; @@ -21,6 +23,10 @@ public sealed class PagedScanRunner( int startPage = options.StartPage ?? (await checkpoints.GetLastPageAsync(options.Locale, cancellationToken)).GetValueOrDefault(0) + 1; + ITimeProvider timeProvider = serviceProvider.GetRequiredService(); + + log.LogInformation("Scanning on {ScanTime}...", timeProvider.Now().DateTime.ToString(CultureInfo.CurrentCulture)); + while (!cancellationToken.IsCancellationRequested) { int currentPage = startPage; @@ -29,7 +35,8 @@ public sealed class PagedScanRunner( // Iterate until empty page or end reached for (; !cancellationToken.IsCancellationRequested && (!end.HasValue || currentPage <= end.Value); currentPage++) { - ScanVoiceWorksHandler handler = serviceProvider.GetRequiredService(); + using var scope = serviceProvider.CreateScope(); + ScanVoiceWorksHandler handler = scope.ServiceProvider.GetRequiredService(); log.LogInformation("Scanning page {Page} (size {Size}, locale {Locale})…", currentPage, pageSize, locale); diff --git a/JSMR.Worker/appsettings.Development.json b/JSMR.Worker/appsettings.Development.json new file mode 100644 index 0000000..faa1e74 --- /dev/null +++ b/JSMR.Worker/appsettings.Development.json @@ -0,0 +1,11 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "ConnectionStrings": { + "AppDb": "Server=localhost;Port=3306;User=root;Password=password;database=VoiceWorks;SslMode=none" + } +} \ No newline at end of file diff --git a/JSMR.Worker/appsettings.json b/JSMR.Worker/appsettings.json new file mode 100644 index 0000000..1b2d3ba --- /dev/null +++ b/JSMR.Worker/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file