Fixed scanning issue. Updated worker.

JSMR.Infrastructure/DI/InfrastructureServiceCollectionExtensions.cs

@@ -48,6 +48,7 @@ public static class InfrastructureServiceCollectionExtensions
 
         services.AddKeyedScoped<IVoiceWorkUpdater, VoiceWorkUpdater>(Locale.Japanese);
         services.AddKeyedScoped<IVoiceWorkUpdater, EnglishVoiceWorkUpdater>(Locale.English);
+        services.AddScoped<IVoiceWorkSearchUpdater, VoiceWorkSearchUpdater>();
 
         //services.AddKeyedScoped<ISupportedLanguage, JapaneseLanguage>(Locale.Japanese);
         //services.AddKeyedScoped<ISupportedLanguage, EnglishLanguage>(Locale.English);
@@ -64,27 +65,34 @@ public static class InfrastructureServiceCollectionExtensions
         services.AddSingleton<ICache, MemoryCacheAdapter>();
         services.AddSingleton<ISpamCircleCache, SpamCircleCache>();
 
-        services.AddHttpClient<IHttpService, HttpService>(client =>
-        {
-            client.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
-        });
-
-        services.AddScoped<IHttpService, HttpService>();
-        services.AddScoped<IHtmlLoader, HtmlLoader>();
-
         services.AddSingleton<ILanguageIdentifier, LanguageIdentifier>();
 
         services.AddSingleton<IClock, Clock>();
         services.AddSingleton<ITimeProvider, TokyoTimeProvider>();
 
-        services.AddHttpClient<IDLSiteClient, DLSiteClient>(httpClient =>
+        services.AddHttpServices();
+
+        return services;
+    }
+
+    private static IServiceCollection AddHttpServices(this IServiceCollection services)
+    {
+        //services.AddHttpClient<IHttpService, HttpService>(client =>
+        //{
+        //    client.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
+        //});
+
+        //services.AddScoped<IHttpService, HttpService>();
+        services.AddScoped<IHtmlLoader, HtmlLoader>();
+
+        // ONE registration for IHttpService as a typed client:
+        services.AddHttpClient<IHttpService, HttpService>((sp, http) =>
         {
-            httpClient.BaseAddress = new Uri("https://www.dlsite.com/");
+            http.BaseAddress = new Uri("https://www.dlsite.com/");
-            httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0 (+contact@example.com)");
+            http.DefaultRequestHeaders.UserAgent.ParseAdd("JSMR/1.0");
-            httpClient.Timeout = TimeSpan.FromSeconds(15);
+            http.Timeout = TimeSpan.FromSeconds(15);
         })
-        .AddResilienceHandler("dlsite", builder =>
+        .AddResilienceHandler("dlsite", builder => {
-        {
             builder.AddRetry(new HttpRetryStrategyOptions
             {
                 MaxRetryAttempts = 3,
@@ -93,21 +101,13 @@ public static class InfrastructureServiceCollectionExtensions
                 BackoffType = DelayBackoffType.Exponential,
                 ShouldHandle = new PredicateBuilder<HttpResponseMessage>()
                     .Handle<HttpRequestException>()
-                    .HandleResult(msg =>
+                    .HandleResult(r => (int)r.StatusCode >= 500 || (int)r.StatusCode == 429)
-                        msg.StatusCode == (HttpStatusCode)429 ||
-                        (int)msg.StatusCode >= 500)
             });
-
-            // (Optional) add a circuit breaker:
-            // builder.AddCircuitBreaker(new HttpCircuitBreakerStrategyOptions
-            // {
-            //     FailureRatio = 0.2,
-            //     SamplingDuration = TimeSpan.FromSeconds(30),
-            //     MinimumThroughput = 20,
-            //     BreakDuration = TimeSpan.FromSeconds(15)
-            // });
         });
 
+        // Register DLSiteClient as a normal scoped service
+        services.AddScoped<IDLSiteClient, DLSiteClient>();
+
         return services;
     }
 }

JSMR.Infrastructure/Data/AppDbContextFactory.cs

@@ -0,0 +1,30 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Design;
+using Microsoft.Extensions.Configuration;
+
+namespace JSMR.Infrastructure.Data;
+
+public sealed class AppDbContextFactory : IDesignTimeDbContextFactory<AppDbContext>
+{
+    public AppDbContext CreateDbContext(string[] args)
+    {
+        // adjust base path if needed (points to the worker for secrets/env)
+        var config = new ConfigurationBuilder()
+            .SetBasePath(Directory.GetCurrentDirectory())
+            .AddJsonFile("appsettings.json", optional: true)
+            .AddJsonFile("appsettings.Development.json", optional: true)
+            .AddUserSecrets(typeof(AppDbContextFactory).Assembly, optional: true)
+            .AddEnvironmentVariables()
+            .Build();
+
+        var conn = config.GetConnectionString("AppDb")
+                   ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb");
+
+        var options = new DbContextOptionsBuilder<AppDbContext>()
+            .UseMySql(conn, ServerVersion.AutoDetect(conn))
+            .EnableSensitiveDataLogging(false)
+            .Options;
+
+        return new AppDbContext(options);
+    }
+}

JSMR.Infrastructure/Integrations/DLSite/DLSiteClient.cs

@@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging;
 
 namespace JSMR.Infrastructure.Integrations.DLSite;
 
-public class DLSiteClient(IHttpService http, ILogger logger) : ApiClient(http, logger), IDLSiteClient
+public class DLSiteClient(IHttpService http, ILogger<DLSiteClient> logger) : ApiClient(http, logger), IDLSiteClient
 {
     public async Task<VoiceWorkDetailCollection> GetVoiceWorkDetailsAsync(string[] productIds, CancellationToken cancellationToken = default)
     {

JSMR.Infrastructure/Integrations/DLSite/Serialization/DictionaryConverter.cs

@@ -6,8 +6,31 @@ namespace JSMR.Infrastructure.Integrations.DLSite.Serialization;
 public sealed class DictionaryConverter<TKey, TValue> : JsonConverter<Dictionary<TKey, TValue>> where TKey : notnull
 {
     public override Dictionary<TKey, TValue>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
-        => JsonSerializer.Deserialize<Dictionary<TKey, TValue>>(ref reader, options);
+    {
+        if (reader.TokenType == JsonTokenType.Null)
+            return null;
+
+        if (reader.TokenType == JsonTokenType.StartArray)
+        {
+            if (!reader.Read())
+                throw new JsonException("Unexpected end while reading array.");
+
+            if (reader.TokenType != JsonTokenType.EndArray)
+                throw new JsonException("Non-empty JSON array cannot be converted to Dictionary.");
+
+            return [];
+        }
+
+        if (reader.TokenType == JsonTokenType.StartObject)
+        {
+            return JsonSerializer.Deserialize<Dictionary<TKey, TValue>>(ref reader, options) ?? [];
+        }
+
+        throw new JsonException($"Unexpected token {reader.TokenType} when reading Dictionary.");
+    }
 
     public override void Write(Utf8JsonWriter writer, Dictionary<TKey, TValue> value, JsonSerializerOptions options)
-        => JsonSerializer.Serialize(writer, value, options);
+    {
+        JsonSerializer.Serialize(writer, (IDictionary<TKey, TValue>)value, options);
+    }
 }

JSMR.Infrastructure/JSMR.Infrastructure.csproj

@@ -19,6 +19,9 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.10" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.10.0" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" />

JSMR.Tests/Integrations/DLSite/DLSiteClientTests.cs

@@ -32,9 +32,9 @@ public class DLSiteClientTests
         var logger = Substitute.For<ILogger<DLSiteClient>>();
         var client = new DLSiteClient(httpService, logger);
 
-        var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163"], CancellationToken.None);
+        var result = await client.GetVoiceWorkDetailsAsync(["RJ01230163", "RJ01536422"], CancellationToken.None);
 
-        result.Count.ShouldBe(1);
+        result.Count.ShouldBe(2);
 
         result.ShouldContainKey("RJ01230163");
         result["RJ01230163"].HasTrial.ShouldBeTrue();
@@ -44,6 +44,9 @@ public class DLSiteClientTests
         result["RJ01230163"].SupportedLanguages.Select(x => x.Language).ShouldContain(Language.English);
         result["RJ01230163"].DownloadCount.ShouldBe(659);
         result["RJ01230163"].WishlistCount.ShouldBe(380);
+
+        // Testing this one for empty array of prices
+        result.ShouldContainKey("RJ01536422");
     }
 
     [Fact]

JSMR.Tests/Integrations/DLSite/Product-Info.json

@@ -219,5 +219,94 @@
       }
     ],
     "default_point_str": "150"
+  },
+  "RJ01536422": {
+    "site_id": "home",
+    "site_id_touch": "hometouch",
+    "maker_id": "RG36156",
+    "age_category": 1,
+    "affiliate_deny": 0,
+    "dl_count": null,
+    "wishlist_count": 94,
+    "dl_format": 0,
+    "rank": [],
+    "rate_average": null,
+    "rate_average_2dp": null,
+    "rate_average_star": null,
+    "rate_count": null,
+    "rate_count_detail": [],
+    "review_count": 0,
+    "price": null,
+    "price_without_tax": null,
+    "price_str": "0",
+    "default_point_rate": 5,
+    "default_point": 0,
+    "product_point_rate": null,
+    "dlsiteplay_work": false,
+    "is_ana": true,
+    "is_sale": true,
+    "on_sale": 0,
+    "is_discount": false,
+    "is_pointup": false,
+    "gift": [],
+    "is_rental": false,
+    "work_rentals": [],
+    "upgrade_min_price": 110,
+    "down_url": "https:\/\/www.dlsite.com\/maniax\/download\/=\/product_id\/RJ01536422.html",
+    "is_tartget": null,
+    "title_id": null,
+    "title_name": null,
+    "title_name_masked": null,
+    "title_volumn": null,
+    "title_work_count": null,
+    "is_title_completed": false,
+    "bulkbuy_key": null,
+    "bonuses": [],
+    "is_limit_work": false,
+    "is_sold_out": false,
+    "limit_stock": 0,
+    "is_reserve_work": false,
+    "is_reservable": false,
+    "is_timesale": false,
+    "timesale_stock": 0,
+    "is_free": false,
+    "is_oly": false,
+    "is_led": false,
+    "is_noreduction": false,
+    "is_wcc": false,
+    "translation_info": {
+      "is_translation_agree": false,
+      "is_volunteer": false,
+      "is_original": true,
+      "is_parent": false,
+      "is_child": false,
+      "is_translation_bonus_child": false,
+      "original_workno": null,
+      "parent_workno": null,
+      "child_worknos": [],
+      "lang": null,
+      "production_trade_price_rate": 0,
+      "translation_bonus_langs": [],
+      "translation_status_for_translator": []
+    },
+    "work_name": "\u73c8\u7432\u5c4b \u7db4 \/ \u3044\u3064\u3082\u3044\u3064\u3067\u3082\u301cAlone with you\u301c",
+    "work_name_masked": "\u73c8\u7432\u5c4b \u7db4 \/ \u3044\u3064\u3082\u3044\u3064\u3067\u3082\u301cAlone with you\u301c",
+    "work_image": "\/\/img.dlsite.jp\/modpub\/images2\/ana\/doujin\/RJ01537000\/RJ01536422_ana_img_main.jpg",
+    "sales_end_info": null,
+    "voice_pack": null,
+    "regist_date": "2026-01-01 00:00:00",
+    "locale_price": [],
+    "locale_price_str": [],
+    "currency_price": null,
+    "work_type": "SOU",
+    "book_type": null,
+    "discount_calc_type": null,
+    "is_garumani_general": false,
+    "is_pack_work": false,
+    "limited_free_terms": [],
+    "official_price": null,
+    "options": "JPN#SND",
+    "custom_genres": [],
+    "default_point_str": "0"
   }
 }

JSMR.Worker/JSMR.Worker.csproj

@@ -5,11 +5,30 @@
     <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <UserSecretsId>f4ef1bd4-0cc5-4663-8108-fbb9c9eef5ae</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="appsettings.json" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="appsettings.Development.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.10">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.12" />
     <PackageReference Include="Serilog" Version="4.3.0" />
+    <PackageReference Include="Serilog.Settings.Configuration" Version="9.0.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.2" />
   </ItemGroup>
 

JSMR.Worker/Options/ScanOptions.cs

@@ -2,7 +2,7 @@
 
 public sealed class ScanOptions
 {
-    public string? Locale { get; init; } = "Japanese"; // maps to your Locale enum
+    public string Locale { get; init; } = "Japanese"; // maps to your Locale enum
     public int? StartPage { get; init; }              // if null, resume from checkpoint or 1
     public int? EndPage { get; init; }                // optional cap
     public int? PageSize { get; init; }               // override default

JSMR.Worker/Program.cs

@@ -11,16 +11,30 @@ using System.CommandLine;
 
 HostApplicationBuilder builder = Host.CreateApplicationBuilder(args);
 
+// Build a single configuration pipeline
+builder.Configuration
+    .SetBasePath(builder.Environment.ContentRootPath)
+    .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
+    .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName}.json", optional: true, reloadOnChange: true)
+    // Add user secrets (works regardless of environment when optional: true)
+    .AddUserSecrets(typeof(Program).Assembly, optional: true)
+    .AddEnvironmentVariables();
+
+// Pull the connection string from config (appsettings or secrets or env)
+string connectionString = builder.Configuration.GetConnectionString("AppDb")
+    ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb");
+
 //builder.Services.AddSerilog(o => o
 //    .WriteTo.Console()
 //    .MinimumLevel.Information());
 
 builder.Services
     .AddApplication()
-    .AddInfrastructure();
+    .AddInfrastructure()
+    .AddMemoryCache();
 
-string connectionString = builder.Configuration.GetConnectionString("AppDb")
+//string connectionString = builder.Configuration.GetConnectionString("AppDb")
-         ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb2");
+//         ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb");
 
 builder.Services.AddDbContextFactory<AppDbContext>(optionsBuilder =>
     optionsBuilder
@@ -106,4 +120,21 @@ rootCommand.Add(scan);
 
 //rootCommand.SetAction(async (parseResult, cancellationToken) => await rootCommand.InvokeAsync("scan"));
 
+Command schemaDumpCommand = new("schema-dump", "Emit EF model as a full create script (desired.sql)");
+
+schemaDumpCommand.SetAction(async (parseResult, cancellationToken) =>
+{
+    using var host = builder.Build();
+    await using var scope = host.Services.CreateAsyncScope();
+    var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+    var sql = db.Database.GenerateCreateScript();
+    var outPath = Path.GetFullPath("desired.sql");
+    await File.WriteAllTextAsync(outPath, sql);
+
+    Console.WriteLine($"[OK] Wrote EF model create script to: {outPath}");
+});
+
+rootCommand.Add(schemaDumpCommand);
+
 return await rootCommand.Parse(args).InvokeAsync();

JSMR.Worker/Properties/launchSettings.json

@@ -0,0 +1,26 @@
+{
+  "profiles": {
+    "Scan (JP, 1 page)": {
+      "commandName": "Project",
+      "commandLineArgs": "scan --locale Japanese --start 1 --end 1 --pageSize 100",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      },
+      "workingDirectory": ""
+    },
+    "Scan (EN, 3 pages)": {
+      "commandName": "Project",
+      "commandLineArgs": "scan --locale English --start 1 --end 3 --pageSize 100",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      }
+    },
+    "Watch (JP, every 5m)": {
+      "commandName": "Project",
+      "commandLineArgs": "scan --locale Japanese --start 1 --watch --every 00:05:00 --pageSize 100",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

JSMR.Worker/Services/ScanRunner.cs

@@ -1,8 +1,10 @@
 using JSMR.Application.Enums;
 using JSMR.Application.Scanning;
+using JSMR.Infrastructure.Common.Time;
 using JSMR.Worker.Options;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using System.Globalization;
 
 namespace JSMR.Worker.Services;
 
@@ -21,6 +23,10 @@ public sealed class PagedScanRunner(
         int startPage = options.StartPage
             ?? (await checkpoints.GetLastPageAsync(options.Locale, cancellationToken)).GetValueOrDefault(0) + 1;
 
+        ITimeProvider timeProvider = serviceProvider.GetRequiredService<ITimeProvider>();
+
+        log.LogInformation("Scanning on {ScanTime}...", timeProvider.Now().DateTime.ToString(CultureInfo.CurrentCulture));
+
         while (!cancellationToken.IsCancellationRequested)
         {
             int currentPage = startPage;
@@ -29,7 +35,8 @@ public sealed class PagedScanRunner(
             // Iterate until empty page or end reached
             for (; !cancellationToken.IsCancellationRequested && (!end.HasValue || currentPage <= end.Value); currentPage++)
             {
-                ScanVoiceWorksHandler handler = serviceProvider.GetRequiredService<ScanVoiceWorksHandler>();
+                using var scope = serviceProvider.CreateScope();
+                ScanVoiceWorksHandler handler = scope.ServiceProvider.GetRequiredService<ScanVoiceWorksHandler>();
 
                 log.LogInformation("Scanning page {Page} (size {Size}, locale {Locale})…", currentPage, pageSize, locale);
 

JSMR.Worker/appsettings.Development.json

@@ -0,0 +1,11 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "ConnectionStrings": {
+    "AppDb": "Server=localhost;Port=3306;User=root;Password=password;database=VoiceWorks;SslMode=none"
+  }
+}

JSMR.Worker/appsettings.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}