Improved performance of integration tests.

.gitea/workflows/ci.yml

@@ -7,6 +7,8 @@ on:
 jobs:
   build-test:
     runs-on: self-hosted
+    env:
+      CI: "true"
     container:
       image: ghcr.io/catthehacker/ubuntu:act-latest
     steps:
@@ -22,5 +24,5 @@ jobs:
             nuget-${{ runner.os }}-
       - run: dotnet restore
       - run: dotnet build --configuration Release --no-restore
-      - run: dotnet test  --configuration Release --no-build --logger "trx;LogFileName=test-results.trx"
+      - run: dotnet test  --configuration Release --no-build -p:CI=true --logger "trx;LogFileName=test-results.trx"
 

JSMR.Tests/Fixtures/CircleSearchProviderFixture.cs

@@ -1,5 +1,6 @@
 using JSMR.Infrastructure.Data;
 using Microsoft.EntityFrameworkCore;
+using Org.BouncyCastle.Asn1.Pkcs;
 
 namespace JSMR.Tests.Fixtures;
 
@@ -33,8 +34,16 @@ public sealed class CircleSearchProviderFixture2(MariaDbContainerFixture contain
 
     public async ValueTask InitializeAsync()
     {
-        DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //    container.RootConnectionString,
+        //    seed: SeedAsync);
+
+        var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        DbContext = await MariaDbClone.CloneFromTemplateAsync(
             container.RootConnectionString,
+            container.TemplateDbName,
+            newDbName: newDb,
             seed: SeedAsync);
     }
 

JSMR.Tests/Fixtures/CreatorSearchProviderFixture.cs

@@ -32,8 +32,16 @@ public sealed class CreatorSearchProviderFixture2(MariaDbContainerFixture contai
 
     public async ValueTask InitializeAsync()
     {
-        DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //    container.RootConnectionString,
+        //    seed: SeedAsync);
+
+        var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        DbContext = await MariaDbClone.CloneFromTemplateAsync(
             container.RootConnectionString,
+            container.TemplateDbName,
+            newDbName: newDb,
             seed: SeedAsync);
     }
 

JSMR.Tests/Fixtures/MariaDbClone.cs

@@ -0,0 +1,126 @@
+using JSMR.Infrastructure.Data;
+using Microsoft.EntityFrameworkCore;
+using MySqlConnector;
+using System.Data;
+
+namespace JSMR.Tests.Fixtures;
+
+public static class MariaDbClone
+{
+    public static async Task CreateTemplateAsync(
+        string rootConn,
+        string templateDbName,
+        Func<AppDbContext, Task>? seedAsync = null)
+    {
+        await using var root = new MySqlConnection(rootConn);
+        await root.OpenAsync();
+
+        await ExecAsync(root, $"DROP DATABASE IF EXISTS `{templateDbName}`;");
+        await ExecAsync(root, $"CREATE DATABASE `{templateDbName}` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;");
+
+        // Run EF once to build schema and seed
+        var templateConn = new MySqlConnectionStringBuilder(rootConn) { Database = templateDbName }.ConnectionString;
+        await using var ctx = AppDb(templateConn);
+        await ctx.Database.EnsureCreatedAsync();
+
+        if (seedAsync != null)
+            await seedAsync(ctx);
+    }
+
+    public static async Task<AppDbContext> CloneFromTemplateAsync(
+        string rootConn,
+        string templateDbName,
+        string newDbName,
+        Func<AppDbContext, Task>? seed = null)
+    {
+        var newConnStr = new MySqlConnectionStringBuilder(rootConn) { Database = newDbName }.ConnectionString;
+
+        await using var root = new MySqlConnection(rootConn);
+        await root.OpenAsync();
+
+        // Create target DB
+        await ExecAsync(root, $"CREATE DATABASE `{newDbName}` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;");
+
+        // Disable FK checks while recreating schema & loading data
+        await ExecAsync(root, $"SET SESSION sql_log_bin = 0;"); // avoid binlog noise (optional)
+        await ExecAsync(root, $"SET SESSION foreign_key_checks = 0;");
+
+        // 1) Create tables using exact DDL from templatedb
+        var tables = await GetTableNamesAsync(root, templateDbName);
+
+        foreach (var table in tables)
+        {
+            var createTable = await ShowCreateTableAsync(root, templateDbName, table);
+            // Run DDL in the new DB (the CREATE statement itself doesn't include db name)
+            await ExecAsync(root, $"USE `{newDbName}`; {createTable};");
+        }
+
+        // 2) Copy data
+        foreach (var table in tables)
+        {
+            var sql = $"INSERT INTO `{newDbName}`.`{table}` SELECT * FROM `{templateDbName}`.`{table}`;";
+            await ExecAsync(root, sql);
+        }
+
+        await ExecAsync(root, $"SET SESSION foreign_key_checks = 1;");
+        await ExecAsync(root, $"SET SESSION sql_log_bin = 1;");
+
+        // Ready-to-use EF context for the cloned DB
+        //return AppDb(newConnStr);
+
+        AppDbContext dbContext = AppDb(newConnStr);
+
+        if (seed != null)
+            await seed(dbContext);
+
+        return dbContext;
+    }
+
+    private static AppDbContext AppDb(string connStr)
+    {
+        var opts = new DbContextOptionsBuilder<AppDbContext>()
+            .UseMySql(connStr, ServerVersion.AutoDetect(connStr), o => o.EnableRetryOnFailure())
+            .EnableSensitiveDataLogging()
+            .Options;
+
+        return new AppDbContext(opts);
+    }
+
+    private static async Task<List<string>> GetTableNamesAsync(MySqlConnection conn, string db)
+    {
+        const string sql = @"
+SELECT TABLE_NAME 
+FROM information_schema.TABLES 
+WHERE TABLE_SCHEMA = @db AND TABLE_TYPE = 'BASE TABLE';";
+
+        var result = new List<string>();
+        await using var cmd = new MySqlCommand(sql, conn) { Parameters = { new("@db", db) } };
+        await using var rdr = await cmd.ExecuteReaderAsync();
+        while (await rdr.ReadAsync())
+            result.Add(rdr.GetString(0));
+        return result;
+    }
+
+    private static async Task<string> ShowCreateTableAsync(MySqlConnection conn, string db, string table)
+    {
+        await using var cmd = new MySqlCommand($"SHOW CREATE TABLE `{db}`.`{table}`;", conn);
+        await using var rdr = await cmd.ExecuteReaderAsync(CommandBehavior.SingleRow);
+        if (!await rdr.ReadAsync())
+            throw new InvalidOperationException($"SHOW CREATE TABLE returned no rows for {db}.{table}");
+
+        // Column 1: table name, Column 2: CREATE TABLE DDL
+        var ddl = rdr.GetString(1);
+
+        // Ensure the statement ends with ';'
+        if (!ddl.TrimEnd().EndsWith(";"))
+            ddl += ";";
+
+        return ddl;
+    }
+
+    private static async Task ExecAsync(MySqlConnection conn, string sql)
+    {
+        await using var cmd = new MySqlCommand(sql, conn) { CommandTimeout = 0 };
+        await cmd.ExecuteNonQueryAsync();
+    }
+}

JSMR.Tests/Fixtures/MariaDbContainerFixture.cs

@@ -0,0 +1,42 @@
+using DotNet.Testcontainers.Builders;
+using JSMR.Tests.Fixtures;
+using JSMR.Tests.Ingestion;
+using Testcontainers.MariaDb;
+
+[assembly: AssemblyFixture(typeof(MariaDbContainerFixture))]
+
+namespace JSMR.Tests.Fixtures;
+
+public sealed class MariaDbContainerFixture : IAsyncLifetime
+{
+    const int MajorVersion = 10;
+    const int MinorVersion = 11;
+    const int Build = 6;
+
+    private MariaDbContainer _container = default!;
+
+    public string RootConnectionString { get; private set; } = default!;
+    public string TemplateDbName { get; } = "jsmr_template";
+
+    public async ValueTask InitializeAsync()
+    {
+        _container = new MariaDbBuilder()
+            .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}")
+            .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw")
+            .WithUsername("root")
+            .WithPassword("rootpw")
+            .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306))
+            .Build();
+
+        await _container.StartAsync();
+
+        RootConnectionString = _container.GetConnectionString();
+
+        // Build the template ONCE with EF + your existing seed
+        await MariaDbClone.CreateTemplateAsync(
+            RootConnectionString,
+            TemplateDbName);
+    }
+
+    public async ValueTask DisposeAsync() => await _container.DisposeAsync();
+}

JSMR.Tests/Fixtures/MariaDbFixture.cs

@@ -1,66 +1,10 @@
-using DotNet.Testcontainers.Builders;
+using JSMR.Infrastructure.Data;
-using DotNet.Testcontainers.Containers;
-using JSMR.Infrastructure.Data;
-using JSMR.Tests.Fixtures;
 using Microsoft.EntityFrameworkCore;
 using MySqlConnector;
 using Testcontainers.MariaDb;
-using Testcontainers.Xunit;
-using Xunit.Sdk;
-
-[assembly: AssemblyFixture(typeof(MariaDbContainerFixture))]
 
 namespace JSMR.Tests.Fixtures;
 
-
-public sealed class MariaDbContainerFixture : IAsyncLifetime
-{
-    const int MajorVersion = 10;
-    const int MinorVersion = 11;
-    const int Build = 6;
-
-    private MariaDbContainer _container = default!;
-    public string RootConnectionString { get; private set; } = default!;
-
-    public async ValueTask InitializeAsync()
-    {
-        //_container = new ContainerBuilder()
-        //    .WithImage("mariadb:11")
-        //    .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw")
-        //    .WithPortBinding(3307, 3306)
-        //    .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(3306))
-        //    .Build();
-
-        //_container = new ContainerBuilder()
-        //   .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}")
-        //   .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw")
-        //   //.WithPortBinding(3307, 3306)
-        //   .WithPortBinding(3306, assignRandomHostPort: true)
-        //   .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306))
-        //   .Build();
-
-        _container = new MariaDbBuilder()
-            .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}")
-            .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw")
-            .WithUsername("root")
-            .WithPassword("rootpw")
-            // no explicit port binding
-            .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306))
-            .Build();
-
-        await _container.StartAsync();
-
-        // No database specified: we’ll create per-test DBs
-        //RootConnectionString = "Server=127.0.0.1;Port=3307;User=root;Password=rootpw;SslMode=none;";
-
-        RootConnectionString = _container.GetConnectionString();
-        var port = _container.GetMappedPublicPort(3306);
-        //RootConnectionString = $"Server=127.0.0.1;Port={port};User=root;Password=rootpw;SslMode=none;";
-    }
-
-    public async ValueTask DisposeAsync() => await _container.DisposeAsync();
-}
-
 public class MariaDbFixture : IAsyncLifetime
 {
     const int MajorVersion = 10;
@@ -125,31 +69,6 @@ public class MariaDbFixture : IAsyncLifetime
     }
 }
 
-[CollectionDefinition("db")]
-public sealed class MariaDbCollection : ICollectionFixture<MariaDbContainerFixture> { }
-
-//public class MariaDbAssemblyFixtureDefinition : IAssemblyFixture<MariaDbContainerFixture> { }
-
-
-//[UsedImplicitly]
-public sealed class MariaDbContainerFixture2(IMessageSink messageSink)
-    : ContainerFixture<MariaDbBuilder, MariaDbContainer>(messageSink)
-{
-    const int MajorVersion = 10;
-    const int MinorVersion = 11;
-    const int Build = 6;
-    
-    public string RootConnectionString => $"Server={Container.IpAddress};Port=3306;User=root;Password=rootpw;SslMode=none;";
-
-    protected override MariaDbBuilder Configure(MariaDbBuilder builder)
-    {
-        return builder.WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}")
-           .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw")
-           .WithPortBinding(3307, 3306)
-           //.WithPortBinding(3306, assignRandomHostPort: true)
-           .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306));
-    }
-}
 
 public static class MariaTestDb
 {

JSMR.Tests/Fixtures/TagSearchProviderFixture.cs

@@ -40,8 +40,16 @@ public sealed class TagSearchProviderFixture2(MariaDbContainerFixture container)
 
     public async ValueTask InitializeAsync()
     {
-        DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //    container.RootConnectionString,
+        //    seed: SeedAsync);
+
+        var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        DbContext = await MariaDbClone.CloneFromTemplateAsync(
             container.RootConnectionString,
+            container.TemplateDbName,
+            newDbName: newDb,
             seed: SeedAsync);
     }
 

JSMR.Tests/Fixtures/VoiceWorkSearchProviderFixture.cs

@@ -1,5 +1,6 @@
 using JSMR.Domain.Enums;
 using JSMR.Infrastructure.Data;
+using JSMR.Tests.Ingestion;
 using Microsoft.EntityFrameworkCore;
 
 namespace JSMR.Tests.Fixtures;
@@ -132,8 +133,16 @@ public sealed class VoiceWorkSearchProviderFixture2(MariaDbContainerFixture cont
 
     public async ValueTask InitializeAsync()
     {
-        DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //DbContext = await MariaTestDb.CreateIsolatedAsync(
+        //    container.RootConnectionString,
+        //    seed: SeedAsync);
+
+        var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        DbContext = await MariaDbClone.CloneFromTemplateAsync(
             container.RootConnectionString,
+            container.TemplateDbName,
+            newDbName: newDb,
             seed: SeedAsync);
     }
 

JSMR.Tests/Ingestion/Japanese/IngestionTestsBase.cs

@@ -4,6 +4,7 @@ using JSMR.Infrastructure.Common.Time;
 using JSMR.Infrastructure.Data;
 using JSMR.Infrastructure.Ingestion;
 using JSMR.Tests.Fixtures;
+using Microsoft.EntityFrameworkCore;
 using NSubstitute;
 using System.Runtime.InteropServices;
 
@@ -13,9 +14,17 @@ public abstract class IngestionTestsBase(MariaDbContainerFixture container)
 {
     protected async Task<AppDbContext> GetAppDbContextAsync()
     {
-        return await MariaTestDb.CreateIsolatedAsync(
+        //return await MariaTestDb.CreateIsolatedAsync(
-           container.RootConnectionString,
+        //   container.RootConnectionString,
-           seed: VoiceWorkIngestionSeedData.SeedAsync);
+        //   seed: VoiceWorkIngestionSeedData.SeedAsync);
+
+        var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        return await MariaDbClone.CloneFromTemplateAsync(
+            container.RootConnectionString,
+            container.TemplateDbName,
+            newDbName: newDb,
+            seed: VoiceWorkIngestionSeedData.SeedAsync);
     }
 
     protected static async Task<VoiceWorkUpsertResult[]> UpsertAsync(AppDbContext dbContext, DateTime dateTime, VoiceWorkIngest[] ingests)

JSMR.Tests/JSMR.Tests.csproj

@@ -1,49 +1,49 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <PropertyGroup>
+	<PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+		<TargetFramework>net9.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
+		<ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
+		<Nullable>enable</Nullable>
-    <IsPackable>false</IsPackable>
+		<IsPackable>false</IsPackable>
-  </PropertyGroup>
+	</PropertyGroup>
 
-  <ItemGroup>
+	<ItemGroup>
-    <None Remove="Integrations\DLSite\Product-Info.json" />
+		<None Remove="Integrations\DLSite\Product-Info.json" />
-    <None Remove="Scanning\English-Page.html" />
+		<None Remove="Scanning\English-Page.html" />
-  </ItemGroup>
+	</ItemGroup>
 
-  <ItemGroup>
+	<ItemGroup>
-    <EmbeddedResource Include="Integrations\DLSite\Product-Info.json" />
+		<EmbeddedResource Include="Integrations\DLSite\Product-Info.json" />
-    <EmbeddedResource Include="Scanning\English-Page-Updated.html" />
+		<EmbeddedResource Include="Scanning\English-Page-Updated.html" />
-    <EmbeddedResource Include="Scanning\Japanese-Page.html" />
+		<EmbeddedResource Include="Scanning\Japanese-Page.html" />
-    <EmbeddedResource Include="Scanning\English-Page.html" />
+		<EmbeddedResource Include="Scanning\English-Page.html" />
-  </ItemGroup>
+	</ItemGroup>
 
-  <ItemGroup>
+	<ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="6.0.4">
+		<PackageReference Include="coverlet.collector" Version="6.0.4">
-      <PrivateAssets>all</PrivateAssets>
+			<PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
+		</PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
-    <PackageReference Include="NSubstitute" Version="5.3.0" />
+		<PackageReference Include="NSubstitute" Version="5.3.0" />
-    <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0" />
+		<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0" />
-    <PackageReference Include="Shouldly" Version="4.3.0" />
+		<PackageReference Include="Shouldly" Version="4.3.0" />
-    <PackageReference Include="Testcontainers" Version="4.8.1" />
+		<PackageReference Include="Testcontainers" Version="4.8.1" />
-    <PackageReference Include="Testcontainers.MariaDb" Version="4.8.1" />
+		<PackageReference Include="Testcontainers.MariaDb" Version="4.8.1" />
-    <PackageReference Include="Testcontainers.XunitV3" Version="4.8.1" />
+		<PackageReference Include="Testcontainers.XunitV3" Version="4.8.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
+		<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
-      <PrivateAssets>all</PrivateAssets>
+			<PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
+		</PackageReference>
-    <PackageReference Include="xunit.v3" Version="3.1.0" />
+		<PackageReference Include="xunit.v3" Version="3.1.0" />
-  </ItemGroup>
+	</ItemGroup>
 
-  <ItemGroup>
+	<ItemGroup>
-    <ProjectReference Include="..\JSMR.Infrastructure\JSMR.Infrastructure.csproj" />
+		<ProjectReference Include="..\JSMR.Infrastructure\JSMR.Infrastructure.csproj" />
-  </ItemGroup>
+	</ItemGroup>
 
-  <ItemGroup>
+	<ItemGroup>
-    <Using Include="Xunit" />
+		<Using Include="Xunit" />
-  </ItemGroup>
+	</ItemGroup>
 
 </Project>