From abd5a81e3e19fe263a9b41e5746310d685260454 Mon Sep 17 00:00:00 2001 From: Brian Bicknell Date: Sun, 2 Nov 2025 13:56:34 -0500 Subject: [PATCH] Improved performance of integration tests. --- .gitea/workflows/ci.yml | 4 +- .../Fixtures/CircleSearchProviderFixture.cs | 11 +- .../Fixtures/CreatorSearchProviderFixture.cs | 10 +- JSMR.Tests/Fixtures/MariaDbClone.cs | 126 ++++++++++++++++++ .../Fixtures/MariaDbContainerFixture.cs | 42 ++++++ JSMR.Tests/Fixtures/MariaDbFixture.cs | 83 +----------- .../Fixtures/TagSearchProviderFixture.cs | 10 +- .../VoiceWorkSearchProviderFixture.cs | 11 +- .../Ingestion/Japanese/IngestionTestsBase.cs | 15 ++- JSMR.Tests/JSMR.Tests.csproj | 80 +++++------ 10 files changed, 262 insertions(+), 130 deletions(-) create mode 100644 JSMR.Tests/Fixtures/MariaDbClone.cs create mode 100644 JSMR.Tests/Fixtures/MariaDbContainerFixture.cs diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1cc13d4..f0c1c96 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -7,6 +7,8 @@ on: jobs: build-test: runs-on: self-hosted + env: + CI: "true" container: image: ghcr.io/catthehacker/ubuntu:act-latest steps: @@ -22,5 +24,5 @@ jobs: nuget-${{ runner.os }}- - run: dotnet restore - run: dotnet build --configuration Release --no-restore - - run: dotnet test --configuration Release --no-build --logger "trx;LogFileName=test-results.trx" + - run: dotnet test --configuration Release --no-build -p:CI=true --logger "trx;LogFileName=test-results.trx" diff --git a/JSMR.Tests/Fixtures/CircleSearchProviderFixture.cs b/JSMR.Tests/Fixtures/CircleSearchProviderFixture.cs index a7c192f..6aac77c 100644 --- a/JSMR.Tests/Fixtures/CircleSearchProviderFixture.cs +++ b/JSMR.Tests/Fixtures/CircleSearchProviderFixture.cs @@ -1,5 +1,6 @@ using JSMR.Infrastructure.Data; using Microsoft.EntityFrameworkCore; +using Org.BouncyCastle.Asn1.Pkcs; namespace JSMR.Tests.Fixtures; @@ -33,8 +34,16 @@ public sealed class CircleSearchProviderFixture2(MariaDbContainerFixture contain public async ValueTask InitializeAsync() { - DbContext = await MariaTestDb.CreateIsolatedAsync( + //DbContext = await MariaTestDb.CreateIsolatedAsync( + // container.RootConnectionString, + // seed: SeedAsync); + + var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + + DbContext = await MariaDbClone.CloneFromTemplateAsync( container.RootConnectionString, + container.TemplateDbName, + newDbName: newDb, seed: SeedAsync); } diff --git a/JSMR.Tests/Fixtures/CreatorSearchProviderFixture.cs b/JSMR.Tests/Fixtures/CreatorSearchProviderFixture.cs index 9e9e7db..af79f0b 100644 --- a/JSMR.Tests/Fixtures/CreatorSearchProviderFixture.cs +++ b/JSMR.Tests/Fixtures/CreatorSearchProviderFixture.cs @@ -32,8 +32,16 @@ public sealed class CreatorSearchProviderFixture2(MariaDbContainerFixture contai public async ValueTask InitializeAsync() { - DbContext = await MariaTestDb.CreateIsolatedAsync( + //DbContext = await MariaTestDb.CreateIsolatedAsync( + // container.RootConnectionString, + // seed: SeedAsync); + + var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + + DbContext = await MariaDbClone.CloneFromTemplateAsync( container.RootConnectionString, + container.TemplateDbName, + newDbName: newDb, seed: SeedAsync); } diff --git a/JSMR.Tests/Fixtures/MariaDbClone.cs b/JSMR.Tests/Fixtures/MariaDbClone.cs new file mode 100644 index 0000000..e4fb656 --- /dev/null +++ b/JSMR.Tests/Fixtures/MariaDbClone.cs @@ -0,0 +1,126 @@ +using JSMR.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using MySqlConnector; +using System.Data; + +namespace JSMR.Tests.Fixtures; + +public static class MariaDbClone +{ + public static async Task CreateTemplateAsync( + string rootConn, + string templateDbName, + Func? seedAsync = null) + { + await using var root = new MySqlConnection(rootConn); + await root.OpenAsync(); + + await ExecAsync(root, $"DROP DATABASE IF EXISTS `{templateDbName}`;"); + await ExecAsync(root, $"CREATE DATABASE `{templateDbName}` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"); + + // Run EF once to build schema and seed + var templateConn = new MySqlConnectionStringBuilder(rootConn) { Database = templateDbName }.ConnectionString; + await using var ctx = AppDb(templateConn); + await ctx.Database.EnsureCreatedAsync(); + + if (seedAsync != null) + await seedAsync(ctx); + } + + public static async Task CloneFromTemplateAsync( + string rootConn, + string templateDbName, + string newDbName, + Func? seed = null) + { + var newConnStr = new MySqlConnectionStringBuilder(rootConn) { Database = newDbName }.ConnectionString; + + await using var root = new MySqlConnection(rootConn); + await root.OpenAsync(); + + // Create target DB + await ExecAsync(root, $"CREATE DATABASE `{newDbName}` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"); + + // Disable FK checks while recreating schema & loading data + await ExecAsync(root, $"SET SESSION sql_log_bin = 0;"); // avoid binlog noise (optional) + await ExecAsync(root, $"SET SESSION foreign_key_checks = 0;"); + + // 1) Create tables using exact DDL from templatedb + var tables = await GetTableNamesAsync(root, templateDbName); + + foreach (var table in tables) + { + var createTable = await ShowCreateTableAsync(root, templateDbName, table); + // Run DDL in the new DB (the CREATE statement itself doesn't include db name) + await ExecAsync(root, $"USE `{newDbName}`; {createTable};"); + } + + // 2) Copy data + foreach (var table in tables) + { + var sql = $"INSERT INTO `{newDbName}`.`{table}` SELECT * FROM `{templateDbName}`.`{table}`;"; + await ExecAsync(root, sql); + } + + await ExecAsync(root, $"SET SESSION foreign_key_checks = 1;"); + await ExecAsync(root, $"SET SESSION sql_log_bin = 1;"); + + // Ready-to-use EF context for the cloned DB + //return AppDb(newConnStr); + + AppDbContext dbContext = AppDb(newConnStr); + + if (seed != null) + await seed(dbContext); + + return dbContext; + } + + private static AppDbContext AppDb(string connStr) + { + var opts = new DbContextOptionsBuilder() + .UseMySql(connStr, ServerVersion.AutoDetect(connStr), o => o.EnableRetryOnFailure()) + .EnableSensitiveDataLogging() + .Options; + + return new AppDbContext(opts); + } + + private static async Task> GetTableNamesAsync(MySqlConnection conn, string db) + { + const string sql = @" +SELECT TABLE_NAME +FROM information_schema.TABLES +WHERE TABLE_SCHEMA = @db AND TABLE_TYPE = 'BASE TABLE';"; + + var result = new List(); + await using var cmd = new MySqlCommand(sql, conn) { Parameters = { new("@db", db) } }; + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + result.Add(rdr.GetString(0)); + return result; + } + + private static async Task ShowCreateTableAsync(MySqlConnection conn, string db, string table) + { + await using var cmd = new MySqlCommand($"SHOW CREATE TABLE `{db}`.`{table}`;", conn); + await using var rdr = await cmd.ExecuteReaderAsync(CommandBehavior.SingleRow); + if (!await rdr.ReadAsync()) + throw new InvalidOperationException($"SHOW CREATE TABLE returned no rows for {db}.{table}"); + + // Column 1: table name, Column 2: CREATE TABLE DDL + var ddl = rdr.GetString(1); + + // Ensure the statement ends with ';' + if (!ddl.TrimEnd().EndsWith(";")) + ddl += ";"; + + return ddl; + } + + private static async Task ExecAsync(MySqlConnection conn, string sql) + { + await using var cmd = new MySqlCommand(sql, conn) { CommandTimeout = 0 }; + await cmd.ExecuteNonQueryAsync(); + } +} \ No newline at end of file diff --git a/JSMR.Tests/Fixtures/MariaDbContainerFixture.cs b/JSMR.Tests/Fixtures/MariaDbContainerFixture.cs new file mode 100644 index 0000000..26d610d --- /dev/null +++ b/JSMR.Tests/Fixtures/MariaDbContainerFixture.cs @@ -0,0 +1,42 @@ +using DotNet.Testcontainers.Builders; +using JSMR.Tests.Fixtures; +using JSMR.Tests.Ingestion; +using Testcontainers.MariaDb; + +[assembly: AssemblyFixture(typeof(MariaDbContainerFixture))] + +namespace JSMR.Tests.Fixtures; + +public sealed class MariaDbContainerFixture : IAsyncLifetime +{ + const int MajorVersion = 10; + const int MinorVersion = 11; + const int Build = 6; + + private MariaDbContainer _container = default!; + + public string RootConnectionString { get; private set; } = default!; + public string TemplateDbName { get; } = "jsmr_template"; + + public async ValueTask InitializeAsync() + { + _container = new MariaDbBuilder() + .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}") + .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw") + .WithUsername("root") + .WithPassword("rootpw") + .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306)) + .Build(); + + await _container.StartAsync(); + + RootConnectionString = _container.GetConnectionString(); + + // Build the template ONCE with EF + your existing seed + await MariaDbClone.CreateTemplateAsync( + RootConnectionString, + TemplateDbName); + } + + public async ValueTask DisposeAsync() => await _container.DisposeAsync(); +} \ No newline at end of file diff --git a/JSMR.Tests/Fixtures/MariaDbFixture.cs b/JSMR.Tests/Fixtures/MariaDbFixture.cs index f518afd..f8c1c64 100644 --- a/JSMR.Tests/Fixtures/MariaDbFixture.cs +++ b/JSMR.Tests/Fixtures/MariaDbFixture.cs @@ -1,66 +1,10 @@ -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; -using JSMR.Infrastructure.Data; -using JSMR.Tests.Fixtures; +using JSMR.Infrastructure.Data; using Microsoft.EntityFrameworkCore; using MySqlConnector; using Testcontainers.MariaDb; -using Testcontainers.Xunit; -using Xunit.Sdk; - -[assembly: AssemblyFixture(typeof(MariaDbContainerFixture))] namespace JSMR.Tests.Fixtures; - -public sealed class MariaDbContainerFixture : IAsyncLifetime -{ - const int MajorVersion = 10; - const int MinorVersion = 11; - const int Build = 6; - - private MariaDbContainer _container = default!; - public string RootConnectionString { get; private set; } = default!; - - public async ValueTask InitializeAsync() - { - //_container = new ContainerBuilder() - // .WithImage("mariadb:11") - // .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw") - // .WithPortBinding(3307, 3306) - // .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(3306)) - // .Build(); - - //_container = new ContainerBuilder() - // .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}") - // .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw") - // //.WithPortBinding(3307, 3306) - // .WithPortBinding(3306, assignRandomHostPort: true) - // .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306)) - // .Build(); - - _container = new MariaDbBuilder() - .WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}") - .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw") - .WithUsername("root") - .WithPassword("rootpw") - // no explicit port binding - .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306)) - .Build(); - - await _container.StartAsync(); - - // No database specified: we’ll create per-test DBs - //RootConnectionString = "Server=127.0.0.1;Port=3307;User=root;Password=rootpw;SslMode=none;"; - - RootConnectionString = _container.GetConnectionString(); - var port = _container.GetMappedPublicPort(3306); - //RootConnectionString = $"Server=127.0.0.1;Port={port};User=root;Password=rootpw;SslMode=none;"; - } - - public async ValueTask DisposeAsync() => await _container.DisposeAsync(); -} - public class MariaDbFixture : IAsyncLifetime { const int MajorVersion = 10; @@ -125,31 +69,6 @@ public class MariaDbFixture : IAsyncLifetime } } -[CollectionDefinition("db")] -public sealed class MariaDbCollection : ICollectionFixture { } - -//public class MariaDbAssemblyFixtureDefinition : IAssemblyFixture { } - - -//[UsedImplicitly] -public sealed class MariaDbContainerFixture2(IMessageSink messageSink) - : ContainerFixture(messageSink) -{ - const int MajorVersion = 10; - const int MinorVersion = 11; - const int Build = 6; - - public string RootConnectionString => $"Server={Container.IpAddress};Port=3306;User=root;Password=rootpw;SslMode=none;"; - - protected override MariaDbBuilder Configure(MariaDbBuilder builder) - { - return builder.WithImage($"mariadb:{MajorVersion}.{MinorVersion}.{Build}") - .WithEnvironment("MARIADB_ROOT_PASSWORD", "rootpw") - .WithPortBinding(3307, 3306) - //.WithPortBinding(3306, assignRandomHostPort: true) - .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(3306)); - } -} public static class MariaTestDb { diff --git a/JSMR.Tests/Fixtures/TagSearchProviderFixture.cs b/JSMR.Tests/Fixtures/TagSearchProviderFixture.cs index 7d8eed9..df2f02c 100644 --- a/JSMR.Tests/Fixtures/TagSearchProviderFixture.cs +++ b/JSMR.Tests/Fixtures/TagSearchProviderFixture.cs @@ -40,8 +40,16 @@ public sealed class TagSearchProviderFixture2(MariaDbContainerFixture container) public async ValueTask InitializeAsync() { - DbContext = await MariaTestDb.CreateIsolatedAsync( + //DbContext = await MariaTestDb.CreateIsolatedAsync( + // container.RootConnectionString, + // seed: SeedAsync); + + var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + + DbContext = await MariaDbClone.CloneFromTemplateAsync( container.RootConnectionString, + container.TemplateDbName, + newDbName: newDb, seed: SeedAsync); } diff --git a/JSMR.Tests/Fixtures/VoiceWorkSearchProviderFixture.cs b/JSMR.Tests/Fixtures/VoiceWorkSearchProviderFixture.cs index c8018fa..70ebff3 100644 --- a/JSMR.Tests/Fixtures/VoiceWorkSearchProviderFixture.cs +++ b/JSMR.Tests/Fixtures/VoiceWorkSearchProviderFixture.cs @@ -1,5 +1,6 @@ using JSMR.Domain.Enums; using JSMR.Infrastructure.Data; +using JSMR.Tests.Ingestion; using Microsoft.EntityFrameworkCore; namespace JSMR.Tests.Fixtures; @@ -132,8 +133,16 @@ public sealed class VoiceWorkSearchProviderFixture2(MariaDbContainerFixture cont public async ValueTask InitializeAsync() { - DbContext = await MariaTestDb.CreateIsolatedAsync( + //DbContext = await MariaTestDb.CreateIsolatedAsync( + // container.RootConnectionString, + // seed: SeedAsync); + + var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + + DbContext = await MariaDbClone.CloneFromTemplateAsync( container.RootConnectionString, + container.TemplateDbName, + newDbName: newDb, seed: SeedAsync); } diff --git a/JSMR.Tests/Ingestion/Japanese/IngestionTestsBase.cs b/JSMR.Tests/Ingestion/Japanese/IngestionTestsBase.cs index bb412fa..c7e029b 100644 --- a/JSMR.Tests/Ingestion/Japanese/IngestionTestsBase.cs +++ b/JSMR.Tests/Ingestion/Japanese/IngestionTestsBase.cs @@ -4,6 +4,7 @@ using JSMR.Infrastructure.Common.Time; using JSMR.Infrastructure.Data; using JSMR.Infrastructure.Ingestion; using JSMR.Tests.Fixtures; +using Microsoft.EntityFrameworkCore; using NSubstitute; using System.Runtime.InteropServices; @@ -13,9 +14,17 @@ public abstract class IngestionTestsBase(MariaDbContainerFixture container) { protected async Task GetAppDbContextAsync() { - return await MariaTestDb.CreateIsolatedAsync( - container.RootConnectionString, - seed: VoiceWorkIngestionSeedData.SeedAsync); + //return await MariaTestDb.CreateIsolatedAsync( + // container.RootConnectionString, + // seed: VoiceWorkIngestionSeedData.SeedAsync); + + var newDb = $"t_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + + return await MariaDbClone.CloneFromTemplateAsync( + container.RootConnectionString, + container.TemplateDbName, + newDbName: newDb, + seed: VoiceWorkIngestionSeedData.SeedAsync); } protected static async Task UpsertAsync(AppDbContext dbContext, DateTime dateTime, VoiceWorkIngest[] ingests) diff --git a/JSMR.Tests/JSMR.Tests.csproj b/JSMR.Tests/JSMR.Tests.csproj index 92ac6d9..e2c5cd9 100644 --- a/JSMR.Tests/JSMR.Tests.csproj +++ b/JSMR.Tests/JSMR.Tests.csproj @@ -1,49 +1,49 @@  - - net9.0 - enable - enable - false - + + net9.0 + enable + enable + false + - - - - + + + + - - - - - - + + + + + + - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + - - - + + + - - - + + +