Added logging to scanner.

JSMR.Application/Scanning/ScanVoiceWorksHandler.cs

@@ -1,10 +1,13 @@
 using JSMR.Application.Common.Caching;
 using JSMR.Application.Scanning.Contracts;
 using JSMR.Application.Scanning.Ports;
+using Microsoft.Extensions.Logging;
+using System.Diagnostics;
 
 namespace JSMR.Application.Scanning;
 
 public sealed class ScanVoiceWorksHandler(
+    ILogger<ScanVoiceWorksHandler> logger,
     IVoiceWorkScannerRepository scannerRepository,
     IVoiceWorkUpdaterRepository updaterRepository,
     ISpamCircleCache spamCircleCache,
@@ -13,43 +16,148 @@ public sealed class ScanVoiceWorksHandler(
 {
     public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
     {
-        IVoiceWorksScanner? scanner = scannerRepository.GetScanner(request.Locale);
-        IVoiceWorkUpdater? updater = updaterRepository.GetUpdater(request.Locale);
-
-        if (scanner is null)
-            throw new InvalidOperationException($"No scanner registered for locale {request.Locale}.");
-
-        if (updater is null)
-            throw new InvalidOperationException($"No updater registered for locale {request.Locale}.");
-
-        VoiceWorkScanOptions options = new(
-            PageNumber: request.PageNumber,
-            PageSize: request.PageSize,
-            ExcludedMakerIds: await spamCircleCache.GetAsync(cancellationToken),
-            ExcludePartiallyAIGeneratedWorks: true,
-            ExcludeAIGeneratedWorks: true
-        );
-
-        VoiceWorkScanResult scanResult = await scanner.ScanPageAsync(options, cancellationToken);
-
-        if (scanResult.EndOfResults)
+        using IDisposable? scope = logger.BeginScope(new Dictionary<string, object>
         {
+            ["Locale"] = request.Locale,
+            ["PageNumber"] = request.PageNumber,
+            ["PageSize"] = request.PageSize
+        });
+
+        Stopwatch stopwatch = Stopwatch.StartNew();
+        string currentPhase = "initialization";
+
+        try
+        {
+            logger.LogInformation(
+                "Starting scan handler for page {PageNumber}, page size {PageSize}, locale {Locale}",
+                request.PageNumber,
+                request.PageSize,
+                request.Locale);
+
+            currentPhase = "resolve_scanner";
+            IVoiceWorksScanner? scanner = scannerRepository.GetScanner(request.Locale);
+
+            currentPhase = "resolve_updater";
+            IVoiceWorkUpdater? updater = updaterRepository.GetUpdater(request.Locale);
+
+            if (scanner is null)
+                throw new InvalidOperationException($"No scanner registered for locale {request.Locale}.");
+
+            if (updater is null)
+                throw new InvalidOperationException($"No updater registered for locale {request.Locale}.");
+
+            currentPhase = "load_spam_circle_cache";
+            var spamStopwatch = Stopwatch.StartNew();
+
+            string[] excludedMakerIds = await spamCircleCache.GetAsync(cancellationToken);
+
+            spamStopwatch.Stop();
+            logger.LogInformation(
+                "Loaded spam circle cache in {ElapsedMs} ms. ExcludedMakerCount={ExcludedMakerCount}",
+                spamStopwatch.ElapsedMilliseconds,
+                excludedMakerIds.Length);
+
+            VoiceWorkScanOptions options = new(
+                PageNumber: request.PageNumber,
+                PageSize: request.PageSize,
+                ExcludedMakerIds: excludedMakerIds,
+                ExcludePartiallyAIGeneratedWorks: true,
+                ExcludeAIGeneratedWorks: true
+            );
+
+            currentPhase = "scan_page";
+            var scanStopwatch = Stopwatch.StartNew();
+
+            VoiceWorkScanResult scanResult = await scanner.ScanPageAsync(options, cancellationToken);
+
+            scanStopwatch.Stop();
+
+            logger.LogInformation(
+                "Scanned source page in {ElapsedMs} ms. EndOfResults={EndOfResults}, ResultCount={ResultCount}",
+                scanStopwatch.ElapsedMilliseconds,
+                scanResult.EndOfResults,
+                scanResult.Works.Length);
+
+
+            if (scanResult.EndOfResults)
+            {
+                stopwatch.Stop();
+
+                logger.LogInformation(
+                    "End of results reached for page {PageNumber}. TotalElapsedMs={ElapsedMs}",
+                    request.PageNumber,
+                    stopwatch.ElapsedMilliseconds);
+
+                return new ScanVoiceWorksResponse(
+                    Results: [],
+                    EndOfResults: true
+                );
+            }
+
+            currentPhase = "build_ingests";
+            var ingestSw = Stopwatch.StartNew();
+
+            VoiceWorkIngest[] ingests = await ingestBuilder.BuildAsync(scanResult, cancellationToken);
+
+            ingestSw.Stop();
+            logger.LogInformation(
+                "Built ingests in {ElapsedMs} ms. IngestCount={IngestCount}",
+                ingestSw.ElapsedMilliseconds,
+                ingests.Length);
+
+            currentPhase = "upsert_voiceworks";
+            var upsertSw = Stopwatch.StartNew();
+
+            VoiceWorkUpsertResult[] upsertResults = await updater.UpsertAsync(ingests, cancellationToken);
+
+            upsertSw.Stop();
+
+            int[] voiceWorkIds = [.. upsertResults.Where(x => x.VoiceWorkId.HasValue).Select(x => x.VoiceWorkId!.Value)];
+
+            int issueCount = upsertResults.Sum(x => x.Issues.Count);
+
+            logger.LogInformation(
+                "Upserted voice works in {ElapsedMs} ms. UpsertResultCount={UpsertResultCount}, VoiceWorkIdCount={VoiceWorkIdCount}, IssueCount={IssueCount}",
+                upsertSw.ElapsedMilliseconds,
+                upsertResults.Length,
+                voiceWorkIds.Length,
+                issueCount);
+
+            currentPhase = "update_search";
+            var searchSw = Stopwatch.StartNew();
+
+            await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
+
+            searchSw.Stop();
+            logger.LogInformation(
+                "Updated search index in {ElapsedMs} ms. VoiceWorkIdCount={VoiceWorkIdCount}",
+                searchSw.ElapsedMilliseconds,
+                voiceWorkIds.Length);
+
+            stopwatch.Stop();
+
+            logger.LogInformation(
+                "Completed scan handler for page {PageNumber} in {ElapsedMs} ms",
+                request.PageNumber,
+                stopwatch.ElapsedMilliseconds);
+
             return new ScanVoiceWorksResponse(
-                Results: [],
-                EndOfResults: true
+                Results: upsertResults,
+                EndOfResults: false
             );
         }
+        catch (Exception ex)
+        {
+            stopwatch.Stop();
 
-        VoiceWorkIngest[] ingests = await ingestBuilder.BuildAsync(scanResult, cancellationToken);
+            logger.LogError(
+                ex,
+                "Scan handler failed during phase {Phase} for page {PageNumber} after {ElapsedMs} ms",
+                currentPhase,
+                request.PageNumber,
+                stopwatch.ElapsedMilliseconds);
 
-        VoiceWorkUpsertResult[] upsertResults = await updater.UpsertAsync(ingests, cancellationToken);
-        int[] voiceWorkIds = [.. upsertResults.Where(x => x.VoiceWorkId.HasValue).Select(x => x.VoiceWorkId!.Value)];
-
-        await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
-
-        return new ScanVoiceWorksResponse(
-            Results: upsertResults,
-            EndOfResults: false
-        );
+            throw;
+        }
     }
 }

JSMR.UI.Blazor/Components/JPagination2.razor

@@ -0,0 +1,113 @@
+@using JSMR.UI.Blazor.Enums
+
+<div class="pagination">
+    <div>
+        <label>@IndexInfo</label>
+    </div>
+    @* <MudPagination class="pager" ShowFirstButton="true" ShowLastButton="true" Count="@((int)Math.Ceiling((decimal)TotalItems / (decimal)PageSize))" Selected="@PageNumber" SelectedChanged="OnSelectedChanged" /> *@
+    <AntDesign.Pagination ShowSizeChanger="false" Total="TotalItems" Current="PageNumber" PageSize="PageSize" OnChange="OnPaginationChanged"></AntDesign.Pagination>
+    <div class="page-sizes">
+        @RightContent
+        <div class="page-size-select">
+            <AntDesign.Select TItem="PageSizeItem"
+                              TItemValue="int"
+                              DataSource="PageSizes3"
+                              LabelName="Text"
+                              ValueName="Value"
+                              Value="PageSize"
+                              ValueChanged="OnPageSizeChanged">
+            </AntDesign.Select>
+        </div>
+        
+        @* <BitDropdown Items="PageSizes2"
+                     Style="min-width: 7rem"
+                     Placeholder="Select..."
+                     TItem="BitDropdownItem<int>"
+                     TValue="int"
+                     Value="PageSize"
+                     ValueChanged="OnPageSizeChanged">
+            <PrefixTemplate>
+                <InputPrefix Graphic="Graphic.Grid" Tooltip="Page Size"></InputPrefix>
+            </PrefixTemplate>
+        </BitDropdown> *@
+    </div>
+</div>
+
+<style>
+    .pagination {
+        padding: 16px 16px;
+        background: transparent;
+        background: #141414;
+    }
+
+    .ant-pagination {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+    }
+
+    .page-size-select {
+        width: 12em;
+    }
+</style>
+
+@code {
+    [Parameter]
+    public int PageNumber { get; set; }
+
+    [Parameter]
+    public EventCallback<int> PageNumberChanged { get; set; }
+
+    [Parameter]
+    public int[] PageSizes { get; set; } = [5, 10, 25, 50, 100];
+
+    List<BitDropdownItem<int>> PageSizes2 => [.. PageSizes.Select(x => new BitDropdownItem<int>() { Text = x.ToString(), Value = x })];
+
+    List<PageSizeItem> PageSizes3 => [.. PageSizes.Select(x => new PageSizeItem() { Text = $"{x} items / page", Value = x })];
+
+    [Parameter]
+    public int PageSize { get; set; }
+
+    [Parameter]
+    public EventCallback<int> PageSizeChanged { get; set; }
+
+    [Parameter]
+    public int TotalItems { get; set; }
+
+    [Parameter]
+    public EventCallback<int> TotalItemsChanged { get; set; }
+
+    [Parameter]
+    public RenderFragment? RightContent { get; set; }
+
+    public string IndexInfo => TotalItems == 0 ? "No items" : $"{StartIndex.ToString("n0")} - {EndIndex.ToString("n0")} of {TotalItems.ToString("n0")} items";
+
+    public int StartIndex => (PageNumber - 1) * PageSize + 1;
+    public int EndIndex => PageNumber * PageSize < TotalItems ? PageNumber * PageSize : TotalItems;
+
+    private async Task OnSelectedChanged(int newPage)
+    {
+        PageNumber = newPage;
+        await PageNumberChanged.InvokeAsync(newPage);
+    }
+
+    private async Task OnPageSizeChanged(int newPageSize)
+    {
+        PageSize = newPageSize;
+        await PageSizeChanged.InvokeAsync(newPageSize);
+    }
+
+    private async Task OnPaginationChanged(AntDesign.PaginationEventArgs args)
+    {
+        int newPage = args.Page;
+
+        PageNumber = newPage;
+        await PageNumberChanged.InvokeAsync(newPage);
+    }
+
+    public class PageSizeItem
+    {
+        public string? Text { get; set; }
+        public int Value { get; set; }
+    }
+}

JSMR.UI.Blazor/Pages/Creators.razor

@@ -66,11 +66,10 @@
         </AntDesign.ActionColumn> *@
             </ColumnDefinitions>
         </AntDesign.Table>
+        <JPagination2 PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
     </Body>
 </AntDesign.Card>
 
-<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
-
 <style>
     .mud-table-root {
         table-layout: fixed;
@@ -87,6 +86,15 @@
         backdrop-filter: blur(12px);
     }
 
+    .ant-card-extra {
+        position: absolute;
+        left: 0;
+        right: 0;
+        margin-left: auto;
+        margin-right: auto;
+        width: 50%;
+    }
+
     .ant-table {
         /* background: inherit; */
     }

JSMR.UI.Blazor/Pages/Tags.razor

@@ -13,67 +13,54 @@
 <PageTitle>Tags</PageTitle>
 
 <div class="fdsfds">
-@* <h1>Tags</h1> *@
+    @* <h1>Tags</h1> *@
 
-@* <MudTextField T="string" Value="Keywords" ValueChanged="OnKeywordsChanged" Immediate="true" DebounceInterval="500" Label="Filter" Variant="MudBlazor.Variant.Text" Adornment="@Adornment.Start" AdornmentIcon="@Icons.Material.Outlined.Search" />
+    @* <MudTextField T="string" Value="Keywords" ValueChanged="OnKeywordsChanged" Immediate="true" DebounceInterval="500" Label="Filter" Variant="MudBlazor.Variant.Text" Adornment="@Adornment.Start" AdornmentIcon="@Icons.Material.Outlined.Search" />
  *@
     <AntDesign.Card Title=@("Tags") Class="ant-blurred-card">
-    <Extra>
-        <AntDesign.Input TValue="string" Value="Keywords" ValueChanged="OnKeywordsChanged" DebounceMilliseconds="500" Placeholder="Filter">
-            <Prefix>
-                <AntDesign.Icon Type="@AntDesign.IconType.Outline.Search" />
-            </Prefix>
-        </AntDesign.Input>
-    </Extra>
-    <Body>
-        <Space Direction="SpaceDirection.Vertical" Size="@($"2em")">
-            @* <SpaceItem>
-                <AntDesign.Input TValue="string" Value="Keywords" ValueChanged="OnKeywordsChanged" DebounceMilliseconds="500" Placeholder="Filter">
-                    <Prefix>
-                        <AntDesign.Icon Type="@AntDesign.IconType.Outline.Search" />
-                    </Prefix>
-                </AntDesign.Input>
-            </SpaceItem> *@
-            <SpaceItem>
-                <AntDesign.Table Responsive
-                                 DataSource="@(searchResults?.Items ?? Enumerable.Empty<TagSearchItem>())"
-                                 Total="@(searchResults?.TotalItems ?? 0)"
-                                 TItem="TagSearchItem"
-                                 Loading="LoadingData"
-                                 HidePagination="@false"
-                                 PageSize="PageSize"
-                                 PageIndex="PageNumber"
-                                 PaginationPosition="bottomCenter"
-                                 RemoteDataSource="@true"
-                                 RowKey="x=>x.TagId"
-                                 OnChange="HandleTableChange">
-                    <ColumnDefinitions>
-                        <AntDesign.PropertyColumn Property="c => c.Name" Title="Name" Sortable SorterMultiple="5" />
-                        <AntDesign.PropertyColumn Property="c => c.EnglishName" Title="English Name" Sortable SorterMultiple="5" />
-                        <AntDesign.PropertyColumn Property="c => c.VoiceWorkCount" Title="Voice Works" Format="n0" HeaderStyle="width: 10em" Sortable SorterMultiple="5">
-                            <AntDesign.Button Size="AntDesign.ButtonSize.Small" Type="AntDesign.ButtonType.Link" OnClick="@(e => NavigateToVoiceWorkSearch(context))" Style="display:flex;justify-self:flex-end;">@context.VoiceWorkCount.ToString("n0")</AntDesign.Button>
-                        </AntDesign.PropertyColumn>
-                        <AntDesign.PropertyColumn Property="c => c.Favorite" Title="Favorite" HeaderStyle="width: 8em" Sortable SortDirections="new[] { AntDesign.SortDirection.Descending }" SorterMultiple="5">
-                            @if (context.Favorite)
-                            {
-                                <AntDesign.Tag Color="AntDesign.TagColor.PurpleInverse">Favorite</AntDesign.Tag>
-                            }
-                        </AntDesign.PropertyColumn>
-                        <AntDesign.PropertyColumn Property="c => c.Blacklisted" Title="Blacklisted" HeaderStyle="width: 9em" Sortable SortDirections="new[] { AntDesign.SortDirection.Descending }" SorterMultiple="5">
-                            @if (context.Blacklisted)
-                            {
-                                <AntDesign.Tag Color="AntDesign.TagColor.RedInverse">Blacklisted</AntDesign.Tag>
-                            }
-                        </AntDesign.PropertyColumn>
-                    </ColumnDefinitions>
-                </AntDesign.Table>
-            </SpaceItem>
-        </Space>
-    </Body>
- </AntDesign.Card>
+        <Extra>
+            <AntDesign.Input TValue="string" Value="Keywords" ValueChanged="OnKeywordsChanged" DebounceMilliseconds="500" Placeholder="Filter">
+                <Prefix>
+                    <AntDesign.Icon Type="@AntDesign.IconType.Outline.Search" />
+                </Prefix>
+            </AntDesign.Input>
+        </Extra>
+        <Body>
+            <AntDesign.Table Responsive
+                             DataSource="@(searchResults?.Items ?? Enumerable.Empty<TagSearchItem>())"
+                             Total="@(searchResults?.TotalItems ?? 0)"
+                             TItem="TagSearchItem"
+                             Loading="LoadingData"
+                             HidePagination="@true"
+                             RemoteDataSource="@true"
+                             RowKey="x=>x.TagId"
+                             OnChange="HandleTableChange">
+                <ColumnDefinitions>
+                    <AntDesign.PropertyColumn Property="c => c.Name" Title="Name" Sortable SorterMultiple="5" />
+                    <AntDesign.PropertyColumn Property="c => c.EnglishName" Title="English Name" Sortable SorterMultiple="5" />
+                    <AntDesign.PropertyColumn Property="c => c.VoiceWorkCount" Title="Voice Works" Format="n0" HeaderStyle="width: 10em" Sortable SorterMultiple="5">
+                        <AntDesign.Button Size="AntDesign.ButtonSize.Small" Type="AntDesign.ButtonType.Link" OnClick="@(e => NavigateToVoiceWorkSearch(context))" Style="display:flex;justify-self:flex-end;">@context.VoiceWorkCount.ToString("n0")</AntDesign.Button>
+                    </AntDesign.PropertyColumn>
+                    <AntDesign.PropertyColumn Property="c => c.Favorite" Title="Favorite" HeaderStyle="width: 8em" Sortable SortDirections="new[] { AntDesign.SortDirection.Descending }" SorterMultiple="5">
+                        @if (context.Favorite)
+                        {
+                            <AntDesign.Tag Color="AntDesign.TagColor.PurpleInverse">Favorite</AntDesign.Tag>
+                        }
+                    </AntDesign.PropertyColumn>
+                    <AntDesign.PropertyColumn Property="c => c.Blacklisted" Title="Blacklisted" HeaderStyle="width: 9em" Sortable SortDirections="new[] { AntDesign.SortDirection.Descending }" SorterMultiple="5">
+                        @if (context.Blacklisted)
+                        {
+                            <AntDesign.Tag Color="AntDesign.TagColor.RedInverse">Blacklisted</AntDesign.Tag>
+                        }
+                    </AntDesign.PropertyColumn>
+                </ColumnDefinitions>
+            </AntDesign.Table>
+            <JPagination2 PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
+        </Body>
+    </AntDesign.Card>
 
-    <Pagination Current="PageNumber" OnChange="OnPaginationChange" PageSize="PageSize" Total="TotalItems" ShowTotal="ShowTotal"></Pagination>
-<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
+    @*  <Pagination Current="PageNumber" OnChange="OnPaginationChange" PageSize="PageSize" Total="TotalItems" ShowTotal="ShowTotal"></Pagination>
+<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" /> *@
 </div>
 
 <style>
@@ -92,14 +79,35 @@
         backdrop-filter: blur(12px);
     }
 
+    .ant-card-extra {
+        position: absolute;
+        left: 0;
+        right: 0;
+        margin-left: auto;
+        margin-right: auto;
+        width: 50%;
+    }
+
     .ant-table {
         /* background: inherit; */
     }
 
-    .ant-table table {
-        table-layout: fixed;
+        .ant-table table {
+            table-layout: fixed;
+        }
+
+    .ant-table-pagination {
+        position: sticky;
+        bottom: 0;
+        padding: 16px 0;
+        margin: 0;
+        background: #141414;
     }
 
+        .ant-table-pagination.ant-pagination {
+            margin: 0;
+        }
+
     .j-pager {
         position: fixed;
         bottom: 0;
@@ -202,8 +210,8 @@
 
     private async Task HandleTableChange(AntDesign.TableModels.QueryModel<TagSearchItem> queryModel)
     {
-        PageNumber = queryModel.PageIndex;
-        PageSize = queryModel.PageSize;
+        // PageNumber = queryModel.PageIndex;
+        // PageSize = queryModel.PageSize;
 
         _sortOptions = MapSortOptions(queryModel);
 

JSMR.Worker/JSMR.Worker.csproj

@@ -31,6 +31,8 @@
     <PackageReference Include="Serilog.Extensions.Hosting" Version="10.0.0" />
     <PackageReference Include="Serilog.Settings.Configuration" Version="10.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
+    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
     <PackageReference Include="Spectre.Console" Version="0.55.2" />
     <PackageReference Include="System.CommandLine" Version="2.0.7" />
   </ItemGroup>

JSMR.Worker/Program.cs

@@ -7,7 +7,9 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using Serilog;
+using Serilog.Events;
 using System.CommandLine;
 using System.Text;
 
@@ -29,9 +31,23 @@ if (builder.Environment.IsDevelopment())
 string connectionString = builder.Configuration.GetConnectionString("AppDb")
     ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb");
 
-//builder.Services.AddSerilog(o => o
-//    .WriteTo.Console()
-//    .MinimumLevel.Warning());
+string? seqUrl = builder.Configuration["Seq:ServerUrl"];
+
+Log.Logger = new LoggerConfiguration()
+    .MinimumLevel.Information()
+    .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
+    .MinimumLevel.Override("System", LogEventLevel.Warning)
+    .MinimumLevel.Override("Polly", LogEventLevel.Warning)
+    .WriteTo.Logger(lc => lc
+        .MinimumLevel.Warning()
+        .WriteTo.Console())
+    .WriteTo.File(
+        path: "logs/jsmr-worker-.log",
+        rollingInterval: RollingInterval.Day)
+    .CreateLogger();
+
+builder.Logging.ClearProviders();
+builder.Services.AddSerilog();
 
 builder.Services
     .AddApplication()