Added logging to scanner.

JSMR.Application/Scanning/ScanVoiceWorksHandler.cs

@@ -1,10 +1,13 @@
 using JSMR.Application.Common.Caching;
 using JSMR.Application.Scanning.Contracts;
 using JSMR.Application.Scanning.Ports;
+using Microsoft.Extensions.Logging;
+using System.Diagnostics;
 
 namespace JSMR.Application.Scanning;
 
 public sealed class ScanVoiceWorksHandler(
+    ILogger<ScanVoiceWorksHandler> logger,
     IVoiceWorkScannerRepository scannerRepository,
     IVoiceWorkUpdaterRepository updaterRepository,
     ISpamCircleCache spamCircleCache,
@@ -13,7 +16,28 @@ public sealed class ScanVoiceWorksHandler(
 {
     public async Task<ScanVoiceWorksResponse> HandleAsync(ScanVoiceWorksRequest request, CancellationToken cancellationToken)
     {
+        using IDisposable? scope = logger.BeginScope(new Dictionary<string, object>
+        {
+            ["Locale"] = request.Locale,
+            ["PageNumber"] = request.PageNumber,
+            ["PageSize"] = request.PageSize
+        });
+
+        Stopwatch stopwatch = Stopwatch.StartNew();
+        string currentPhase = "initialization";
+
+        try
+        {
+            logger.LogInformation(
+                "Starting scan handler for page {PageNumber}, page size {PageSize}, locale {Locale}",
+                request.PageNumber,
+                request.PageSize,
+                request.Locale);
+
+            currentPhase = "resolve_scanner";
             IVoiceWorksScanner? scanner = scannerRepository.GetScanner(request.Locale);
+
+            currentPhase = "resolve_updater";
             IVoiceWorkUpdater? updater = updaterRepository.GetUpdater(request.Locale);
 
             if (scanner is null)
@@ -22,34 +46,118 @@ public sealed class ScanVoiceWorksHandler(
             if (updater is null)
                 throw new InvalidOperationException($"No updater registered for locale {request.Locale}.");
 
+            currentPhase = "load_spam_circle_cache";
+            var spamStopwatch = Stopwatch.StartNew();
+
+            string[] excludedMakerIds = await spamCircleCache.GetAsync(cancellationToken);
+
+            spamStopwatch.Stop();
+            logger.LogInformation(
+                "Loaded spam circle cache in {ElapsedMs} ms. ExcludedMakerCount={ExcludedMakerCount}",
+                spamStopwatch.ElapsedMilliseconds,
+                excludedMakerIds.Length);
+
             VoiceWorkScanOptions options = new(
                 PageNumber: request.PageNumber,
                 PageSize: request.PageSize,
-            ExcludedMakerIds: await spamCircleCache.GetAsync(cancellationToken),
+                ExcludedMakerIds: excludedMakerIds,
                 ExcludePartiallyAIGeneratedWorks: true,
                 ExcludeAIGeneratedWorks: true
             );
 
+            currentPhase = "scan_page";
+            var scanStopwatch = Stopwatch.StartNew();
+
             VoiceWorkScanResult scanResult = await scanner.ScanPageAsync(options, cancellationToken);
 
+            scanStopwatch.Stop();
+
+            logger.LogInformation(
+                "Scanned source page in {ElapsedMs} ms. EndOfResults={EndOfResults}, ResultCount={ResultCount}",
+                scanStopwatch.ElapsedMilliseconds,
+                scanResult.EndOfResults,
+                scanResult.Works.Length);
+
+
             if (scanResult.EndOfResults)
             {
+                stopwatch.Stop();
+
+                logger.LogInformation(
+                    "End of results reached for page {PageNumber}. TotalElapsedMs={ElapsedMs}",
+                    request.PageNumber,
+                    stopwatch.ElapsedMilliseconds);
+
                 return new ScanVoiceWorksResponse(
                     Results: [],
                     EndOfResults: true
                 );
             }
 
+            currentPhase = "build_ingests";
+            var ingestSw = Stopwatch.StartNew();
+
             VoiceWorkIngest[] ingests = await ingestBuilder.BuildAsync(scanResult, cancellationToken);
 
+            ingestSw.Stop();
+            logger.LogInformation(
+                "Built ingests in {ElapsedMs} ms. IngestCount={IngestCount}",
+                ingestSw.ElapsedMilliseconds,
+                ingests.Length);
+
+            currentPhase = "upsert_voiceworks";
+            var upsertSw = Stopwatch.StartNew();
+
             VoiceWorkUpsertResult[] upsertResults = await updater.UpsertAsync(ingests, cancellationToken);
+
+            upsertSw.Stop();
+
             int[] voiceWorkIds = [.. upsertResults.Where(x => x.VoiceWorkId.HasValue).Select(x => x.VoiceWorkId!.Value)];
 
+            int issueCount = upsertResults.Sum(x => x.Issues.Count);
+
+            logger.LogInformation(
+                "Upserted voice works in {ElapsedMs} ms. UpsertResultCount={UpsertResultCount}, VoiceWorkIdCount={VoiceWorkIdCount}, IssueCount={IssueCount}",
+                upsertSw.ElapsedMilliseconds,
+                upsertResults.Length,
+                voiceWorkIds.Length,
+                issueCount);
+
+            currentPhase = "update_search";
+            var searchSw = Stopwatch.StartNew();
+
             await searchUpdater.UpdateAsync(voiceWorkIds, cancellationToken);
 
+            searchSw.Stop();
+            logger.LogInformation(
+                "Updated search index in {ElapsedMs} ms. VoiceWorkIdCount={VoiceWorkIdCount}",
+                searchSw.ElapsedMilliseconds,
+                voiceWorkIds.Length);
+
+            stopwatch.Stop();
+
+            logger.LogInformation(
+                "Completed scan handler for page {PageNumber} in {ElapsedMs} ms",
+                request.PageNumber,
+                stopwatch.ElapsedMilliseconds);
+
             return new ScanVoiceWorksResponse(
                 Results: upsertResults,
                 EndOfResults: false
             );
         }
+        catch (Exception ex)
+        {
+            stopwatch.Stop();
+
+            logger.LogError(
+                ex,
+                "Scan handler failed during phase {Phase} for page {PageNumber} after {ElapsedMs} ms",
+                currentPhase,
+                request.PageNumber,
+                stopwatch.ElapsedMilliseconds);
+
+            throw;
+        }
+    }
 }

JSMR.UI.Blazor/Components/JPagination2.razor

@@ -0,0 +1,113 @@
+@using JSMR.UI.Blazor.Enums
+
+<div class="pagination">
+    <div>
+        <label>@IndexInfo</label>
+    </div>
+    @* <MudPagination class="pager" ShowFirstButton="true" ShowLastButton="true" Count="@((int)Math.Ceiling((decimal)TotalItems / (decimal)PageSize))" Selected="@PageNumber" SelectedChanged="OnSelectedChanged" /> *@
+    <AntDesign.Pagination ShowSizeChanger="false" Total="TotalItems" Current="PageNumber" PageSize="PageSize" OnChange="OnPaginationChanged"></AntDesign.Pagination>
+    <div class="page-sizes">
+        @RightContent
+        <div class="page-size-select">
+            <AntDesign.Select TItem="PageSizeItem"
+                              TItemValue="int"
+                              DataSource="PageSizes3"
+                              LabelName="Text"
+                              ValueName="Value"
+                              Value="PageSize"
+                              ValueChanged="OnPageSizeChanged">
+            </AntDesign.Select>
+        </div>
+        
+        @* <BitDropdown Items="PageSizes2"
+                     Style="min-width: 7rem"
+                     Placeholder="Select..."
+                     TItem="BitDropdownItem<int>"
+                     TValue="int"
+                     Value="PageSize"
+                     ValueChanged="OnPageSizeChanged">
+            <PrefixTemplate>
+                <InputPrefix Graphic="Graphic.Grid" Tooltip="Page Size"></InputPrefix>
+            </PrefixTemplate>
+        </BitDropdown> *@
+    </div>
+</div>
+
+<style>
+    .pagination {
+        padding: 16px 16px;
+        background: transparent;
+        background: #141414;
+    }
+
+    .ant-pagination {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+    }
+
+    .page-size-select {
+        width: 12em;
+    }
+</style>
+
+@code {
+    [Parameter]
+    public int PageNumber { get; set; }
+
+    [Parameter]
+    public EventCallback<int> PageNumberChanged { get; set; }
+
+    [Parameter]
+    public int[] PageSizes { get; set; } = [5, 10, 25, 50, 100];
+
+    List<BitDropdownItem<int>> PageSizes2 => [.. PageSizes.Select(x => new BitDropdownItem<int>() { Text = x.ToString(), Value = x })];
+
+    List<PageSizeItem> PageSizes3 => [.. PageSizes.Select(x => new PageSizeItem() { Text = $"{x} items / page", Value = x })];
+
+    [Parameter]
+    public int PageSize { get; set; }
+
+    [Parameter]
+    public EventCallback<int> PageSizeChanged { get; set; }
+
+    [Parameter]
+    public int TotalItems { get; set; }
+
+    [Parameter]
+    public EventCallback<int> TotalItemsChanged { get; set; }
+
+    [Parameter]
+    public RenderFragment? RightContent { get; set; }
+
+    public string IndexInfo => TotalItems == 0 ? "No items" : $"{StartIndex.ToString("n0")} - {EndIndex.ToString("n0")} of {TotalItems.ToString("n0")} items";
+
+    public int StartIndex => (PageNumber - 1) * PageSize + 1;
+    public int EndIndex => PageNumber * PageSize < TotalItems ? PageNumber * PageSize : TotalItems;
+
+    private async Task OnSelectedChanged(int newPage)
+    {
+        PageNumber = newPage;
+        await PageNumberChanged.InvokeAsync(newPage);
+    }
+
+    private async Task OnPageSizeChanged(int newPageSize)
+    {
+        PageSize = newPageSize;
+        await PageSizeChanged.InvokeAsync(newPageSize);
+    }
+
+    private async Task OnPaginationChanged(AntDesign.PaginationEventArgs args)
+    {
+        int newPage = args.Page;
+
+        PageNumber = newPage;
+        await PageNumberChanged.InvokeAsync(newPage);
+    }
+
+    public class PageSizeItem
+    {
+        public string? Text { get; set; }
+        public int Value { get; set; }
+    }
+}

JSMR.UI.Blazor/Pages/Creators.razor

@@ -66,11 +66,10 @@
         </AntDesign.ActionColumn> *@
             </ColumnDefinitions>
         </AntDesign.Table>
+        <JPagination2 PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
     </Body>
 </AntDesign.Card>
 
-<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
-
 <style>
     .mud-table-root {
         table-layout: fixed;
@@ -87,6 +86,15 @@
         backdrop-filter: blur(12px);
     }
 
+    .ant-card-extra {
+        position: absolute;
+        left: 0;
+        right: 0;
+        margin-left: auto;
+        margin-right: auto;
+        width: 50%;
+    }
+
     .ant-table {
         /* background: inherit; */
     }

JSMR.UI.Blazor/Pages/Tags.razor

@@ -13,9 +13,9 @@
 <PageTitle>Tags</PageTitle>
 
 <div class="fdsfds">
-@* <h1>Tags</h1> *@
+    @* <h1>Tags</h1> *@
 
-@* <MudTextField T="string" Value="Keywords" ValueChanged="OnKeywordsChanged" Immediate="true" DebounceInterval="500" Label="Filter" Variant="MudBlazor.Variant.Text" Adornment="@Adornment.Start" AdornmentIcon="@Icons.Material.Outlined.Search" />
+    @* <MudTextField T="string" Value="Keywords" ValueChanged="OnKeywordsChanged" Immediate="true" DebounceInterval="500" Label="Filter" Variant="MudBlazor.Variant.Text" Adornment="@Adornment.Start" AdornmentIcon="@Icons.Material.Outlined.Search" />
  *@
     <AntDesign.Card Title=@("Tags") Class="ant-blurred-card">
         <Extra>
@@ -26,24 +26,12 @@
             </AntDesign.Input>
         </Extra>
         <Body>
-        <Space Direction="SpaceDirection.Vertical" Size="@($"2em")">
-            @* <SpaceItem>
-                <AntDesign.Input TValue="string" Value="Keywords" ValueChanged="OnKeywordsChanged" DebounceMilliseconds="500" Placeholder="Filter">
-                    <Prefix>
-                        <AntDesign.Icon Type="@AntDesign.IconType.Outline.Search" />
-                    </Prefix>
-                </AntDesign.Input>
-            </SpaceItem> *@
-            <SpaceItem>
             <AntDesign.Table Responsive
                              DataSource="@(searchResults?.Items ?? Enumerable.Empty<TagSearchItem>())"
                              Total="@(searchResults?.TotalItems ?? 0)"
                              TItem="TagSearchItem"
                              Loading="LoadingData"
-                                 HidePagination="@false"
+                             HidePagination="@true"
-                                 PageSize="PageSize"
-                                 PageIndex="PageNumber"
-                                 PaginationPosition="bottomCenter"
                              RemoteDataSource="@true"
                              RowKey="x=>x.TagId"
                              OnChange="HandleTableChange">
@@ -67,13 +55,12 @@
                     </AntDesign.PropertyColumn>
                 </ColumnDefinitions>
             </AntDesign.Table>
-            </SpaceItem>
+            <JPagination2 PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
-        </Space>
         </Body>
     </AntDesign.Card>
 
-    <Pagination Current="PageNumber" OnChange="OnPaginationChange" PageSize="PageSize" Total="TotalItems" ShowTotal="ShowTotal"></Pagination>
+    @*  <Pagination Current="PageNumber" OnChange="OnPaginationChange" PageSize="PageSize" Total="TotalItems" ShowTotal="ShowTotal"></Pagination>
-<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" />
+<JPagination PageNumber="PageNumber" PageNumberChanged="OnPageNumberChanged" PageSize="PageSize" PageSizeChanged="OnPageSizeChanged" TotalItems="TotalItems" /> *@
 </div>
 
 <style>
@@ -92,6 +79,15 @@
         backdrop-filter: blur(12px);
     }
 
+    .ant-card-extra {
+        position: absolute;
+        left: 0;
+        right: 0;
+        margin-left: auto;
+        margin-right: auto;
+        width: 50%;
+    }
+
     .ant-table {
         /* background: inherit; */
     }
@@ -100,6 +96,18 @@
             table-layout: fixed;
         }
 
+    .ant-table-pagination {
+        position: sticky;
+        bottom: 0;
+        padding: 16px 0;
+        margin: 0;
+        background: #141414;
+    }
+
+        .ant-table-pagination.ant-pagination {
+            margin: 0;
+        }
+
     .j-pager {
         position: fixed;
         bottom: 0;
@@ -202,8 +210,8 @@
 
     private async Task HandleTableChange(AntDesign.TableModels.QueryModel<TagSearchItem> queryModel)
     {
-        PageNumber = queryModel.PageIndex;
+        // PageNumber = queryModel.PageIndex;
-        PageSize = queryModel.PageSize;
+        // PageSize = queryModel.PageSize;
 
         _sortOptions = MapSortOptions(queryModel);
 

JSMR.Worker/JSMR.Worker.csproj

@@ -31,6 +31,8 @@
     <PackageReference Include="Serilog.Extensions.Hosting" Version="10.0.0" />
     <PackageReference Include="Serilog.Settings.Configuration" Version="10.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
+    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
     <PackageReference Include="Spectre.Console" Version="0.55.2" />
     <PackageReference Include="System.CommandLine" Version="2.0.7" />
   </ItemGroup>

JSMR.Worker/Program.cs

@@ -7,7 +7,9 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using Serilog;
+using Serilog.Events;
 using System.CommandLine;
 using System.Text;
 
@@ -29,9 +31,23 @@ if (builder.Environment.IsDevelopment())
 string connectionString = builder.Configuration.GetConnectionString("AppDb")
     ?? throw new InvalidOperationException("Missing ConnectionStrings:AppDb");
 
-//builder.Services.AddSerilog(o => o
+string? seqUrl = builder.Configuration["Seq:ServerUrl"];
-//    .WriteTo.Console()
+
-//    .MinimumLevel.Warning());
+Log.Logger = new LoggerConfiguration()
+    .MinimumLevel.Information()
+    .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
+    .MinimumLevel.Override("System", LogEventLevel.Warning)
+    .MinimumLevel.Override("Polly", LogEventLevel.Warning)
+    .WriteTo.Logger(lc => lc
+        .MinimumLevel.Warning()
+        .WriteTo.Console())
+    .WriteTo.File(
+        path: "logs/jsmr-worker-.log",
+        rollingInterval: RollingInterval.Day)
+    .CreateLogger();
+
+builder.Logging.ClearProviders();
+builder.Services.AddSerilog();
 
 builder.Services
     .AddApplication()