using MangaReader.Core.Data; using MangaReader.Core.Metadata; using Microsoft.EntityFrameworkCore; using SixLabors.ImageSharp; using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.PixelFormats; using System.Security.Cryptography; using System.Text.RegularExpressions; namespace MangaReader.Core.Pipeline; public partial class MangaPipeline(MangaContext context) : IMangaPipeline { enum TitleType { Primary, Secondary } public async Task RunMetadataAsync(MangaMetadataPipelineRequest request, CancellationToken cancellationToken) { string sourceName = request.SourceName; string sourceUrl = request.SourceUrl; SourceManga sourceManga = request.SourceManga; Source source = await GetOrAddSourceAsync(sourceName); Manga manga = await GetOrAddMangaAsync(sourceManga, sourceUrl); MangaSource mangaSource = await AddMangaSourceAsync(sourceUrl, manga, source); await AddSourceTitleAsync(mangaSource, sourceManga.Title, TitleType.Primary); await AddTitleAsync(manga, sourceManga.Title, TitleType.Primary); foreach (SourceMangaDescription description in sourceManga.Descriptions) { await AddSourceDescriptionAsync(mangaSource, description); await AddDescriptionAsync(manga, description); } foreach (SourceMangaTitle alternateTitle in sourceManga.AlternateTitles) { await AddSourceTitleAsync(mangaSource, alternateTitle, TitleType.Secondary); await AddTitleAsync(manga, alternateTitle, TitleType.Secondary); } foreach (string genre in sourceManga.Genres) { await LinkGenreAsync(manga, genre); } foreach (SourceMangaContributor contributor in sourceManga.Contributors) { await LinkMangaContributorAsync(manga, contributor); } foreach (SourceMangaChapter chapter in sourceManga.Chapters) { await AddChapterAsync(mangaSource, chapter); } foreach (string coverArtUrl in sourceManga.CoverArtUrls) { await AddCoverAsync(mangaSource, coverArtUrl); } context.SaveChanges(); } private async Task GetOrAddSourceAsync(string sourceName) { Source? source = await context.Sources.FirstOrDefaultAsync(s => s.Name == sourceName); if (source != null) return source; source = new() { Name = sourceName }; context.Sources.Add(source); return source; } private async Task GetOrAddMangaAsync(SourceManga sourceManga, string sourceUrl) { //Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => // manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name)); Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => manga.Sources.Any(mangaSource => mangaSource.Url == sourceUrl)); if (manga != null) { manga.Year = sourceManga.Year ?? manga.Year; return manga; } manga = new() { Slug = GenerateSlug(sourceManga.Title.Name), }; context.Add(manga); return manga; } private static string GenerateSlug(string title) { title = title.ToLowerInvariant(); title = RemoveInvalidCharsRegex().Replace(title, ""); // remove invalid chars title = RemoveSpacesWithDashRegex().Replace(title, "-"); // replace spaces with dash return title.Trim('-'); } [GeneratedRegex(@"[^a-z0-9\s-]")] private static partial Regex RemoveInvalidCharsRegex(); [GeneratedRegex(@"\s+")] private static partial Regex RemoveSpacesWithDashRegex(); private async Task AddMangaSourceAsync(string sourceUrl, Manga manga, Source source) { MangaSource? mangaSource = await context.MangaSources.FirstOrDefaultAsync(ms => ms.Manga == manga && ms.Source == source && ms.Url == sourceUrl); if (mangaSource != null) return mangaSource; mangaSource = new() { Manga = manga, Source = source, Url = sourceUrl }; context.MangaSources.Add(mangaSource); return mangaSource; } private async Task AddSourceTitleAsync(MangaSource mangaSource, SourceMangaTitle sourceMangaTitle, TitleType titleType) { SourceTitle? sourceTitle = await context.SourceTitles.FirstOrDefaultAsync(mt => mt.MangaSource == mangaSource && mt.Name == sourceMangaTitle.Name); if (sourceTitle != null) return; sourceTitle = new() { MangaSource = mangaSource, Name = sourceMangaTitle.Name, Language = sourceMangaTitle.Language, IsPrimary = titleType == TitleType.Primary }; context.SourceTitles.Add(sourceTitle); } private async Task AddSourceDescriptionAsync(MangaSource mangaSource, SourceMangaDescription? sourceMangaDescription) { if (sourceMangaDescription == null) return; SourceDescription? sourceDescription = await context.SourceDescriptions.FirstOrDefaultAsync(md => md.MangaSource == mangaSource && md.Language == sourceMangaDescription.Language); if (sourceDescription != null) { sourceDescription.Text = sourceMangaDescription.Name; return; } sourceDescription = new() { MangaSource = mangaSource, Text = sourceMangaDescription.Name, Language = sourceMangaDescription.Language }; context.SourceDescriptions.Add(sourceDescription); } private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle, TitleType titleType) { MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt => mt.Manga == manga && mt.Name == sourceMangaTitle.Name); if (mangaTitle != null) return; mangaTitle = new() { Manga = manga, Name = sourceMangaTitle.Name, Language = sourceMangaTitle.Language, IsPrimary = titleType == TitleType.Primary }; context.MangaTitles.Add(mangaTitle); } private async Task AddDescriptionAsync(Manga manga, SourceMangaDescription? sourceMangaDescription) { if (sourceMangaDescription == null) return; MangaDescription? mangaDescription = await context.MangaDescriptions.FirstOrDefaultAsync(md => md.Manga == manga && md.Language == sourceMangaDescription.Language); if (mangaDescription != null) { mangaDescription.Text = sourceMangaDescription.Name; return; } mangaDescription = new() { Manga = manga, Text = sourceMangaDescription.Name, Language = sourceMangaDescription.Language }; context.MangaDescriptions.Add(mangaDescription); } private async Task LinkGenreAsync(Manga manga, string genreName) { Genre genre = await GetOrAddGenreAsync(genreName); MangaGenre? mangaGenre = await context.MangaGenres.FirstOrDefaultAsync(x => x.Manga == manga && x.Genre == genre); if (mangaGenre != null) return; mangaGenre = new() { Manga = manga, Genre = genre }; context.MangaGenres.Add(mangaGenre); } private async Task GetOrAddGenreAsync(string genreName) { Genre? genre = await context.Genres.FirstOrDefaultAsync(x => x.Name == genreName); if (genre != null) return genre; genre = new() { Name = genreName, }; await context.Genres.AddAsync(genre); return genre; } private async Task LinkMangaContributorAsync(Manga manga, SourceMangaContributor sourceMangaContributor) { Contributor contributor = await GetOrAddContributorAsync(sourceMangaContributor.Name); MangaContributor? mangaContributor = await context.MangaContributors.FirstOrDefaultAsync(x => x.Manga == manga && x.Contributor == contributor && x.Role == sourceMangaContributor.Role); if (mangaContributor != null) return; mangaContributor = new() { Manga = manga, Contributor = contributor, Role = sourceMangaContributor.Role }; context.MangaContributors.Add(mangaContributor); } private async Task GetOrAddContributorAsync(string contributorName) { Contributor? trackedContributor = context.ChangeTracker .Entries() .Select(e => e.Entity) .FirstOrDefault(c => c.Name == contributorName); if (trackedContributor is not null) return trackedContributor; Contributor? contributor = await context.Contributors.FirstOrDefaultAsync(x => x.Name == contributorName); if (contributor == null) { contributor = new() { Name = contributorName, }; await context.Contributors.AddAsync(contributor); } return contributor; } private async Task AddChapterAsync(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { SourceChapter sourceChapter = await GetSourceChapter(mangaSource, sourceMangaChapter) ?? AddSourceChapter(mangaSource, sourceMangaChapter); if (sourceChapter.VolumeNumber is null && sourceMangaChapter.Volume is not null) { sourceChapter.VolumeNumber = sourceMangaChapter.Volume; } if (sourceChapter.Title is null && sourceMangaChapter.Title is not null) { sourceChapter.Title = sourceMangaChapter.Title; } } private async Task GetSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { return await context.SourceChapters.FirstOrDefaultAsync(x => x.MangaSource == mangaSource && x.ChapterNumber == sourceMangaChapter.Number); } private SourceChapter AddSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { SourceChapter sourceChapter = new() { MangaSource = mangaSource, ChapterNumber = sourceMangaChapter.Number, Url = sourceMangaChapter.Url }; context.SourceChapters.Add(sourceChapter); return sourceChapter; } private async Task AddCoverAsync(MangaSource mangaSource, string coverArtUrl) { SourceCover? sourceCover = await context.SourceCovers.FirstOrDefaultAsync(x => x.MangaSource == mangaSource && x.Url == coverArtUrl); if (sourceCover == null) { sourceCover = new() { MangaSource = mangaSource, Url = coverArtUrl }; context.SourceCovers.Add(sourceCover); } } //public async Task RunCoversAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken) //{ // SourceCover[] sourceCovers = await context.SourceCovers // .Where(x => x.MangaCoverId == null) // .ToArrayAsync(cancellationToken); // foreach (SourceCover sourceCover in sourceCovers) // { // await AddOrUpdateCoverAsync(sourceCover, cancellationToken); // } //} //private async Task AddOrUpdateCoverAsync(SourceCover sourceCover, CancellationToken cancellationToken) //{ // HttpClient client = httpClientFactory.CreateClient(); // client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0"); // using HttpResponseMessage responseMessage = await client.GetAsync(sourceCover.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); // responseMessage.EnsureSuccessStatusCode(); // await using Stream networkStream = await responseMessage.Content.ReadAsStreamAsync(cancellationToken); // Image image = await Image.LoadAsync(networkStream, cancellationToken); // networkStream.Position = 0; // IImageFormat imageFormat = await Image.DetectFormatAsync(networkStream, cancellationToken); // // Normalize output extension (you can keep WEBP if your viewer supports it; WinUI often prefers JPEG/PNG) // string extension = (imageFormat.Name?.ToLowerInvariant()) switch // { // "png" => "png", // _ => "jpg" // }; // // Save to memory to hash & maybe convert formats // await using MemoryStream memoryStream = new(); // if (extension == "png") // await image.SaveAsPngAsync(memoryStream, cancellationToken); // else // await image.SaveAsJpegAsync(memoryStream, cancellationToken); // memoryStream.Position = 0; // // Compute hash for dedupe // string sha256; // using (var sha = SHA256.Create()) // sha256 = Convert.ToHexString(sha.ComputeHash(memoryStream)).ToLowerInvariant(); // memoryStream.Position = 0; // // Check if we already have this exact file // var existing = await context.MangaCovers.FirstOrDefaultAsync(c => c.Sha256 == sha256 && c.MangaId == sc.MangaSource.MangaId, ct); // var existing2 = await context.MangaCovers.FirstOrDefaultAsync(c => c. == sha256 && c.MangaId == sc.MangaSource.MangaId, ct); // MangaCover cover; // if (existing is not null) // { // cover = existing; // } // else // { // // Choose file path // var folder = _paths.GetMangaCoverFolder(sc.MangaSource.MangaId); // var guid = Guid.NewGuid(); // var fileName = $"{guid}.{ext}"; // var fullPath = Path.Combine(folder, fileName); // // Write to disk // await using (var fs = File.Create(fullPath)) // await ms.CopyToAsync(fs, ct); // // Create DB record // cover = new MangaCover // { // Manga = sc.MangaSource.Manga, // Guid = guid, // FileExtension = ext, // Sha256 = sha256, // Width = image.Width, // Height = image.Height, // IsPrimary = false // }; // context.MangaCovers.Add(cover); // } // // Link and mark done // sourceCover.MangaCover = cover; // //sc.Status = CoverDownloadStatus.Done; // //sc.Error = null; // await context.SaveChangesAsync(cancellationToken); //} public async Task RunPagesAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken) { SourceChapter? sourceChapter = await context.SourceChapters.FirstOrDefaultAsync(x => x.SourceChapterId == request.SourceChapterId, cancellationToken); if (sourceChapter == null) return; int currentPageNumber = 1; foreach (string pageImageUrl in request.PageImageUrls) { await AddOrUpdateSourcePageAsync(sourceChapter, currentPageNumber++, pageImageUrl, cancellationToken); } } private async Task AddOrUpdateSourcePageAsync(SourceChapter sourceChapter, int pageNumber, string pageImageUrl, CancellationToken cancellationToken) { SourcePage? sourcePage = await context.SourcePages.FirstOrDefaultAsync(x => x.Chapter == sourceChapter && x.PageNumber == pageNumber, cancellationToken); if (sourcePage == null) { sourcePage = new() { Chapter = sourceChapter, PageNumber = pageNumber, Url = pageImageUrl }; await context.SourcePages.AddAsync(sourcePage, cancellationToken); } else { sourcePage.Url = pageImageUrl; } } }