using MangaReader.Core.Data; using MangaReader.Core.Metadata; using Microsoft.EntityFrameworkCore; using System.Text.RegularExpressions; namespace MangaReader.Core.Pipeline; public partial class MangaPipeline(MangaContext context) : IMangaPipeline { enum TitleType { Primary, Secondary } public async Task RunMetadataAsync(MangaMetadataPipelineRequest request, CancellationToken cancellationToken) { string sourceName = request.SourceName; string sourceUrl = request.SourceUrl; SourceManga sourceManga = request.SourceManga; Source source = await GetOrAddSourceAsync(sourceName); Manga manga = await GetOrAddMangaAsync(sourceManga, sourceUrl); MangaSource mangaSource = await AddMangaSourceAsync(sourceUrl, manga, source); await AddSourceTitleAsync(mangaSource, sourceManga.Title, TitleType.Primary); await AddTitleAsync(manga, sourceManga.Title, TitleType.Primary); foreach (SourceMangaDescription description in sourceManga.Descriptions) { await AddSourceDescriptionAsync(mangaSource, description); //await AddDescriptionAsync(mangaSource, description); } foreach (SourceMangaTitle alternateTitle in sourceManga.AlternateTitles) { await AddSourceTitleAsync(mangaSource, alternateTitle, TitleType.Secondary); await AddTitleAsync(manga, alternateTitle, TitleType.Secondary); } foreach (string genre in sourceManga.Genres) { await LinkGenreAsync(manga, genre); } foreach (SourceMangaContributor contributor in sourceManga.Contributors) { await LinkMangaContributorAsync(manga, contributor); } foreach (SourceMangaChapter chapter in sourceManga.Chapters) { await AddChapterAsync(mangaSource, chapter); } foreach (string coverArtUrl in sourceManga.CoverArtUrls) { await AddCoverAsync(mangaSource, coverArtUrl); } context.SaveChanges(); } private async Task GetOrAddSourceAsync(string sourceName) { Source? source = await context.Sources.FirstOrDefaultAsync(s => s.Name == sourceName); if (source != null) return source; source = new() { Name = sourceName }; context.Sources.Add(source); return source; } private async Task GetOrAddMangaAsync(SourceManga sourceManga, string sourceUrl) { //Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => // manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name)); Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => manga.Sources.Any(mangaSource => mangaSource.Url == sourceUrl)); if (manga != null) return manga; manga = new() { Slug = GenerateSlug(sourceManga.Title.Name), }; context.Add(manga); return manga; } private static string GenerateSlug(string title) { title = title.ToLowerInvariant(); title = RemoveInvalidCharsRegex().Replace(title, ""); // remove invalid chars title = RemoveSpacesWithDashRegex().Replace(title, "-"); // replace spaces with dash return title.Trim('-'); } [GeneratedRegex(@"[^a-z0-9\s-]")] private static partial Regex RemoveInvalidCharsRegex(); [GeneratedRegex(@"\s+")] private static partial Regex RemoveSpacesWithDashRegex(); private async Task AddMangaSourceAsync(string sourceUrl, Manga manga, Source source) { MangaSource? mangaSource = await context.MangaSources.FirstOrDefaultAsync(ms => ms.Manga == manga && ms.Source == source && ms.Url == sourceUrl); if (mangaSource != null) return mangaSource; mangaSource = new() { Manga = manga, Source = source, Url = sourceUrl }; context.MangaSources.Add(mangaSource); return mangaSource; } private async Task AddSourceTitleAsync(MangaSource mangaSource, SourceMangaTitle sourceMangaTitle, TitleType titleType) { SourceTitle? sourceTitle = await context.SourceTitles.FirstOrDefaultAsync(mt => mt.MangaSource == mangaSource && mt.Name == sourceMangaTitle.Name); if (sourceTitle != null) return; sourceTitle = new() { MangaSource = mangaSource, Name = sourceMangaTitle.Name, Language = sourceMangaTitle.Language, IsPrimary = titleType == TitleType.Primary }; context.SourceTitles.Add(sourceTitle); } private async Task AddSourceDescriptionAsync(MangaSource mangaSource, SourceMangaDescription? sourceMangaDescription) { if (sourceMangaDescription == null) return; SourceDescription? sourceDescription = await context.SourceDescriptions.FirstOrDefaultAsync(md => md.MangaSource == mangaSource && md.Language == sourceMangaDescription.Language); if (sourceDescription != null) { sourceDescription.Text = sourceMangaDescription.Name; return; } sourceDescription = new() { MangaSource = mangaSource, Text = sourceMangaDescription.Name, Language = sourceMangaDescription.Language }; context.SourceDescriptions.Add(sourceDescription); } private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle, TitleType titleType) { MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt => mt.Manga == manga && mt.Name == sourceMangaTitle.Name); if (mangaTitle != null) return; mangaTitle = new() { Manga = manga, Name = sourceMangaTitle.Name, Language = sourceMangaTitle.Language, IsPrimary = titleType == TitleType.Primary }; context.MangaTitles.Add(mangaTitle); } private async Task AddDescriptionAsync(Manga manga, SourceMangaDescription? sourceMangaDescription) { if (sourceMangaDescription == null) return; MangaDescription? mangaDescription = await context.MangaDescriptions.FirstOrDefaultAsync(md => md.Manga == manga && md.Language == sourceMangaDescription.Language); if (mangaDescription != null) { mangaDescription.Text = sourceMangaDescription.Name; return; } mangaDescription = new() { Manga = manga, Text = sourceMangaDescription.Name, Language = sourceMangaDescription.Language }; context.MangaDescriptions.Add(mangaDescription); } private async Task LinkGenreAsync(Manga manga, string genreName) { Genre genre = await GetOrAddGenreAsync(genreName); MangaGenre? mangaGenre = await context.MangaGenres.FirstOrDefaultAsync(x => x.Manga == manga && x.Genre == genre); if (mangaGenre != null) return; mangaGenre = new() { Manga = manga, Genre = genre }; context.MangaGenres.Add(mangaGenre); } private async Task GetOrAddGenreAsync(string genreName) { Genre? genre = await context.Genres.FirstOrDefaultAsync(x => x.Name == genreName); if (genre != null) return genre; genre = new() { Name = genreName, }; await context.Genres.AddAsync(genre); return genre; } private async Task LinkMangaContributorAsync(Manga manga, SourceMangaContributor sourceMangaContributor) { Contributor contributor = await GetOrAddContributorAsync(sourceMangaContributor.Name); MangaContributor? mangaContributor = await context.MangaContributors.FirstOrDefaultAsync(x => x.Manga == manga && x.Contributor == contributor && x.Role == sourceMangaContributor.Role); if (mangaContributor != null) return; mangaContributor = new() { Manga = manga, Contributor = contributor, Role = sourceMangaContributor.Role }; context.MangaContributors.Add(mangaContributor); } private async Task GetOrAddContributorAsync(string contributorName) { Contributor? trackedContributor = context.ChangeTracker .Entries() .Select(e => e.Entity) .FirstOrDefault(c => c.Name == contributorName); if (trackedContributor is not null) return trackedContributor; Contributor? contributor = await context.Contributors.FirstOrDefaultAsync(x => x.Name == contributorName); if (contributor == null) { contributor = new() { Name = contributorName, }; await context.Contributors.AddAsync(contributor); } return contributor; } private async Task AddChapterAsync(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { SourceChapter sourceChapter = await GetSourceChapter(mangaSource, sourceMangaChapter) ?? AddSourceChapter(mangaSource, sourceMangaChapter); if (sourceChapter.VolumeNumber is null && sourceMangaChapter.Volume is not null) { sourceChapter.VolumeNumber = sourceMangaChapter.Volume; } if (sourceChapter.Title is null && sourceMangaChapter.Title is not null) { sourceChapter.Title = sourceMangaChapter.Title; } } private async Task GetSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { return await context.SourceChapters.FirstOrDefaultAsync(x => x.MangaSource == mangaSource && x.ChapterNumber == sourceMangaChapter.Number); } private SourceChapter AddSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter) { SourceChapter sourceChapter = new() { MangaSource = mangaSource, ChapterNumber = sourceMangaChapter.Number, Url = sourceMangaChapter.Url }; context.SourceChapters.Add(sourceChapter); return sourceChapter; } private async Task AddCoverAsync(MangaSource mangaSource, string coverArtUrl) { SourceCover? sourceCover = await context.SourceCovers.FirstOrDefaultAsync(x => x.MangaSource == mangaSource && x.Url == coverArtUrl); if (sourceCover == null) { sourceCover = new() { MangaSource = mangaSource, Url = coverArtUrl }; context.SourceCovers.Add(sourceCover); } } public async Task RunPagesAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken) { SourceChapter? sourceChapter = await context.SourceChapters.FirstOrDefaultAsync(x => x.SourceChapterId == request.SourceChapterId, cancellationToken); if (sourceChapter == null) return; int currentPageNumber = 1; foreach (string pageImageUrl in request.PageImageUrls) { await AddOrUpdateSourcePageAsync(sourceChapter, currentPageNumber++, pageImageUrl, cancellationToken); } } private async Task AddOrUpdateSourcePageAsync(SourceChapter sourceChapter, int pageNumber, string pageImageUrl, CancellationToken cancellationToken) { SourcePage? sourcePage = await context.SourcePages.FirstOrDefaultAsync(x => x.Chapter == sourceChapter && x.PageNumber == pageNumber, cancellationToken); if (sourcePage == null) { sourcePage = new() { Chapter = sourceChapter, PageNumber = pageNumber, Url = pageImageUrl }; await context.SourcePages.AddAsync(sourcePage, cancellationToken); } else { sourcePage.Url = pageImageUrl; } } }