494 lines
16 KiB
C#
494 lines
16 KiB
C#
using MangaReader.Core.Data;
|
|
using MangaReader.Core.Metadata;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using SixLabors.ImageSharp;
|
|
using SixLabors.ImageSharp.Formats;
|
|
using SixLabors.ImageSharp.PixelFormats;
|
|
using System.Security.Cryptography;
|
|
using System.Text.RegularExpressions;
|
|
|
|
namespace MangaReader.Core.Pipeline;
|
|
|
|
public partial class MangaPipeline(MangaContext context) : IMangaPipeline
|
|
{
|
|
enum TitleType
|
|
{
|
|
Primary,
|
|
Secondary
|
|
}
|
|
|
|
public async Task RunMetadataAsync(MangaMetadataPipelineRequest request, CancellationToken cancellationToken)
|
|
{
|
|
string sourceName = request.SourceName;
|
|
string sourceUrl = request.SourceUrl;
|
|
SourceManga sourceManga = request.SourceManga;
|
|
|
|
Source source = await GetOrAddSourceAsync(sourceName);
|
|
Manga manga = await GetOrAddMangaAsync(sourceManga, sourceUrl);
|
|
MangaSource mangaSource = await AddMangaSourceAsync(sourceUrl, manga, source);
|
|
|
|
await AddSourceTitleAsync(mangaSource, sourceManga.Title, TitleType.Primary);
|
|
await AddTitleAsync(manga, sourceManga.Title, TitleType.Primary);
|
|
|
|
foreach (SourceMangaDescription description in sourceManga.Descriptions)
|
|
{
|
|
await AddSourceDescriptionAsync(mangaSource, description);
|
|
await AddDescriptionAsync(manga, description);
|
|
}
|
|
|
|
foreach (SourceMangaTitle alternateTitle in sourceManga.AlternateTitles)
|
|
{
|
|
await AddSourceTitleAsync(mangaSource, alternateTitle, TitleType.Secondary);
|
|
await AddTitleAsync(manga, alternateTitle, TitleType.Secondary);
|
|
}
|
|
|
|
foreach (string genre in sourceManga.Genres)
|
|
{
|
|
await LinkGenreAsync(manga, genre);
|
|
}
|
|
|
|
foreach (SourceMangaContributor contributor in sourceManga.Contributors)
|
|
{
|
|
await LinkMangaContributorAsync(manga, contributor);
|
|
}
|
|
|
|
foreach (SourceMangaChapter chapter in sourceManga.Chapters)
|
|
{
|
|
await AddChapterAsync(mangaSource, chapter);
|
|
}
|
|
|
|
foreach (string coverArtUrl in sourceManga.CoverArtUrls)
|
|
{
|
|
await AddCoverAsync(mangaSource, coverArtUrl);
|
|
}
|
|
|
|
context.SaveChanges();
|
|
}
|
|
|
|
private async Task<Source> GetOrAddSourceAsync(string sourceName)
|
|
{
|
|
Source? source = await context.Sources.FirstOrDefaultAsync(s => s.Name == sourceName);
|
|
|
|
if (source != null)
|
|
return source;
|
|
|
|
source = new()
|
|
{
|
|
Name = sourceName
|
|
};
|
|
|
|
context.Sources.Add(source);
|
|
|
|
return source;
|
|
}
|
|
|
|
private async Task<Manga> GetOrAddMangaAsync(SourceManga sourceManga, string sourceUrl)
|
|
{
|
|
//Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga =>
|
|
// manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name));
|
|
|
|
Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga =>
|
|
manga.Sources.Any(mangaSource => mangaSource.Url == sourceUrl));
|
|
|
|
if (manga != null)
|
|
{
|
|
manga.Year = sourceManga.Year ?? manga.Year;
|
|
return manga;
|
|
}
|
|
|
|
manga = new()
|
|
{
|
|
Slug = GenerateSlug(sourceManga.Title.Name),
|
|
};
|
|
|
|
context.Add(manga);
|
|
|
|
return manga;
|
|
}
|
|
|
|
private static string GenerateSlug(string title)
|
|
{
|
|
title = title.ToLowerInvariant();
|
|
title = RemoveInvalidCharsRegex().Replace(title, ""); // remove invalid chars
|
|
title = RemoveSpacesWithDashRegex().Replace(title, "-"); // replace spaces with dash
|
|
|
|
return title.Trim('-');
|
|
}
|
|
|
|
[GeneratedRegex(@"[^a-z0-9\s-]")]
|
|
private static partial Regex RemoveInvalidCharsRegex();
|
|
|
|
[GeneratedRegex(@"\s+")]
|
|
private static partial Regex RemoveSpacesWithDashRegex();
|
|
|
|
private async Task<MangaSource> AddMangaSourceAsync(string sourceUrl, Manga manga, Source source)
|
|
{
|
|
MangaSource? mangaSource = await context.MangaSources.FirstOrDefaultAsync(ms =>
|
|
ms.Manga == manga && ms.Source == source && ms.Url == sourceUrl);
|
|
|
|
if (mangaSource != null)
|
|
return mangaSource;
|
|
|
|
mangaSource = new()
|
|
{
|
|
Manga = manga,
|
|
Source = source,
|
|
Url = sourceUrl
|
|
};
|
|
|
|
context.MangaSources.Add(mangaSource);
|
|
|
|
return mangaSource;
|
|
}
|
|
|
|
private async Task AddSourceTitleAsync(MangaSource mangaSource, SourceMangaTitle sourceMangaTitle, TitleType titleType)
|
|
{
|
|
SourceTitle? sourceTitle = await context.SourceTitles.FirstOrDefaultAsync(mt =>
|
|
mt.MangaSource == mangaSource && mt.Name == sourceMangaTitle.Name);
|
|
|
|
if (sourceTitle != null)
|
|
return;
|
|
|
|
sourceTitle = new()
|
|
{
|
|
MangaSource = mangaSource,
|
|
Name = sourceMangaTitle.Name,
|
|
Language = sourceMangaTitle.Language,
|
|
IsPrimary = titleType == TitleType.Primary
|
|
};
|
|
|
|
context.SourceTitles.Add(sourceTitle);
|
|
}
|
|
|
|
private async Task AddSourceDescriptionAsync(MangaSource mangaSource, SourceMangaDescription? sourceMangaDescription)
|
|
{
|
|
if (sourceMangaDescription == null)
|
|
return;
|
|
|
|
SourceDescription? sourceDescription = await context.SourceDescriptions.FirstOrDefaultAsync(md =>
|
|
md.MangaSource == mangaSource && md.Language == sourceMangaDescription.Language);
|
|
|
|
if (sourceDescription != null)
|
|
{
|
|
sourceDescription.Text = sourceMangaDescription.Name;
|
|
return;
|
|
}
|
|
|
|
sourceDescription = new()
|
|
{
|
|
MangaSource = mangaSource,
|
|
Text = sourceMangaDescription.Name,
|
|
Language = sourceMangaDescription.Language
|
|
};
|
|
|
|
context.SourceDescriptions.Add(sourceDescription);
|
|
}
|
|
|
|
private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle, TitleType titleType)
|
|
{
|
|
MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt =>
|
|
mt.Manga == manga && mt.Name == sourceMangaTitle.Name);
|
|
|
|
if (mangaTitle != null)
|
|
return;
|
|
|
|
mangaTitle = new()
|
|
{
|
|
Manga = manga,
|
|
Name = sourceMangaTitle.Name,
|
|
Language = sourceMangaTitle.Language,
|
|
IsPrimary = titleType == TitleType.Primary
|
|
};
|
|
|
|
context.MangaTitles.Add(mangaTitle);
|
|
}
|
|
|
|
private async Task AddDescriptionAsync(Manga manga, SourceMangaDescription? sourceMangaDescription)
|
|
{
|
|
if (sourceMangaDescription == null)
|
|
return;
|
|
|
|
MangaDescription? mangaDescription = await context.MangaDescriptions.FirstOrDefaultAsync(md =>
|
|
md.Manga == manga && md.Language == sourceMangaDescription.Language);
|
|
|
|
if (mangaDescription != null)
|
|
{
|
|
mangaDescription.Text = sourceMangaDescription.Name;
|
|
return;
|
|
}
|
|
|
|
mangaDescription = new()
|
|
{
|
|
Manga = manga,
|
|
Text = sourceMangaDescription.Name,
|
|
Language = sourceMangaDescription.Language
|
|
};
|
|
|
|
context.MangaDescriptions.Add(mangaDescription);
|
|
}
|
|
|
|
private async Task LinkGenreAsync(Manga manga, string genreName)
|
|
{
|
|
Genre genre = await GetOrAddGenreAsync(genreName);
|
|
|
|
MangaGenre? mangaGenre = await context.MangaGenres.FirstOrDefaultAsync(x => x.Manga == manga && x.Genre == genre);
|
|
|
|
if (mangaGenre != null)
|
|
return;
|
|
|
|
mangaGenre = new()
|
|
{
|
|
Manga = manga,
|
|
Genre = genre
|
|
};
|
|
|
|
context.MangaGenres.Add(mangaGenre);
|
|
}
|
|
|
|
private async Task<Genre> GetOrAddGenreAsync(string genreName)
|
|
{
|
|
Genre? genre = await context.Genres.FirstOrDefaultAsync(x => x.Name == genreName);
|
|
|
|
if (genre != null)
|
|
return genre;
|
|
|
|
genre = new()
|
|
{
|
|
Name = genreName,
|
|
};
|
|
|
|
await context.Genres.AddAsync(genre);
|
|
|
|
return genre;
|
|
}
|
|
|
|
private async Task LinkMangaContributorAsync(Manga manga, SourceMangaContributor sourceMangaContributor)
|
|
{
|
|
Contributor contributor = await GetOrAddContributorAsync(sourceMangaContributor.Name);
|
|
|
|
MangaContributor? mangaContributor = await context.MangaContributors.FirstOrDefaultAsync(x =>
|
|
x.Manga == manga && x.Contributor == contributor && x.Role == sourceMangaContributor.Role);
|
|
|
|
if (mangaContributor != null)
|
|
return;
|
|
|
|
mangaContributor = new()
|
|
{
|
|
Manga = manga,
|
|
Contributor = contributor,
|
|
Role = sourceMangaContributor.Role
|
|
};
|
|
|
|
context.MangaContributors.Add(mangaContributor);
|
|
}
|
|
|
|
private async Task<Contributor> GetOrAddContributorAsync(string contributorName)
|
|
{
|
|
Contributor? trackedContributor = context.ChangeTracker
|
|
.Entries<Contributor>()
|
|
.Select(e => e.Entity)
|
|
.FirstOrDefault(c => c.Name == contributorName);
|
|
|
|
if (trackedContributor is not null)
|
|
return trackedContributor;
|
|
|
|
Contributor? contributor = await context.Contributors.FirstOrDefaultAsync(x => x.Name == contributorName);
|
|
|
|
if (contributor == null)
|
|
{
|
|
contributor = new()
|
|
{
|
|
Name = contributorName,
|
|
};
|
|
|
|
await context.Contributors.AddAsync(contributor);
|
|
}
|
|
|
|
return contributor;
|
|
}
|
|
|
|
private async Task AddChapterAsync(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter)
|
|
{
|
|
SourceChapter sourceChapter = await GetSourceChapter(mangaSource, sourceMangaChapter)
|
|
?? AddSourceChapter(mangaSource, sourceMangaChapter);
|
|
|
|
if (sourceChapter.VolumeNumber is null && sourceMangaChapter.Volume is not null)
|
|
{
|
|
sourceChapter.VolumeNumber = sourceMangaChapter.Volume;
|
|
}
|
|
|
|
if (sourceChapter.Title is null && sourceMangaChapter.Title is not null)
|
|
{
|
|
sourceChapter.Title = sourceMangaChapter.Title;
|
|
}
|
|
}
|
|
|
|
private async Task<SourceChapter?> GetSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter)
|
|
{
|
|
return await context.SourceChapters.FirstOrDefaultAsync(x =>
|
|
x.MangaSource == mangaSource && x.ChapterNumber == sourceMangaChapter.Number);
|
|
}
|
|
|
|
private SourceChapter AddSourceChapter(MangaSource mangaSource, SourceMangaChapter sourceMangaChapter)
|
|
{
|
|
SourceChapter sourceChapter = new()
|
|
{
|
|
MangaSource = mangaSource,
|
|
ChapterNumber = sourceMangaChapter.Number,
|
|
Url = sourceMangaChapter.Url
|
|
};
|
|
|
|
context.SourceChapters.Add(sourceChapter);
|
|
|
|
return sourceChapter;
|
|
}
|
|
|
|
private async Task AddCoverAsync(MangaSource mangaSource, string coverArtUrl)
|
|
{
|
|
SourceCover? sourceCover = await context.SourceCovers.FirstOrDefaultAsync(x =>
|
|
x.MangaSource == mangaSource && x.Url == coverArtUrl);
|
|
|
|
if (sourceCover == null)
|
|
{
|
|
sourceCover = new()
|
|
{
|
|
MangaSource = mangaSource,
|
|
Url = coverArtUrl
|
|
};
|
|
|
|
context.SourceCovers.Add(sourceCover);
|
|
}
|
|
}
|
|
|
|
//public async Task RunCoversAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken)
|
|
//{
|
|
// SourceCover[] sourceCovers = await context.SourceCovers
|
|
// .Where(x => x.MangaCoverId == null)
|
|
// .ToArrayAsync(cancellationToken);
|
|
|
|
// foreach (SourceCover sourceCover in sourceCovers)
|
|
// {
|
|
// await AddOrUpdateCoverAsync(sourceCover, cancellationToken);
|
|
// }
|
|
//}
|
|
|
|
//private async Task AddOrUpdateCoverAsync(SourceCover sourceCover, CancellationToken cancellationToken)
|
|
//{
|
|
// HttpClient client = httpClientFactory.CreateClient();
|
|
// client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0");
|
|
|
|
// using HttpResponseMessage responseMessage = await client.GetAsync(sourceCover.Url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
|
|
// responseMessage.EnsureSuccessStatusCode();
|
|
|
|
// await using Stream networkStream = await responseMessage.Content.ReadAsStreamAsync(cancellationToken);
|
|
|
|
// Image image = await Image.LoadAsync(networkStream, cancellationToken);
|
|
// networkStream.Position = 0;
|
|
|
|
// IImageFormat imageFormat = await Image.DetectFormatAsync(networkStream, cancellationToken);
|
|
|
|
// // Normalize output extension (you can keep WEBP if your viewer supports it; WinUI often prefers JPEG/PNG)
|
|
// string extension = (imageFormat.Name?.ToLowerInvariant()) switch
|
|
// {
|
|
// "png" => "png",
|
|
// _ => "jpg"
|
|
// };
|
|
|
|
// // Save to memory to hash & maybe convert formats
|
|
// await using MemoryStream memoryStream = new();
|
|
|
|
// if (extension == "png")
|
|
// await image.SaveAsPngAsync(memoryStream, cancellationToken);
|
|
// else
|
|
// await image.SaveAsJpegAsync(memoryStream, cancellationToken);
|
|
|
|
// memoryStream.Position = 0;
|
|
|
|
// // Compute hash for dedupe
|
|
// string sha256;
|
|
// using (var sha = SHA256.Create())
|
|
// sha256 = Convert.ToHexString(sha.ComputeHash(memoryStream)).ToLowerInvariant();
|
|
// memoryStream.Position = 0;
|
|
|
|
// // Check if we already have this exact file
|
|
// var existing = await context.MangaCovers.FirstOrDefaultAsync(c => c.Sha256 == sha256 && c.MangaId == sc.MangaSource.MangaId, ct);
|
|
// var existing2 = await context.MangaCovers.FirstOrDefaultAsync(c => c. == sha256 && c.MangaId == sc.MangaSource.MangaId, ct);
|
|
|
|
// MangaCover cover;
|
|
|
|
// if (existing is not null)
|
|
// {
|
|
// cover = existing;
|
|
// }
|
|
// else
|
|
// {
|
|
// // Choose file path
|
|
// var folder = _paths.GetMangaCoverFolder(sc.MangaSource.MangaId);
|
|
// var guid = Guid.NewGuid();
|
|
// var fileName = $"{guid}.{ext}";
|
|
// var fullPath = Path.Combine(folder, fileName);
|
|
|
|
// // Write to disk
|
|
// await using (var fs = File.Create(fullPath))
|
|
// await ms.CopyToAsync(fs, ct);
|
|
|
|
// // Create DB record
|
|
// cover = new MangaCover
|
|
// {
|
|
// Manga = sc.MangaSource.Manga,
|
|
// Guid = guid,
|
|
// FileExtension = ext,
|
|
// Sha256 = sha256,
|
|
// Width = image.Width,
|
|
// Height = image.Height,
|
|
// IsPrimary = false
|
|
// };
|
|
|
|
// context.MangaCovers.Add(cover);
|
|
// }
|
|
|
|
// // Link and mark done
|
|
// sourceCover.MangaCover = cover;
|
|
// //sc.Status = CoverDownloadStatus.Done;
|
|
// //sc.Error = null;
|
|
|
|
// await context.SaveChangesAsync(cancellationToken);
|
|
//}
|
|
|
|
public async Task RunPagesAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken)
|
|
{
|
|
SourceChapter? sourceChapter = await context.SourceChapters.FirstOrDefaultAsync(x => x.SourceChapterId == request.SourceChapterId, cancellationToken);
|
|
|
|
if (sourceChapter == null)
|
|
return;
|
|
|
|
int currentPageNumber = 1;
|
|
|
|
foreach (string pageImageUrl in request.PageImageUrls)
|
|
{
|
|
await AddOrUpdateSourcePageAsync(sourceChapter, currentPageNumber++, pageImageUrl, cancellationToken);
|
|
}
|
|
}
|
|
|
|
private async Task AddOrUpdateSourcePageAsync(SourceChapter sourceChapter, int pageNumber, string pageImageUrl, CancellationToken cancellationToken)
|
|
{
|
|
SourcePage? sourcePage = await context.SourcePages.FirstOrDefaultAsync(x =>
|
|
x.Chapter == sourceChapter && x.PageNumber == pageNumber, cancellationToken);
|
|
|
|
if (sourcePage == null)
|
|
{
|
|
sourcePage = new()
|
|
{
|
|
Chapter = sourceChapter,
|
|
PageNumber = pageNumber,
|
|
Url = pageImageUrl
|
|
};
|
|
|
|
await context.SourcePages.AddAsync(sourcePage, cancellationToken);
|
|
}
|
|
else
|
|
{
|
|
sourcePage.Url = pageImageUrl;
|
|
}
|
|
}
|
|
} |