From f760cff21f1de2d3ffd36df948c6808257307b79 Mon Sep 17 00:00:00 2001 From: Brian Bicknell Date: Fri, 23 May 2025 02:40:06 -0400 Subject: [PATCH] Added manga data and pipeline. --- MangaReader.Core/Data/ChapterPage.cs | 11 + MangaReader.Core/Data/ChapterSource.cs | 12 + MangaReader.Core/Data/Genre.cs | 7 + MangaReader.Core/Data/Manga.cs | 298 +----------------- MangaReader.Core/Data/MangaChapter.cs | 16 + MangaReader.Core/Data/MangaContext.cs | 193 ++++++++++++ MangaReader.Core/Data/MangaCover.cs | 14 + MangaReader.Core/Data/MangaGenre.cs | 10 + MangaReader.Core/Data/MangaSource.cs | 12 + MangaReader.Core/Data/MangaTitle.cs | 12 + MangaReader.Core/Data/Source.cs | 7 + MangaReader.Core/Data/TitleType.cs | 12 + MangaReader.Core/MangaReader.Core.csproj | 2 +- MangaReader.Core/Pipeline/IMangaPipeline.cs | 8 + MangaReader.Core/Pipeline/MangaPipeline.cs | 144 +++++++++ .../IMangaSearchProvider.cs} | 4 +- .../MangaSearchProviderBase.cs} | 4 +- .../MangaSearchResult.cs | 3 +- .../NatoManga/NatoMangaSearchProvider.cs} | 5 +- .../NatoManga/NatoMangaSearchResult.cs | 2 +- .../WebCrawlers/IMangaWebCrawler.cs | 2 +- .../MangaNato/MangaNatoWebCrawler.cs | 10 +- .../WebCrawlers/MangaWebCrawler.cs | 2 +- .../NatoManga/NatoMangaWebCrawler.cs | 10 +- .../{MangaDTO.cs => SourceManga.cs} | 4 +- ...ngaChapterDTO.cs => SourceMangaChapter.cs} | 4 +- .../NatoManga/NatoMangaWebSearchTests.cs | 8 +- 27 files changed, 490 insertions(+), 326 deletions(-) create mode 100644 MangaReader.Core/Data/ChapterPage.cs create mode 100644 MangaReader.Core/Data/ChapterSource.cs create mode 100644 MangaReader.Core/Data/Genre.cs create mode 100644 MangaReader.Core/Data/MangaChapter.cs create mode 100644 MangaReader.Core/Data/MangaContext.cs create mode 100644 MangaReader.Core/Data/MangaCover.cs create mode 100644 MangaReader.Core/Data/MangaGenre.cs create mode 100644 MangaReader.Core/Data/MangaSource.cs create mode 100644 MangaReader.Core/Data/MangaTitle.cs create mode 100644 MangaReader.Core/Data/Source.cs create mode 100644 MangaReader.Core/Data/TitleType.cs create mode 100644 MangaReader.Core/Pipeline/IMangaPipeline.cs create mode 100644 MangaReader.Core/Pipeline/MangaPipeline.cs rename MangaReader.Core/{WebSearch/IMangaWebSearch.cs => Search/IMangaSearchProvider.cs} (70%) rename MangaReader.Core/{WebSearch/MangaWebSearchBase.cs => Search/MangaSearchProviderBase.cs} (86%) rename MangaReader.Core/{WebSearch => Search}/MangaSearchResult.cs (71%) rename MangaReader.Core/{WebSearch/NatoManga/NatoMangaWebSearch.cs => Search/NatoManga/NatoMangaSearchProvider.cs} (76%) rename MangaReader.Core/{WebSearch => Search}/NatoManga/NatoMangaSearchResult.cs (86%) rename MangaReader.Core/WebCrawlers/{MangaDTO.cs => SourceManga.cs} (85%) rename MangaReader.Core/WebCrawlers/{MangaChapterDTO.cs => SourceMangaChapter.cs} (75%) diff --git a/MangaReader.Core/Data/ChapterPage.cs b/MangaReader.Core/Data/ChapterPage.cs new file mode 100644 index 0000000..932f759 --- /dev/null +++ b/MangaReader.Core/Data/ChapterPage.cs @@ -0,0 +1,11 @@ +namespace MangaReader.Core.Data; + +public class ChapterPage +{ + public int ChapterPageId { get; set; } + + public int MangaChapterId { get; set; } + public required MangaChapter MangaChapter { get; set; } + + public int PageNumber { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/ChapterSource.cs b/MangaReader.Core/Data/ChapterSource.cs new file mode 100644 index 0000000..89285c5 --- /dev/null +++ b/MangaReader.Core/Data/ChapterSource.cs @@ -0,0 +1,12 @@ +namespace MangaReader.Core.Data; + +public class ChapterSource +{ + public int MangaChapterId { get; set; } + public required MangaChapter Chapter { get; set; } + + public int SourceId { get; set; } + public required Source Source { get; set; } + + public required string Url { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/Genre.cs b/MangaReader.Core/Data/Genre.cs new file mode 100644 index 0000000..0de23ba --- /dev/null +++ b/MangaReader.Core/Data/Genre.cs @@ -0,0 +1,7 @@ +namespace MangaReader.Core.Data; + +public class Genre +{ + public int GenreId { get; set; } + public required string Name { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/Manga.cs b/MangaReader.Core/Data/Manga.cs index cc69fa9..0245cff 100644 --- a/MangaReader.Core/Data/Manga.cs +++ b/MangaReader.Core/Data/Manga.cs @@ -1,6 +1,4 @@ -using Microsoft.EntityFrameworkCore; - -namespace MangaReader.Core.Data; +namespace MangaReader.Core.Data; public class Manga { @@ -23,298 +21,4 @@ public class Manga Genres = new HashSet(); Chapters = new HashSet(); } -} - -public class MangaCover -{ - public int MangaCoverId { get; set; } - - public int MangaId { get; set; } - public required Manga Manga { get; set; } - - public required Guid Guid { get; set; } - public required string FileExtension { get; set; } - public string? Description { get; set; } - public bool IsPrimary { get; set; } -} - -public class MangaTitle -{ - public int MangaTitleId { get; set; } - - public int MangaId { get; set; } - public required Manga Manga { get; set; } - - public required string TitleEntry { get; set; } - public TitleType TitleType { get; set; } -} - -public enum TitleType -{ - Primary, - OfficialTranslation, - FanTranslation, - Synonym, - Abbreviation, - Romaji, - Japanese -} - -public class Source -{ - public int SourceId { get; set; } - public required string Name { get; set; } -} - -public class MangaSource -{ - public int MangaId { get; set; } - public required Manga Manga { get; set; } - - public int SourceId { get; set; } - public required Source Source { get; set; } - - public required string Url { get; set; } -} - -public class Genre -{ - public int GenreId { get; set; } - public required string Name { get; set; } -} - -public class MangaGenre -{ - public int MangaId { get; set; } - public required Manga Manga { get; set; } - - public int GenreId { get; set; } - public required Genre Genre { get; set; } -} - -public class MangaChapter -{ - public int MangaChapterId { get; set; } - - public int MangaId { get; set; } - public required Manga Manga { get; set; } - - public int? VolumeNumber { get; set; } - public int ChapterNumber { get; set; } - public string? Title { get; set; } - - public virtual ICollection Sources { get; set; } = []; - public virtual ICollection Pages { get; set; } = []; -} - -public class ChapterSource -{ - public int MangaChapterId { get; set; } - public required MangaChapter Chapter { get; set; } - - public int SourceId { get; set; } - public required Source Source { get; set; } - - public required string Url { get; set; } -} - -public class ChapterPage -{ - public int ChapterPageId { get; set; } - - public int MangaChapterId { get; set; } - public required MangaChapter MangaChapter { get; set; } - - public int PageNumber { get; set; } -} - - -public class MangaContext(DbContextOptions options) : DbContext(options) -{ - public DbSet Mangas { get; set; } - public DbSet MangaCovers { get; set; } - public DbSet MangaTitles { get; set; } - public DbSet Sources { get; set; } - public DbSet MangaSources { get; set; } - public DbSet Genres { get; set; } - public DbSet MangaGenres { get; set; } - public DbSet MangaChapters { get; set; } - public DbSet ChapterSources { get; set; } - public DbSet ChapterPages { get; set; } - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - base.OnModelCreating(modelBuilder); - - ConfigureManga(modelBuilder); - ConfigureMangaCover(modelBuilder); - ConfigureMangaTitle(modelBuilder); - ConfigureSource(modelBuilder); - ConfigureMangaSource(modelBuilder); - ConfigureGenre(modelBuilder); - ConfigureMangaGenre(modelBuilder); - ConfigureMangaChapter(modelBuilder); - ConfigureChapterSource(modelBuilder); - ConfigureChapterPage(modelBuilder); - } - - private static void ConfigureManga(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(x => x.MangaId); - - modelBuilder.Entity() - .HasIndex(x => x.Slug) - .IsUnique(); - } - - private static void ConfigureMangaCover(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(x => x.MangaCoverId); - - modelBuilder - .Entity() - .HasOne(x => x.Manga) - .WithMany(x => x.Covers) - .HasForeignKey(x => x.MangaId) - .OnDelete(DeleteBehavior.Cascade); - - modelBuilder.Entity() - .HasIndex(x => x.Guid) - .IsUnique(); - - //modelBuilder - // .Entity() - // .HasIndex(x => new { x.MangaId, x.IsPrimary }) - // .IsUnique() - // .HasFilter("[IsPrimary] = 1"); // Enforce only one primary cover per manga - } - - private static void ConfigureMangaTitle(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(mangaTitle => mangaTitle.MangaTitleId); - - modelBuilder.Entity() - .HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.TitleEntry }) - .IsUnique(); - - modelBuilder - .Entity() - .HasIndex(mangaTitle => mangaTitle.TitleEntry); - - modelBuilder - .Entity() - .HasOne(x => x.Manga) - .WithMany(x => x.Titles) - .HasForeignKey(x => x.MangaId) - .OnDelete(DeleteBehavior.Cascade); - } - - private static void ConfigureSource(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(x => x.SourceId); - - modelBuilder - .Entity() - .HasIndex(x => x.Name) - .IsUnique(true); - } - - private static void ConfigureMangaSource(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(mangaSource => new { mangaSource.MangaId, mangaSource.SourceId }); - - modelBuilder.Entity() - .HasIndex(x => x.Url) - .IsUnique(); - - modelBuilder - .Entity() - .HasOne(x => x.Manga) - .WithMany(x => x.Sources) - .HasForeignKey(x => x.MangaId) - .OnDelete(DeleteBehavior.Cascade); - } - - private static void ConfigureGenre(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(x => x.GenreId); - - modelBuilder - .Entity() - .HasIndex(x => x.Name) - .IsUnique(true); - } - - private static void ConfigureMangaGenre(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(mangaGenre => new { mangaGenre.MangaId, mangaGenre.GenreId }); - - modelBuilder - .Entity() - .HasOne(x => x.Manga) - .WithMany(x => x.Genres) - .HasForeignKey(x => x.MangaId) - .OnDelete(DeleteBehavior.Cascade); - } - - private static void ConfigureMangaChapter(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(x => x.MangaChapterId); - - modelBuilder - .Entity() - .HasOne(x => x.Manga) - .WithMany(x => x.Chapters) - .HasForeignKey(x => x.MangaId) - .OnDelete(DeleteBehavior.Cascade); - } - - private static void ConfigureChapterSource(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(chapterSource => new { chapterSource.MangaChapterId, chapterSource.SourceId }); - - modelBuilder - .Entity() - .HasOne(x => x.Chapter) - .WithMany(x => x.Sources) - .HasForeignKey(x => x.MangaChapterId) - .OnDelete(DeleteBehavior.Cascade); - } - - private static void ConfigureChapterPage(ModelBuilder modelBuilder) - { - modelBuilder - .Entity() - .HasKey(chapterPage => chapterPage.ChapterPageId); - - modelBuilder - .Entity() - .HasIndex(chapterPage => new { chapterPage.MangaChapterId, chapterPage.PageNumber }) - .IsUnique(true); - - modelBuilder - .Entity() - .HasOne(x => x.MangaChapter) - .WithMany(x => x.Pages) - .HasForeignKey(x => x.MangaChapterId) - .OnDelete(DeleteBehavior.Cascade); - } } \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaChapter.cs b/MangaReader.Core/Data/MangaChapter.cs new file mode 100644 index 0000000..a2f9d38 --- /dev/null +++ b/MangaReader.Core/Data/MangaChapter.cs @@ -0,0 +1,16 @@ +namespace MangaReader.Core.Data; + +public class MangaChapter +{ + public int MangaChapterId { get; set; } + + public int MangaId { get; set; } + public required Manga Manga { get; set; } + + public int? VolumeNumber { get; set; } + public required float ChapterNumber { get; set; } + public string? Title { get; set; } + + public virtual ICollection Sources { get; set; } = []; + public virtual ICollection Pages { get; set; } = []; +} \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaContext.cs b/MangaReader.Core/Data/MangaContext.cs new file mode 100644 index 0000000..45c5562 --- /dev/null +++ b/MangaReader.Core/Data/MangaContext.cs @@ -0,0 +1,193 @@ +using Microsoft.EntityFrameworkCore; + +namespace MangaReader.Core.Data; + +public class MangaContext(DbContextOptions options) : DbContext(options) +{ + public DbSet Mangas { get; set; } + public DbSet MangaCovers { get; set; } + public DbSet MangaTitles { get; set; } + public DbSet Sources { get; set; } + public DbSet MangaSources { get; set; } + public DbSet Genres { get; set; } + public DbSet MangaGenres { get; set; } + public DbSet MangaChapters { get; set; } + public DbSet ChapterSources { get; set; } + public DbSet ChapterPages { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + ConfigureManga(modelBuilder); + ConfigureMangaCover(modelBuilder); + ConfigureMangaTitle(modelBuilder); + ConfigureSource(modelBuilder); + ConfigureMangaSource(modelBuilder); + ConfigureGenre(modelBuilder); + ConfigureMangaGenre(modelBuilder); + ConfigureMangaChapter(modelBuilder); + ConfigureChapterSource(modelBuilder); + ConfigureChapterPage(modelBuilder); + } + + private static void ConfigureManga(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(x => x.MangaId); + + modelBuilder.Entity() + .HasIndex(x => x.Slug) + .IsUnique(); + } + + private static void ConfigureMangaCover(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(x => x.MangaCoverId); + + modelBuilder + .Entity() + .HasOne(x => x.Manga) + .WithMany(x => x.Covers) + .HasForeignKey(x => x.MangaId) + .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .HasIndex(x => x.Guid) + .IsUnique(); + + //modelBuilder + // .Entity() + // .HasIndex(x => new { x.MangaId, x.IsPrimary }) + // .IsUnique() + // .HasFilter("[IsPrimary] = 1"); // Enforce only one primary cover per manga + } + + private static void ConfigureMangaTitle(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(mangaTitle => mangaTitle.MangaTitleId); + + modelBuilder.Entity() + .HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.TitleEntry }) + .IsUnique(); + + modelBuilder + .Entity() + .HasIndex(mangaTitle => mangaTitle.TitleEntry); + + modelBuilder + .Entity() + .HasOne(x => x.Manga) + .WithMany(x => x.Titles) + .HasForeignKey(x => x.MangaId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureSource(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(x => x.SourceId); + + modelBuilder + .Entity() + .HasIndex(x => x.Name) + .IsUnique(true); + } + + private static void ConfigureMangaSource(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(mangaSource => new { mangaSource.MangaId, mangaSource.SourceId }); + + modelBuilder.Entity() + .HasIndex(x => x.Url) + .IsUnique(); + + modelBuilder + .Entity() + .HasOne(x => x.Manga) + .WithMany(x => x.Sources) + .HasForeignKey(x => x.MangaId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureGenre(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(x => x.GenreId); + + modelBuilder + .Entity() + .HasIndex(x => x.Name) + .IsUnique(true); + } + + private static void ConfigureMangaGenre(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(mangaGenre => new { mangaGenre.MangaId, mangaGenre.GenreId }); + + modelBuilder + .Entity() + .HasOne(x => x.Manga) + .WithMany(x => x.Genres) + .HasForeignKey(x => x.MangaId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureMangaChapter(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(x => x.MangaChapterId); + + modelBuilder + .Entity() + .HasOne(x => x.Manga) + .WithMany(x => x.Chapters) + .HasForeignKey(x => x.MangaId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureChapterSource(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(chapterSource => new { chapterSource.MangaChapterId, chapterSource.SourceId }); + + modelBuilder + .Entity() + .HasOne(x => x.Chapter) + .WithMany(x => x.Sources) + .HasForeignKey(x => x.MangaChapterId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureChapterPage(ModelBuilder modelBuilder) + { + modelBuilder + .Entity() + .HasKey(chapterPage => chapterPage.ChapterPageId); + + modelBuilder + .Entity() + .HasIndex(chapterPage => new { chapterPage.MangaChapterId, chapterPage.PageNumber }) + .IsUnique(true); + + modelBuilder + .Entity() + .HasOne(x => x.MangaChapter) + .WithMany(x => x.Pages) + .HasForeignKey(x => x.MangaChapterId) + .OnDelete(DeleteBehavior.Cascade); + } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaCover.cs b/MangaReader.Core/Data/MangaCover.cs new file mode 100644 index 0000000..fa254e8 --- /dev/null +++ b/MangaReader.Core/Data/MangaCover.cs @@ -0,0 +1,14 @@ +namespace MangaReader.Core.Data; + +public class MangaCover +{ + public int MangaCoverId { get; set; } + + public int MangaId { get; set; } + public required Manga Manga { get; set; } + + public required Guid Guid { get; set; } + public required string FileExtension { get; set; } + public string? Description { get; set; } + public bool IsPrimary { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaGenre.cs b/MangaReader.Core/Data/MangaGenre.cs new file mode 100644 index 0000000..d279b6e --- /dev/null +++ b/MangaReader.Core/Data/MangaGenre.cs @@ -0,0 +1,10 @@ +namespace MangaReader.Core.Data; + +public class MangaGenre +{ + public int MangaId { get; set; } + public required Manga Manga { get; set; } + + public int GenreId { get; set; } + public required Genre Genre { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaSource.cs b/MangaReader.Core/Data/MangaSource.cs new file mode 100644 index 0000000..380b519 --- /dev/null +++ b/MangaReader.Core/Data/MangaSource.cs @@ -0,0 +1,12 @@ +namespace MangaReader.Core.Data; + +public class MangaSource +{ + public int MangaId { get; set; } + public required Manga Manga { get; set; } + + public int SourceId { get; set; } + public required Source Source { get; set; } + + public required string Url { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/MangaTitle.cs b/MangaReader.Core/Data/MangaTitle.cs new file mode 100644 index 0000000..dff039f --- /dev/null +++ b/MangaReader.Core/Data/MangaTitle.cs @@ -0,0 +1,12 @@ +namespace MangaReader.Core.Data; + +public class MangaTitle +{ + public int MangaTitleId { get; set; } + + public int MangaId { get; set; } + public required Manga Manga { get; set; } + + public required string TitleEntry { get; set; } + public TitleType TitleType { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/Source.cs b/MangaReader.Core/Data/Source.cs new file mode 100644 index 0000000..93a9191 --- /dev/null +++ b/MangaReader.Core/Data/Source.cs @@ -0,0 +1,7 @@ +namespace MangaReader.Core.Data; + +public class Source +{ + public int SourceId { get; set; } + public required string Name { get; set; } +} \ No newline at end of file diff --git a/MangaReader.Core/Data/TitleType.cs b/MangaReader.Core/Data/TitleType.cs new file mode 100644 index 0000000..3f13274 --- /dev/null +++ b/MangaReader.Core/Data/TitleType.cs @@ -0,0 +1,12 @@ +namespace MangaReader.Core.Data; + +public enum TitleType +{ + Primary, + OfficialTranslation, + FanTranslation, + Synonym, + Abbreviation, + Romaji, + Japanese +} \ No newline at end of file diff --git a/MangaReader.Core/MangaReader.Core.csproj b/MangaReader.Core/MangaReader.Core.csproj index 32dae53..0aa01e1 100644 --- a/MangaReader.Core/MangaReader.Core.csproj +++ b/MangaReader.Core/MangaReader.Core.csproj @@ -12,7 +12,7 @@ - + diff --git a/MangaReader.Core/Pipeline/IMangaPipeline.cs b/MangaReader.Core/Pipeline/IMangaPipeline.cs new file mode 100644 index 0000000..789d24c --- /dev/null +++ b/MangaReader.Core/Pipeline/IMangaPipeline.cs @@ -0,0 +1,8 @@ +using MangaReader.Core.WebCrawlers; + +namespace MangaReader.Core.Pipeline; + +public interface IMangaPipeline +{ + Task RunAsync(SourceManga mangaDto); +} \ No newline at end of file diff --git a/MangaReader.Core/Pipeline/MangaPipeline.cs b/MangaReader.Core/Pipeline/MangaPipeline.cs new file mode 100644 index 0000000..a1144a3 --- /dev/null +++ b/MangaReader.Core/Pipeline/MangaPipeline.cs @@ -0,0 +1,144 @@ +using MangaReader.Core.Data; +using MangaReader.Core.WebCrawlers; +using Microsoft.EntityFrameworkCore; +using System.Text.RegularExpressions; + +namespace MangaReader.Core.Pipeline; + +public partial class MangaPipeline(MangaContext context) : IMangaPipeline +{ + public async Task RunAsync(SourceManga sourceManga) + { + Manga manga = await GetOrAddMangaAsync(sourceManga); + + foreach (string alternateTitle in sourceManga.AlternateTitles) + { + await AddTitleAsync(manga, alternateTitle); + } + + foreach (string genre in sourceManga.Genres) + { + await LinkGenreAsync(manga, genre); + } + + foreach (SourceMangaChapter chapter in sourceManga.Chapters) + { + await AddChapterAsync(manga, chapter); + } + + context.SaveChanges(); + } + + private async Task GetOrAddMangaAsync(SourceManga sourceManga) + { + Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => manga.Title == sourceManga.Title); + + if (manga != null) + return manga; + + manga = new() + { + Title = sourceManga.Title, + Slug = GenerateSlug(sourceManga.Title), + }; + + context.Add(manga); + + return manga; + } + + private static string GenerateSlug(string title) + { + title = title.ToLowerInvariant(); + title = RemoveInvalidCharsRegex().Replace(title, ""); // remove invalid chars + title = RemoveSpacesWithDashRegex().Replace(title, "-"); // replace spaces with dash + + return title.Trim('-'); + } + + [GeneratedRegex(@"[^a-z0-9\s-]")] + private static partial Regex RemoveInvalidCharsRegex(); + + [GeneratedRegex(@"\s+")] + private static partial Regex RemoveSpacesWithDashRegex(); + + private async Task AddTitleAsync(Manga manga, string title) + { + MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt => mt.Manga == manga && mt.TitleEntry == title); + + if (mangaTitle != null) + return; + + mangaTitle = new() + { + Manga = manga, + TitleEntry = title, + }; + + context.MangaTitles.Add(mangaTitle); + } + + private async Task LinkGenreAsync(Manga manga, string genreName) + { + Genre genre = await GetOrAddGenreAsync(genreName); + + MangaGenre? mangaGenre = await context.MangaGenres.FirstOrDefaultAsync(x => x.Manga == manga && x.Genre == genre); + + if (mangaGenre != null) + return; + + mangaGenre = new() + { + Manga = manga, + Genre = genre + }; + + context.MangaGenres.Add(mangaGenre); + } + + private async Task GetOrAddGenreAsync(string genreName) + { + Genre? genre = await context.Genres.FirstOrDefaultAsync(x => x.Name == genreName); + + if (genre != null) + return genre; + + genre = new() + { + Name = genreName, + }; + + await context.Genres.AddAsync(genre); + + return genre; + } + + private async Task AddChapterAsync(Manga manga, SourceMangaChapter sourceeMangaChapter) + { + MangaChapter mangaChapter = await context.MangaChapters.FirstOrDefaultAsync(x => x.ChapterNumber == sourceeMangaChapter.Number) + ?? AddMangaChapter(manga, sourceeMangaChapter); + + if (mangaChapter.VolumeNumber is null && sourceeMangaChapter.Volume is not null) + { + mangaChapter.VolumeNumber = sourceeMangaChapter.Volume; + } + + if (mangaChapter.Title is null && sourceeMangaChapter.Name is not null) + { + mangaChapter.Title = sourceeMangaChapter.Name; + } + } + + private MangaChapter AddMangaChapter(Manga manga, SourceMangaChapter sourceeMangaChapter) + { + MangaChapter mangaChapter = new() + { + Manga = manga, + ChapterNumber = sourceeMangaChapter.Number + }; + + context.MangaChapters.Add(mangaChapter); + + return mangaChapter; + } +} diff --git a/MangaReader.Core/WebSearch/IMangaWebSearch.cs b/MangaReader.Core/Search/IMangaSearchProvider.cs similarity index 70% rename from MangaReader.Core/WebSearch/IMangaWebSearch.cs rename to MangaReader.Core/Search/IMangaSearchProvider.cs index 446f368..6cfd7f4 100644 --- a/MangaReader.Core/WebSearch/IMangaWebSearch.cs +++ b/MangaReader.Core/Search/IMangaSearchProvider.cs @@ -1,6 +1,6 @@ -namespace MangaReader.Core.WebSearch; +namespace MangaReader.Core.Search; -public interface IMangaWebSearch +public interface IMangaSearchProvider { Task SearchAsync(string keyword); } diff --git a/MangaReader.Core/WebSearch/MangaWebSearchBase.cs b/MangaReader.Core/Search/MangaSearchProviderBase.cs similarity index 86% rename from MangaReader.Core/WebSearch/MangaWebSearchBase.cs rename to MangaReader.Core/Search/MangaSearchProviderBase.cs index ce4f739..92348b3 100644 --- a/MangaReader.Core/WebSearch/MangaWebSearchBase.cs +++ b/MangaReader.Core/Search/MangaSearchProviderBase.cs @@ -1,9 +1,9 @@ using MangaReader.Core.HttpService; using System.Text.Json; -namespace MangaReader.Core.WebSearch; +namespace MangaReader.Core.Search; -public abstract class MangaWebSearchBase(IHttpService httpService) : IMangaWebSearch +public abstract class MangaSearchProviderBase(IHttpService httpService) : IMangaSearchProvider { private static JsonSerializerOptions _jsonSerializerOptions = new() { diff --git a/MangaReader.Core/WebSearch/MangaSearchResult.cs b/MangaReader.Core/Search/MangaSearchResult.cs similarity index 71% rename from MangaReader.Core/WebSearch/MangaSearchResult.cs rename to MangaReader.Core/Search/MangaSearchResult.cs index e144c4c..bb71a1d 100644 --- a/MangaReader.Core/WebSearch/MangaSearchResult.cs +++ b/MangaReader.Core/Search/MangaSearchResult.cs @@ -1,7 +1,8 @@ -namespace MangaReader.Core.WebSearch; +namespace MangaReader.Core.Search; public record MangaSearchResult { + public required string Source { get; init; } public required string Url { get; init; } public required string Title { get; init; } public string? Author { get; init; } diff --git a/MangaReader.Core/WebSearch/NatoManga/NatoMangaWebSearch.cs b/MangaReader.Core/Search/NatoManga/NatoMangaSearchProvider.cs similarity index 76% rename from MangaReader.Core/WebSearch/NatoManga/NatoMangaWebSearch.cs rename to MangaReader.Core/Search/NatoManga/NatoMangaSearchProvider.cs index 2e14834..d578ce8 100644 --- a/MangaReader.Core/WebSearch/NatoManga/NatoMangaWebSearch.cs +++ b/MangaReader.Core/Search/NatoManga/NatoMangaSearchProvider.cs @@ -1,8 +1,8 @@ using MangaReader.Core.HttpService; -namespace MangaReader.Core.WebSearch.NatoManga; +namespace MangaReader.Core.Search.NatoManga; -public class NatoMangaWebSearch(IHttpService httpService) : MangaWebSearchBase(httpService) +public class NatoMangaSearchProvider(IHttpService httpService) : MangaSearchProviderBase(httpService) { // https://www.natomanga.com/home/search/json?searchword=gal_can_t_be_kind @@ -16,6 +16,7 @@ public class NatoMangaWebSearch(IHttpService httpService) : MangaWebSearchBase mangaSearchResults = searchResult.Select(searchResult => new MangaSearchResult() { + Source = "NatoManga", Title = searchResult.Name, Url = searchResult.Url }); diff --git a/MangaReader.Core/WebSearch/NatoManga/NatoMangaSearchResult.cs b/MangaReader.Core/Search/NatoManga/NatoMangaSearchResult.cs similarity index 86% rename from MangaReader.Core/WebSearch/NatoManga/NatoMangaSearchResult.cs rename to MangaReader.Core/Search/NatoManga/NatoMangaSearchResult.cs index 91feddf..427d432 100644 --- a/MangaReader.Core/WebSearch/NatoManga/NatoMangaSearchResult.cs +++ b/MangaReader.Core/Search/NatoManga/NatoMangaSearchResult.cs @@ -1,4 +1,4 @@ -namespace MangaReader.Core.WebSearch.NatoManga; +namespace MangaReader.Core.Search.NatoManga; public record NatoMangaSearchResult { diff --git a/MangaReader.Core/WebCrawlers/IMangaWebCrawler.cs b/MangaReader.Core/WebCrawlers/IMangaWebCrawler.cs index 0352795..46c00c4 100644 --- a/MangaReader.Core/WebCrawlers/IMangaWebCrawler.cs +++ b/MangaReader.Core/WebCrawlers/IMangaWebCrawler.cs @@ -2,5 +2,5 @@ public interface IMangaWebCrawler { - MangaDTO GetManga(string url); + SourceManga GetManga(string url); } \ No newline at end of file diff --git a/MangaReader.Core/WebCrawlers/MangaNato/MangaNatoWebCrawler.cs b/MangaReader.Core/WebCrawlers/MangaNato/MangaNatoWebCrawler.cs index 9351d20..479d885 100644 --- a/MangaReader.Core/WebCrawlers/MangaNato/MangaNatoWebCrawler.cs +++ b/MangaReader.Core/WebCrawlers/MangaNato/MangaNatoWebCrawler.cs @@ -6,12 +6,12 @@ namespace MangaReader.Core.WebCrawlers.MangaNato; public class MangaNatoWebCrawler : MangaWebCrawler { - public override MangaDTO GetManga(string url) + public override SourceManga GetManga(string url) { HtmlDocument document = GetHtmlDocument(url); MangaNatoMangaDocument node = new(document); - MangaDTO manga = new() + SourceManga manga = new() { Title = node.TitleNode.InnerText, AlternateTitles = GetAlternateTitles(node.AlternateTitlesNode), @@ -101,9 +101,9 @@ public class MangaNatoWebCrawler : MangaWebCrawler return (int)Math.Round(average / best * 100); } - private static List GetChapters(HtmlNodeCollection chapterNodes) + private static List GetChapters(HtmlNodeCollection chapterNodes) { - List chapters = []; + List chapters = []; foreach (var node in chapterNodes) { @@ -111,7 +111,7 @@ public class MangaNatoWebCrawler : MangaWebCrawler HtmlNode chapterViewNode = node.SelectSingleNode(".//span[contains(@class, 'chapter-view')]"); HtmlNode chapterTimeNode = node.SelectSingleNode(".//span[contains(@class, 'chapter-time')]"); - MangaChapterDTO chapter = new() + SourceMangaChapter chapter = new() { Number = GetChapterNumber(chapterNameNode), Name = chapterNameNode.InnerText, diff --git a/MangaReader.Core/WebCrawlers/MangaWebCrawler.cs b/MangaReader.Core/WebCrawlers/MangaWebCrawler.cs index 8fa9d13..31c2d78 100644 --- a/MangaReader.Core/WebCrawlers/MangaWebCrawler.cs +++ b/MangaReader.Core/WebCrawlers/MangaWebCrawler.cs @@ -4,7 +4,7 @@ namespace MangaReader.Core.WebCrawlers; public abstract class MangaWebCrawler : IMangaWebCrawler { - public abstract MangaDTO GetManga(string url); + public abstract SourceManga GetManga(string url); protected virtual HtmlDocument GetHtmlDocument(string url) { diff --git a/MangaReader.Core/WebCrawlers/NatoManga/NatoMangaWebCrawler.cs b/MangaReader.Core/WebCrawlers/NatoManga/NatoMangaWebCrawler.cs index 0f1b484..709c143 100644 --- a/MangaReader.Core/WebCrawlers/NatoManga/NatoMangaWebCrawler.cs +++ b/MangaReader.Core/WebCrawlers/NatoManga/NatoMangaWebCrawler.cs @@ -6,12 +6,12 @@ namespace MangaReader.Core.WebCrawlers.NatoManga; public class NatoMangaWebCrawler : MangaWebCrawler { - public override MangaDTO GetManga(string url) + public override SourceManga GetManga(string url) { HtmlDocument document = GetHtmlDocument(url); NatoMangaHtmlDocument node = new(document); - MangaDTO manga = new() + SourceManga manga = new() { Title = node.TitleNode?.InnerText ?? string.Empty, //AlternateTitles = GetAlternateTitles(node.AlternateTitlesNode), @@ -116,9 +116,9 @@ public class NatoMangaWebCrawler : MangaWebCrawler return (int)Math.Round(average / best * 100); } - private static List GetChapters(HtmlNodeCollection? chapterNodes) + private static List GetChapters(HtmlNodeCollection? chapterNodes) { - List chapters = []; + List chapters = []; if (chapterNodes == null) return chapters; @@ -137,7 +137,7 @@ public class NatoMangaWebCrawler : MangaWebCrawler if (chapterNameNode == null) continue; - MangaChapterDTO chapter = new() + SourceMangaChapter chapter = new() { Number = GetChapterNumber(chapterNameNode), Name = chapterNameNode.InnerText, diff --git a/MangaReader.Core/WebCrawlers/MangaDTO.cs b/MangaReader.Core/WebCrawlers/SourceManga.cs similarity index 85% rename from MangaReader.Core/WebCrawlers/MangaDTO.cs rename to MangaReader.Core/WebCrawlers/SourceManga.cs index aa51208..19e2663 100644 --- a/MangaReader.Core/WebCrawlers/MangaDTO.cs +++ b/MangaReader.Core/WebCrawlers/SourceManga.cs @@ -1,6 +1,6 @@ namespace MangaReader.Core.WebCrawlers; -public class MangaDTO +public class SourceManga { public required string Title { get; set; } public string? Description { get; set; } @@ -12,5 +12,5 @@ public class MangaDTO public long? Views { get; set; } public float? RatingPercent { get; set; } public int? Votes { get; set; } - public List Chapters { get; set; } = []; + public List Chapters { get; set; } = []; } \ No newline at end of file diff --git a/MangaReader.Core/WebCrawlers/MangaChapterDTO.cs b/MangaReader.Core/WebCrawlers/SourceMangaChapter.cs similarity index 75% rename from MangaReader.Core/WebCrawlers/MangaChapterDTO.cs rename to MangaReader.Core/WebCrawlers/SourceMangaChapter.cs index ed72876..8351444 100644 --- a/MangaReader.Core/WebCrawlers/MangaChapterDTO.cs +++ b/MangaReader.Core/WebCrawlers/SourceMangaChapter.cs @@ -1,9 +1,9 @@ namespace MangaReader.Core.WebCrawlers; -public class MangaChapterDTO +public class SourceMangaChapter { public int? Volume { get; set; } - public float? Number { get; set; } + public required float Number { get; set; } public string? Name { get; set; } public required string Url { get; set; } public long? Views { get; set; } diff --git a/MangaReader.Tests/WebSearch/NatoManga/NatoMangaWebSearchTests.cs b/MangaReader.Tests/WebSearch/NatoManga/NatoMangaWebSearchTests.cs index 8a4802e..194d303 100644 --- a/MangaReader.Tests/WebSearch/NatoManga/NatoMangaWebSearchTests.cs +++ b/MangaReader.Tests/WebSearch/NatoManga/NatoMangaWebSearchTests.cs @@ -1,6 +1,6 @@ using MangaReader.Core.HttpService; -using MangaReader.Core.WebSearch; -using MangaReader.Core.WebSearch.NatoManga; +using MangaReader.Core.Search; +using MangaReader.Core.Search.NatoManga; using MangaReader.Tests.Utilities; using NSubstitute; using Shouldly; @@ -20,8 +20,8 @@ public class NatoMangaWebSearchTests httpService.GetStringAsync(Arg.Any()) .Returns(Task.FromResult(searchResultJson)); - NatoMangaWebSearch webSearch = new(httpService); - MangaSearchResult[] searchResult = await webSearch.SearchAsync("Gals Can't Be Kind"); + NatoMangaSearchProvider searchProvider = new(httpService); + MangaSearchResult[] searchResult = await searchProvider.SearchAsync("Gals Can't Be Kind"); searchResult.Length.ShouldBe(2); searchResult[0].Title.ShouldBe("Gal Can't Be Kind to Otaku!");