brister/manga-reader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added abstraction layeer IHtmlLoader. Finished reorganizing test project folder structure.

Browse Source
This commit is contained in:
Brian Bicknell
2025-06-09 00:09:59 -04:00
parent b5d22c3c7e
commit c26ed11bfc
30 changed files with 1966 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
MangaReader.Core/Data/Manga.cs
View File

@@ -4,11 +4,10 @@ public class Manga
{
public int MangaId { get; set; }
public required string Slug { get; set; }
public required string Title { get; set; }
public string? Description { get; set; }
public virtual ICollection<MangaCover> Covers { get; set; } = [];
public virtual ICollection<MangaTitle> Titles { get; set; } = [];
public virtual ICollection<MangaDescription> Descriptions { get; set; } = [];
public virtual ICollection<MangaSource> Sources { get; set; } = [];
public virtual ICollection<MangaContributor> Contributors { get; set; } = [];
public virtual ICollection<MangaGenre> Genres { get; set; } = [];

48
MangaReader.Core/Data/MangaContext.cs
View File

@@ -7,6 +7,7 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
public DbSet<Manga> Mangas { get; set; }
public DbSet<MangaCover> MangaCovers { get; set; }
public DbSet<MangaTitle> MangaTitles { get; set; }
public DbSet<MangaDescription> MangaDescriptions { get; set; }
public DbSet<Source> Sources { get; set; }
public DbSet<MangaSource> MangaSources { get; set; }
public DbSet<Contributor> Contributors { get; set; }
@@ -24,6 +25,7 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
ConfigureManga(modelBuilder);
ConfigureMangaCover(modelBuilder);
ConfigureMangaTitle(modelBuilder);
ConfigureMangaDescription(modelBuilder);
ConfigureSource(modelBuilder);
ConfigureMangaSource(modelBuilder);
ConfigureContributor(modelBuilder);
@@ -41,9 +43,9 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
.Entity<Manga>()
.HasKey(x => x.MangaId);
modelBuilder.Entity<Manga>()
.HasIndex(x => x.Title)
.IsUnique();
//modelBuilder.Entity<Manga>()
// .HasIndex(x => x.Title)
// .IsUnique();
modelBuilder.Entity<Manga>()
.HasIndex(x => x.Slug)
@@ -81,7 +83,15 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
.HasKey(mangaTitle => mangaTitle.MangaTitleId);
modelBuilder.Entity<MangaTitle>()
.HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.Name })
.Property(mt => mt.Name)
.IsRequired();
modelBuilder.Entity<MangaTitle>()
.Property(mt => mt.Language)
.IsRequired();
modelBuilder.Entity<MangaTitle>()
.HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.Name, mangaTitle.Language })
.IsUnique();
modelBuilder
@@ -96,6 +106,36 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureMangaDescription(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaDescription>()
.HasKey(mangaTitle => mangaTitle.MangaTitleId);
modelBuilder.Entity<MangaDescription>()
.Property(mt => mt.Name)
.IsRequired();
modelBuilder.Entity<MangaDescription>()
.Property(mt => mt.Language)
.IsRequired();
modelBuilder.Entity<MangaDescription>()
.HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.Name, mangaTitle.Language })
.IsUnique();
modelBuilder
.Entity<MangaDescription>()
.HasIndex(mangaTitle => mangaTitle.Name);
modelBuilder
.Entity<MangaDescription>()
.HasOne(x => x.Manga)
.WithMany(x => x.Descriptions)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureSource(ModelBuilder modelBuilder)
{
modelBuilder

7
MangaReader.Core/Data/MangaTitle.cs
View File

@@ -1,4 +1,6 @@
namespace MangaReader.Core.Data;
using MangaReader.Core.Common;
namespace MangaReader.Core.Data;
public class MangaTitle
{
@@ -8,5 +10,6 @@ public class MangaTitle
public required Manga Manga { get; set; }
public required string Name { get; set; }
public TitleType TitleType { get; set; }
public required Language Language { get; set; }
public bool IsPrimary { get; set; }
}

20
MangaReader.Core/Data/TitleType.cs
View File

@@ -1,12 +1,12 @@
namespace MangaReader.Core.Data;
public enum TitleType
{
Primary,
OfficialTranslation,
FanTranslation,
Synonym,
Abbreviation,
Romaji,
Japanese
}
//public enum TitleType
//{
// Primary,
// OfficialTranslation,
// FanTranslation,
// Synonym,
// Abbreviation,
// Romaji,
// Japanese
//}

2
MangaReader.Core/Extensions/ServiceCollectionExtensions.cs
View File

@@ -1,4 +1,4 @@
using MangaReader.Core.HttpService;
using MangaReader.Core.Http;
using MangaReader.Core.Metadata;
using MangaReader.Core.Search;
using MangaReader.Core.Sources.MangaDex.Api;

16
MangaReader.Core/Http/HtmlLoader.cs Normal file
View File

@@ -0,0 +1,16 @@
using HtmlAgilityPack;
namespace MangaReader.Core.Http;
public class HtmlLoader(IHttpService httpService) : IHtmlLoader
{
public async Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken)
{
string html = await httpService.GetStringAsync(url, cancellationToken);
HtmlDocument doc = new();
doc.LoadHtml(html);
return doc;
}
}

2
MangaReader.Core/HttpService/HttpService.cs → MangaReader.Core/Http/HttpService.cs
View File

@@ -1,4 +1,4 @@
namespace MangaReader.Core.HttpService;
namespace MangaReader.Core.Http;
public class HttpService(HttpClient httpClient) : IHttpService
{

8
MangaReader.Core/Http/IHtmlLoader.cs Normal file
View File

@@ -0,0 +1,8 @@
using HtmlAgilityPack;
namespace MangaReader.Core.Http;
public interface IHtmlLoader
{
Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken);
}

2
MangaReader.Core/HttpService/IHttpService.cs → MangaReader.Core/Http/IHttpService.cs
View File

@@ -1,4 +1,4 @@
namespace MangaReader.Core.HttpService;
namespace MangaReader.Core.Http;
public interface IHttpService
{

22
MangaReader.Core/Metadata/MangaWebCrawler.cs
View File

@@ -1,21 +1,17 @@
using HtmlAgilityPack;
namespace MangaReader.Core.Metadata;
namespace MangaReader.Core.Metadata;
public abstract class MangaWebCrawler : IMangaMetadataProvider
{
public abstract string SourceId { get; }
public abstract Task<SourceManga?> GetMangaAsync(string url, CancellationToken cancellationToken);
protected virtual async Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken)
{
HtmlWeb web = new()
{
UsingCacheIfExists = false
};
//protected virtual async Task<HtmlDocument> GetHtmlDocumentAsync(string url, CancellationToken cancellationToken)
//{
// HtmlWeb web = new()
// {
// UsingCacheIfExists = false
// };
//return web.Load(url);
return await web.LoadFromWebAsync(url, cancellationToken);
}
// return await web.LoadFromWebAsync(url, cancellationToken);
//}
}

4
MangaReader.Core/Metadata/SourceManga.cs
View File

@@ -2,8 +2,8 @@
public class SourceManga
{
public required string Title { get; set; }
public string? Description { get; set; }
public required SourceMangaTitle Title { get; set; }
public SourceMangaDescription? Description { get; set; }
public List<SourceMangaTitle> AlternateTitles { get; set; } = [];
public SourceMangaContributor[] Contributors { get; set; } = [];
public MangaStatus Status { get; set; } = MangaStatus.Unknown;

9
MangaReader.Core/Metadata/SourceMangaDescription.cs Normal file
View File

@@ -0,0 +1,9 @@
using MangaReader.Core.Common;
namespace MangaReader.Core.Metadata;
public class SourceMangaDescription
{
public required string Name { get; set; }
public Language Language { get; set; }
}

2
MangaReader.Core/Metadata/SourceMangaTitle.cs
View File

@@ -4,6 +4,6 @@ namespace MangaReader.Core.Metadata;
public class SourceMangaTitle
{
public required string Title { get; set; }
public required string Name { get; set; }
public Language Language { get; set; }
}

63
MangaReader.Core/Pipeline/MangaPipeline.cs
View File

@@ -7,6 +7,12 @@ namespace MangaReader.Core.Pipeline;
public partial class MangaPipeline(MangaContext context) : IMangaPipeline
{
enum TitleType
{
Primary,
Secondary
}
public async Task RunAsync(MangaPipelineRequest request)
{
string sourceName = request.SourceName;
@@ -17,10 +23,12 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
Manga manga = await GetOrAddMangaAsync(sourceManga);
await AddMangaSourceAsync(sourceUrl, manga, source);
await AddTitleAsync(manga, sourceManga.Title, TitleType.Primary);
await AddDescriptionAsync(manga, sourceManga.Description);
foreach (SourceMangaTitle alternateTitle in sourceManga.AlternateTitles)
{
await AddTitleAsync(manga, alternateTitle);
await AddTitleAsync(manga, alternateTitle, TitleType.Secondary);
}
foreach (string genre in sourceManga.Genres)
@@ -55,15 +63,15 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
private async Task<Manga> GetOrAddMangaAsync(SourceManga sourceManga)
{
Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga => manga.Title == sourceManga.Title);
Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga =>
manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name));
if (manga != null)
return manga;
manga = new()
{
Title = sourceManga.Title,
Slug = GenerateSlug(sourceManga.Title),
Slug = GenerateSlug(sourceManga.Title.Name),
};
context.Add(manga);
@@ -104,10 +112,10 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
context.MangaSources.Add(mangaSource);
}
private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle)
private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle, TitleType titleType)
{
MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt =>
mt.Manga == manga && mt.Name == sourceMangaTitle.Title);
mt.Manga == manga && mt.Name == sourceMangaTitle.Name);
if (mangaTitle != null)
return;
@@ -115,12 +123,35 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
mangaTitle = new()
{
Manga = manga,
Name = sourceMangaTitle.Title,
Name = sourceMangaTitle.Name,
Language = sourceMangaTitle.Language,
IsPrimary = titleType == TitleType.Primary
};
context.MangaTitles.Add(mangaTitle);
}
private async Task AddDescriptionAsync(Manga manga, SourceMangaDescription? sourceMangaDescription)
{
if (sourceMangaDescription == null)
return;
MangaDescription? mangaDescription = await context.MangaDescriptions.FirstOrDefaultAsync(md =>
md.Manga == manga && md.Name == sourceMangaDescription.Name);
if (mangaDescription != null)
return;
mangaDescription = new()
{
Manga = manga,
Name = sourceMangaDescription.Name,
Language = sourceMangaDescription.Language
};
context.MangaDescriptions.Add(mangaDescription);
}
private async Task LinkGenreAsync(Manga manga, string genreName)
{
Genre genre = await GetOrAddGenreAsync(genreName);
@@ -156,28 +187,28 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
return genre;
}
private async Task AddChapterAsync(Manga manga, SourceMangaChapter sourceeMangaChapter)
private async Task AddChapterAsync(Manga manga, SourceMangaChapter sourceMangaChapter)
{
MangaChapter mangaChapter = await context.MangaChapters.FirstOrDefaultAsync(x => x.ChapterNumber == sourceeMangaChapter.Number)
?? AddMangaChapter(manga, sourceeMangaChapter);
MangaChapter mangaChapter = await context.MangaChapters.FirstOrDefaultAsync(x => x.ChapterNumber == sourceMangaChapter.Number)
?? AddMangaChapter(manga, sourceMangaChapter);
if (mangaChapter.VolumeNumber is null && sourceeMangaChapter.Volume is not null)
if (mangaChapter.VolumeNumber is null && sourceMangaChapter.Volume is not null)
{
mangaChapter.VolumeNumber = sourceeMangaChapter.Volume;
mangaChapter.VolumeNumber = sourceMangaChapter.Volume;
}
if (mangaChapter.Title is null && sourceeMangaChapter.Title is not null)
if (mangaChapter.Title is null && sourceMangaChapter.Title is not null)
{
mangaChapter.Title = sourceeMangaChapter.Title;
mangaChapter.Title = sourceMangaChapter.Title;
}
}
private MangaChapter AddMangaChapter(Manga manga, SourceMangaChapter sourceeMangaChapter)
private MangaChapter AddMangaChapter(Manga manga, SourceMangaChapter sourceMangaChapter)
{
MangaChapter mangaChapter = new()
{
Manga = manga,
ChapterNumber = sourceeMangaChapter.Number
ChapterNumber = sourceMangaChapter.Number
};
context.MangaChapters.Add(mangaChapter);

2
MangaReader.Core/Search/MangaSearchProviderBase.cs
View File

@@ -1,4 +1,4 @@
using MangaReader.Core.HttpService;
using MangaReader.Core.Http;
using System.Text.Json;
namespace MangaReader.Core.Search;

2
MangaReader.Core/Sources/MangaDex/Api/MangaDexClient.cs
View File

@@ -1,4 +1,4 @@
using MangaReader.Core.HttpService;
using MangaReader.Core.Http;
using System.Text;
using System.Text.Json;

13
MangaReader.Core/Sources/MangaDex/Metadata/MangaDexMetadataProvider.cs
View File

@@ -50,7 +50,16 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
return mangaGuid;
}
private static string GetTitle(MangaAttributes attributes)
private static SourceMangaTitle GetTitle(MangaAttributes attributes)
{
return new()
{
Name = GetTileName(attributes),
Language = Language.English
};
}
private static string GetTileName(MangaAttributes attributes)
{
if (attributes.Title.TryGetValue("en", out string? title))
return title;
@@ -81,7 +90,7 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
SourceMangaTitle sourceMangaTitle = new()
{
Title = alternateTitle[alternateTitleKey],
Name = alternateTitle[alternateTitleKey],
Language = language
};

19
MangaReader.Core/Sources/MangaNato/Metadata/MangaNatoWebCrawler.cs
View File

@@ -1,23 +1,28 @@
using HtmlAgilityPack;
using MangaReader.Core.Common;
using MangaReader.Core.Http;
using MangaReader.Core.Metadata;
using System.Text;
using System.Web;
namespace MangaReader.Core.Sources.MangaNato.Metadata;
public class MangaNatoWebCrawler : MangaWebCrawler
public class MangaNatoWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
{
public override string SourceId => "MangaNato";
public override async Task<SourceManga?> GetMangaAsync(string url, CancellationToken cancellationToken)
{
HtmlDocument document = await GetHtmlDocumentAsync(url, cancellationToken);
HtmlDocument document = await htmlLoader.GetHtmlDocumentAsync(url, cancellationToken);
MangaNatoMangaDocument node = new(document);
SourceManga manga = new()
{
Title = node.TitleNode?.InnerText ?? string.Empty,
Title = new()
{
Name = node.TitleNode?.InnerText ?? string.Empty,
Language = Language.Unknown
},
AlternateTitles = GetAlternateTitles(node.AlternateTitlesNode),
Contributors = GetContributors(node.AuthorsNode),
Status = GetStatus(node.StatusNode),
@@ -26,7 +31,11 @@ public class MangaNatoWebCrawler : MangaWebCrawler
RatingPercent = GetRatingPercent(node.AverageRatingNode, node.BestRatingNode),
Votes = node.VotesNode != null ? int.Parse(node.VotesNode.InnerText) : 0,
Views = GetViews(node.ViewsNode),
Description = GetTextFromNodes(node.StoryDescriptionTextNodes),
Description = new()
{
Name = GetTextFromNodes(node.StoryDescriptionTextNodes),
Language = Language.Unknown
},
Chapters = GetChapters(node.ChapterNodes)
};
@@ -46,7 +55,7 @@ public class MangaNatoWebCrawler : MangaWebCrawler
{
SourceMangaTitle sourceMangaTitle = new()
{
Title = title,
Name = title,
Language = Language.Unknown
};

2
MangaReader.Core/Sources/NatoManga/Api/NatoMangaClient.cs
View File

@@ -1,4 +1,4 @@
using MangaReader.Core.HttpService;
using MangaReader.Core.Http;
using System.Globalization;
using System.Text;
using System.Text.Json;

11
MangaReader.Core/Sources/NatoManga/Metadata/NatoMangaWebCrawler.cs
View File

@@ -1,20 +1,25 @@
using HtmlAgilityPack;
using MangaReader.Core.Http;
using MangaReader.Core.Metadata;
namespace MangaReader.Core.Sources.NatoManga.Metadata;
public class NatoMangaWebCrawler : MangaWebCrawler
public class NatoMangaWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
{
public override string SourceId => "NatoManga";
public override async Task<SourceManga?> GetMangaAsync(string url, CancellationToken cancellationToken)
{
HtmlDocument document = await GetHtmlDocumentAsync(url, cancellationToken);
HtmlDocument document = await htmlLoader.GetHtmlDocumentAsync(url, cancellationToken);
NatoMangaHtmlDocument node = new(document);
SourceManga manga = new()
{
Title = node.TitleNode?.InnerText ?? string.Empty,
Title = new()
{
Name = node.TitleNode?.InnerText ?? string.Empty,
Language = Common.Language.Unknown
},
Genres = GetGenres(node.GenresNode),
Chapters = GetChapters(node.ChapterNodes)
};