brister/manga-reader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restrcutured the database, and updated pipeline to include cover art.

Browse Source
This commit is contained in:
Brian Bicknell
2025-10-14 00:06:31 -04:00
parent 33e521e8bb
commit 4797d3c559
29 changed files with 488 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MangaReader.Core/Data/Manga.cs
View File

@@ -7,7 +7,7 @@ public class Manga
public virtual ICollection<MangaCover> Covers { get; set; } = [];
public virtual ICollection<MangaTitle> Titles { get; set; } = [];
//public virtual ICollection<MangaDescription> Descriptions { get; set; } = [];
public virtual ICollection<MangaDescription> Descriptions { get; set; } = [];
public virtual ICollection<MangaSource> Sources { get; set; } = [];
public virtual ICollection<MangaContributor> Contributors { get; set; } = [];
public virtual ICollection<MangaGenre> Genres { get; set; } = [];

106
MangaReader.Core/Data/MangaContext.cs
View File

@@ -17,6 +17,10 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
//public DbSet<MangaChapter> MangaChapters { get; set; }
//public DbSet<ChapterSource> ChapterSources { get; set; }
//public DbSet<ChapterPage> ChapterPages { get; set; }
public DbSet<SourceTitle> SourceTitles { get; set; }
public DbSet<SourceDescription> SourceDescriptions { get; set; }
public DbSet<SourceCover> SourceCovers { get; set; }
public DbSet<SourceChapter> SourceChapters { get; set; }
public DbSet<SourcePage> SourcePages { get; set; }
@@ -37,6 +41,9 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
//ConfigureMangaChapter(modelBuilder);
//ConfigureChapterSource(modelBuilder);
//ConfigureChapterPage(modelBuilder);
ConfigureSourceTitle(modelBuilder);
ConfigureSourceDescription(modelBuilder);
ConfigureMangaSourceCover(modelBuilder);
ConfigureMangaSourceChapter(modelBuilder);
ConfigureSourcePage(modelBuilder);
}
@@ -117,7 +124,7 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
.HasKey(mangaDescription => mangaDescription.MangaDescriptionId);
modelBuilder.Entity<MangaDescription>()
.Property(mangaDescription => mangaDescription.Name)
.Property(mangaDescription => mangaDescription.Text)
.IsRequired();
modelBuilder.Entity<MangaDescription>()
@@ -125,18 +132,18 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
.IsRequired();
modelBuilder.Entity<MangaDescription>()
.HasIndex(mangaDescription => new { mangaDescription.MangaSourceId, mangaDescription.Name, mangaDescription.Language })
.HasIndex(mangaDescription => new { mangaDescription.MangaId, mangaDescription.Text, mangaDescription.Language })
.IsUnique();
modelBuilder
.Entity<MangaDescription>()
.HasIndex(mangaDescription => mangaDescription.Name);
.HasIndex(mangaDescription => mangaDescription.Text);
modelBuilder
.Entity<MangaDescription>()
.HasOne(x => x.MangaSource)
.HasOne(x => x.Manga)
.WithMany(x => x.Descriptions)
.HasForeignKey(x => x.MangaSourceId)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
@@ -273,6 +280,95 @@ public class MangaContext(DbContextOptions options) : DbContext(options)
// .OnDelete(DeleteBehavior.Cascade);
//}
private static void ConfigureSourceTitle(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<SourceTitle>()
.HasKey(x => x.SourceTitleId);
modelBuilder
.Entity<SourceTitle>()
.Property(x => x.Name)
.IsRequired()
.HasMaxLength(512);
// Avoid duplicate rows coming from the same source record
modelBuilder
.Entity<SourceTitle>()
.HasIndex(x => new { x.MangaSourceId, x.Language, x.Name })
.IsUnique();
modelBuilder
.Entity<SourceTitle>()
.HasOne(x => x.MangaSource)
.WithMany(x => x.Titles)
.HasForeignKey(x => x.MangaSourceId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureSourceDescription(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<SourceDescription>()
.HasKey(x => x.SourceDescriptionId);
modelBuilder
.Entity<SourceDescription>()
.Property(x => x.Text)
.IsRequired();
// If sources can emit multiple descriptions per language, keep it non-unique.
// If not, uncomment:
//modelBuilder
// .Entity<SourceDescription>()
// .HasIndex(x => new { x.MangaSourceId, x.Language })
// .IsUnique();
modelBuilder
.Entity<SourceDescription>()
.HasOne(x => x.MangaSource)
.WithMany(x => x.Descriptions)
.HasForeignKey(x => x.MangaSourceId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureMangaSourceCover(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<SourceCover>()
.HasKey(sourceCover => sourceCover.SourceCoverId);
modelBuilder
.Entity<SourceCover>()
.Property(x => x.Url)
.IsRequired()
.HasMaxLength(2048);
modelBuilder
.Entity<SourceCover>()
.HasIndex(sourceCover => new { sourceCover.MangaSourceId, sourceCover.Url })
.IsUnique(true);
modelBuilder
.Entity<SourceCover>()
.HasOne(x => x.MangaSource)
.WithMany(x => x.Covers)
.HasForeignKey(x => x.MangaSourceId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder
.Entity<SourceCover>()
.HasOne(x => x.MangaCover)
.WithMany() // or .WithMany(c => c.SourceCovers) if you add a collection nav on MangaCover
.HasForeignKey(x => x.MangaCoverId)
.OnDelete(DeleteBehavior.SetNull);
// Helpful for lookups when you dedupe/file-link after download
modelBuilder
.Entity<SourceCover>()
.HasIndex(x => x.MangaCoverId);
}
private static void ConfigureMangaSourceChapter(ModelBuilder modelBuilder)
{
modelBuilder

6
MangaReader.Core/Data/MangaDescription.cs
View File

@@ -6,9 +6,9 @@ public class MangaDescription
{
public int MangaDescriptionId { get; set; }
public int MangaSourceId { get; set; }
public required MangaSource MangaSource { get; set; }
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public required string Name { get; set; }
public required string Text { get; set; }
public required Language Language { get; set; }
}

4
MangaReader.Core/Data/MangaSource.cs
View File

@@ -12,6 +12,8 @@ public class MangaSource
public required string Url { get; set; }
public virtual ICollection<MangaDescription> Descriptions { get; set; } = [];
public virtual ICollection<SourceTitle> Titles { get; set; } = [];
public virtual ICollection<SourceDescription> Descriptions { get; set; } = [];
public virtual ICollection<SourceCover> Covers { get; set; } = [];
public virtual ICollection<SourceChapter> Chapters { get; set; } = [];
}

14
MangaReader.Core/Data/SourceCover.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace MangaReader.Core.Data;
public class SourceCover
{
public int SourceCoverId { get; set; }
public int MangaSourceId { get; set; }
public required MangaSource MangaSource { get; set; }
public required string Url { get; set; }
public int? MangaCoverId { get; set; }
public MangaCover? MangaCover { get; set; }
}

14
MangaReader.Core/Data/SourceDescription.cs Normal file
View File

@@ -0,0 +1,14 @@
using MangaReader.Core.Common;
namespace MangaReader.Core.Data;
public class SourceDescription
{
public int SourceDescriptionId { get; set; }
public int MangaSourceId { get; set; }
public required MangaSource MangaSource { get; set; }
public required string Text { get; set; }
public required Language Language { get; set; }
}

15
MangaReader.Core/Data/SourceTitle.cs Normal file
View File

@@ -0,0 +1,15 @@
using MangaReader.Core.Common;
namespace MangaReader.Core.Data;
public class SourceTitle
{
public int SourceTitleId { get; set; }
public int MangaSourceId { get; set; }
public required MangaSource MangaSource { get; set; }
public required string Name { get; set; }
public required Language Language { get; set; }
public bool IsPrimary { get; set; }
}

8
MangaReader.Core/MangaReader.Core.csproj
View File

@@ -7,10 +7,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.12.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.6" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.6" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.9" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.9" />
</ItemGroup>
<ItemGroup>

10
MangaReader.Core/Metadata/SourceManga.cs
View File

@@ -3,15 +3,15 @@
public class SourceManga
{
public required SourceMangaTitle Title { get; set; }
public SourceMangaDescription? Description { get; set; }
public List<SourceMangaTitle> AlternateTitles { get; set; } = [];
public SourceMangaDescription[] Descriptions { get; set; } = [];
public SourceMangaTitle[] AlternateTitles { get; set; } = [];
public SourceMangaContributor[] Contributors { get; set; } = [];
public MangaStatus Status { get; set; } = MangaStatus.Unknown;
public List<string> Genres { get; set; } = [];
public string[] Genres { get; set; } = [];
public DateTime? UpdateDate { get; set; }
public long? Views { get; set; }
public float? RatingPercent { get; set; }
public int? Votes { get; set; }
public List<SourceMangaChapter> Chapters { get; set; } = [];
public string[] CoverArt { get; set; } = [];
public SourceMangaChapter[] Chapters { get; set; } = [];
public string[] CoverArtUrls { get; set; } = [];
}

93
MangaReader.Core/Pipeline/MangaPipeline.cs
View File

@@ -20,14 +20,21 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
SourceManga sourceManga = request.SourceManga;
Source source = await GetOrAddSourceAsync(sourceName);
Manga manga = await GetOrAddMangaAsync(sourceManga);
Manga manga = await GetOrAddMangaAsync(sourceManga, sourceUrl);
MangaSource mangaSource = await AddMangaSourceAsync(sourceUrl, manga, source);
await AddSourceTitleAsync(mangaSource, sourceManga.Title, TitleType.Primary);
await AddTitleAsync(manga, sourceManga.Title, TitleType.Primary);
await AddDescriptionAsync(mangaSource, sourceManga.Description);
foreach (SourceMangaDescription description in sourceManga.Descriptions)
{
await AddSourceDescriptionAsync(mangaSource, description);
//await AddDescriptionAsync(mangaSource, description);
}
foreach (SourceMangaTitle alternateTitle in sourceManga.AlternateTitles)
{
await AddSourceTitleAsync(mangaSource, alternateTitle, TitleType.Secondary);
await AddTitleAsync(manga, alternateTitle, TitleType.Secondary);
}
@@ -46,6 +53,11 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
await AddChapterAsync(mangaSource, chapter);
}
foreach (string coverArtUrl in sourceManga.CoverArtUrls)
{
await AddCoverAsync(mangaSource, coverArtUrl);
}
context.SaveChanges();
}
@@ -66,10 +78,13 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
return source;
}
private async Task<Manga> GetOrAddMangaAsync(SourceManga sourceManga)
private async Task<Manga> GetOrAddMangaAsync(SourceManga sourceManga, string sourceUrl)
{
//Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga =>
// manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name));
Manga? manga = await context.Mangas.FirstOrDefaultAsync(manga =>
manga.Titles.Any(mangaTitle => mangaTitle.Name == sourceManga.Title.Name));
manga.Sources.Any(mangaSource => mangaSource.Url == sourceUrl));
if (manga != null)
return manga;
@@ -119,6 +134,49 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
return mangaSource;
}
private async Task AddSourceTitleAsync(MangaSource mangaSource, SourceMangaTitle sourceMangaTitle, TitleType titleType)
{
SourceTitle? sourceTitle = await context.SourceTitles.FirstOrDefaultAsync(mt =>
mt.MangaSource == mangaSource && mt.Name == sourceMangaTitle.Name);
if (sourceTitle != null)
return;
sourceTitle = new()
{
MangaSource = mangaSource,
Name = sourceMangaTitle.Name,
Language = sourceMangaTitle.Language,
IsPrimary = titleType == TitleType.Primary
};
context.SourceTitles.Add(sourceTitle);
}
private async Task AddSourceDescriptionAsync(MangaSource mangaSource, SourceMangaDescription? sourceMangaDescription)
{
if (sourceMangaDescription == null)
return;
SourceDescription? sourceDescription = await context.SourceDescriptions.FirstOrDefaultAsync(md =>
md.MangaSource == mangaSource && md.Language == sourceMangaDescription.Language);
if (sourceDescription != null)
{
sourceDescription.Text = sourceMangaDescription.Name;
return;
}
sourceDescription = new()
{
MangaSource = mangaSource,
Text = sourceMangaDescription.Name,
Language = sourceMangaDescription.Language
};
context.SourceDescriptions.Add(sourceDescription);
}
private async Task AddTitleAsync(Manga manga, SourceMangaTitle sourceMangaTitle, TitleType titleType)
{
MangaTitle? mangaTitle = await context.MangaTitles.FirstOrDefaultAsync(mt =>
@@ -138,24 +196,24 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
context.MangaTitles.Add(mangaTitle);
}
private async Task AddDescriptionAsync(MangaSource mangaSource, SourceMangaDescription? sourceMangaDescription)
private async Task AddDescriptionAsync(Manga manga, SourceMangaDescription? sourceMangaDescription)
{
if (sourceMangaDescription == null)
return;
MangaDescription? mangaDescription = await context.MangaDescriptions.FirstOrDefaultAsync(md =>
md.MangaSource == mangaSource && md.Language == sourceMangaDescription.Language);
md.Manga == manga && md.Language == sourceMangaDescription.Language);
if (mangaDescription != null)
{
mangaDescription.Name = sourceMangaDescription.Name;
mangaDescription.Text = sourceMangaDescription.Name;
return;
}
mangaDescription = new()
{
MangaSource = mangaSource,
Name = sourceMangaDescription.Name,
Manga = manga,
Text = sourceMangaDescription.Name,
Language = sourceMangaDescription.Language
};
@@ -278,6 +336,23 @@ public partial class MangaPipeline(MangaContext context) : IMangaPipeline
return sourceChapter;
}
private async Task AddCoverAsync(MangaSource mangaSource, string coverArtUrl)
{
SourceCover? sourceCover = await context.SourceCovers.FirstOrDefaultAsync(x =>
x.MangaSource == mangaSource && x.Url == coverArtUrl);
if (sourceCover == null)
{
sourceCover = new()
{
MangaSource = mangaSource,
Url = coverArtUrl
};
context.SourceCovers.Add(sourceCover);
}
}
public async Task RunPagesAsync(MangaPagePipelineRequest request, CancellationToken cancellationToken)
{
SourceChapter? sourceChapter = await context.SourceChapters.FirstOrDefaultAsync(x => x.SourceChapterId == request.SourceChapterId, cancellationToken);

65
MangaReader.Core/Sources/MangaDex/Metadata/MangaDexMetadataProvider.cs
View File

@@ -31,10 +31,11 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
{
Title = GetTitle(mangaAttributes),
AlternateTitles = GetAlternateTitles(mangaAttributes),
Descriptions = GetDescriptions(mangaAttributes),
Genres = GetGenres(mangaAttributes),
Contributors = GetContributors(mangaRelationships),
Chapters = GetChapters(mangaDexFeedResponse),
CoverArt = GetCoverArt(mangaGuid, coverArtResponse)
CoverArtUrls = GetCoverArtUrls(mangaGuid, coverArtResponse)
};
}
@@ -52,22 +53,27 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
private static SourceMangaTitle GetTitle(MangaAttributes attributes)
{
(string title, Language langauge) = GetTileAndLanguage(attributes);
return new()
{
Name = GetTileName(attributes),
Language = Language.English
Name = title,
Language = langauge
};
}
private static string GetTileName(MangaAttributes attributes)
private static (string title, Language langauge) GetTileAndLanguage(MangaAttributes attributes)
{
if (attributes.Title.TryGetValue("en", out string? title))
return title;
if (attributes.Title.TryGetValue("en", out string? englishTitle))
return (englishTitle, Language.English);
return string.Empty;
if (attributes.Title.TryGetValue("ja-ro", out string? japaneseTitle))
return (japaneseTitle, Language.Japanese);
return (string.Empty, Language.Unknown);
}
private static List<SourceMangaTitle> GetAlternateTitles(MangaAttributes attributes)
private static SourceMangaTitle[] GetAlternateTitles(MangaAttributes attributes)
{
if (attributes.AltTitles == null || attributes.AltTitles.Count == 0)
return [];
@@ -98,10 +104,41 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
}
}
return sourceMangaTitles;
return [.. sourceMangaTitles];
}
private static List<string> GetGenres(MangaAttributes attributes)
private static SourceMangaDescription[] GetDescriptions(MangaAttributes attributes)
{
if (attributes.AltTitles == null || attributes.AltTitles.Count == 0)
return [];
Dictionary<string, Language> languageIdMap = new()
{
{ "en", Language.English },
{ "ja", Language.Japanese },
{ "ja-ro", Language.Romaji },
};
List<SourceMangaDescription> sourceMangaDescriptions = [];
foreach (string key in attributes.Description.Keys)
{
if (languageIdMap.TryGetValue(key, out Language language) == false)
continue;
SourceMangaDescription sourceMangaDescription = new()
{
Name = attributes.Description[key],
Language = language
};
sourceMangaDescriptions.Add(sourceMangaDescription);
}
return [.. sourceMangaDescriptions];
}
private static string[] GetGenres(MangaAttributes attributes)
{
if (attributes.Tags == null || attributes.Tags.Count == 0)
return [];
@@ -119,7 +156,7 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
tags.Add(tagEntity.Attributes.Name.FirstOrDefault().Value);
}
return tags;
return [.. tags];
}
private static SourceMangaContributor[] GetContributors(List<MangaDexEntity> relationships)
@@ -160,7 +197,7 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
return [.. contributors];
}
private static List<SourceMangaChapter> GetChapters(MangaDexResponse? mangaDexFeedResponse)
private static SourceMangaChapter[] GetChapters(MangaDexResponse? mangaDexFeedResponse)
{
if (mangaDexFeedResponse == null || mangaDexFeedResponse is not MangaDexCollectionResponse collectionResponse)
return [];
@@ -191,10 +228,10 @@ public class MangaDexMetadataProvider(IMangaDexClient mangaDexClient) : IMangaMe
chapters.Add(chapter);
}
return chapters;
return [.. chapters];
}
private static string[] GetCoverArt(Guid mangaGuid, MangaDexResponse? coverArtResponse)
private static string[] GetCoverArtUrls(Guid mangaGuid, MangaDexResponse? coverArtResponse)
{
if (coverArtResponse == null || coverArtResponse is not MangaDexCollectionResponse collectionResponse)
return [];

27
MangaReader.Core/Sources/MangaNato/Metadata/MangaNatoWebCrawler.cs
View File

@@ -31,18 +31,21 @@ public class MangaNatoWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
RatingPercent = GetRatingPercent(node.AverageRatingNode, node.BestRatingNode),
Votes = node.VotesNode != null ? int.Parse(node.VotesNode.InnerText) : 0,
Views = GetViews(node.ViewsNode),
Description = new()
{
Name = GetTextFromNodes(node.StoryDescriptionTextNodes),
Language = Language.Unknown
},
Descriptions =
[
new()
{
Name = GetTextFromNodes(node.StoryDescriptionTextNodes),
Language = Language.Unknown
}
],
Chapters = GetChapters(node.ChapterNodes)
};
return manga;
}
private static List<SourceMangaTitle> GetAlternateTitles(HtmlNode? node)
private static SourceMangaTitle[] GetAlternateTitles(HtmlNode? node)
{
if (node == null)
return [];
@@ -98,7 +101,7 @@ public class MangaNatoWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
};
}
private static List<string> GetGenres(HtmlNode? node)
private static string[] GetGenres(HtmlNode? node)
{
if (node == null)
return [];
@@ -162,12 +165,12 @@ public class MangaNatoWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
return (int)Math.Round(average / best * 100);
}
private static List<SourceMangaChapter> GetChapters(HtmlNodeCollection? chapterNodes)
private static SourceMangaChapter[] GetChapters(HtmlNodeCollection? chapterNodes)
{
List<SourceMangaChapter> chapters = [];
if (chapterNodes == null)
return chapters;
return [];
List<SourceMangaChapter> chapters = [];
foreach (var node in chapterNodes)
{
@@ -187,7 +190,7 @@ public class MangaNatoWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
chapters.Add(chapter);
}
return chapters;
return [.. chapters];
}
private static float GetChapterNumber(HtmlNode? chapterNameNode)

12
MangaReader.Core/Sources/NatoManga/Metadata/NatoMangaWebCrawler.cs
View File

@@ -27,7 +27,7 @@ public class NatoMangaWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
return manga;
}
private static List<string> GetGenres(HtmlNode? node)
private static string[] GetGenres(HtmlNode? node)
{
if (node == null)
return [];
@@ -77,12 +77,12 @@ public class NatoMangaWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
};
}
private static List<SourceMangaChapter> GetChapters(HtmlNodeCollection? chapterNodes)
private static SourceMangaChapter[] GetChapters(HtmlNodeCollection? chapterNodes)
{
List<SourceMangaChapter> chapters = [];
if (chapterNodes == null)
return chapters;
return [];
List<SourceMangaChapter> chapters = [];
foreach (var node in chapterNodes)
{
@@ -110,7 +110,7 @@ public class NatoMangaWebCrawler(IHtmlLoader htmlLoader) : MangaWebCrawler
chapters.Add(chapter);
}
return chapters;
return [.. chapters];
}
private static float GetChapterNumber(HtmlNode chapterNameNode)