Add project files.

This commit is contained in:
2025-05-21 19:39:09 -04:00
parent 7d2b71fe95
commit ec1713c95f
27 changed files with 5843 additions and 0 deletions

View File

@@ -0,0 +1,320 @@
using Microsoft.EntityFrameworkCore;
namespace MangaReader.Core.Data;
public class Manga
{
public int MangaId { get; set; }
public required string Slug { get; set; }
public required string Title { get; set; }
public string? Description { get; set; }
public virtual ICollection<MangaCover> Covers { get; set; }
public virtual ICollection<MangaTitle> Titles { get; set; }
public virtual ICollection<MangaSource> Sources { get; set; }
public virtual ICollection<MangaGenre> Genres { get; set; }
public virtual ICollection<MangaChapter> Chapters { get; set; }
public Manga()
{
Covers = new HashSet<MangaCover>();
Titles = new HashSet<MangaTitle>();
Sources = new HashSet<MangaSource>();
Genres = new HashSet<MangaGenre>();
Chapters = new HashSet<MangaChapter>();
}
}
public class MangaCover
{
public int MangaCoverId { get; set; }
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public required Guid Guid { get; set; }
public required string FileExtension { get; set; }
public string? Description { get; set; }
public bool IsPrimary { get; set; }
}
public class MangaTitle
{
public int MangaTitleId { get; set; }
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public required string TitleEntry { get; set; }
public TitleType TitleType { get; set; }
}
public enum TitleType
{
Primary,
OfficialTranslation,
FanTranslation,
Synonym,
Abbreviation,
Romaji,
Japanese
}
public class Source
{
public int SourceId { get; set; }
public required string Name { get; set; }
}
public class MangaSource
{
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public int SourceId { get; set; }
public required Source Source { get; set; }
public required string Url { get; set; }
}
public class Genre
{
public int GenreId { get; set; }
public required string Name { get; set; }
}
public class MangaGenre
{
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public int GenreId { get; set; }
public required Genre Genre { get; set; }
}
public class MangaChapter
{
public int MangaChapterId { get; set; }
public int MangaId { get; set; }
public required Manga Manga { get; set; }
public int? VolumeNumber { get; set; }
public int ChapterNumber { get; set; }
public string? Title { get; set; }
public virtual ICollection<ChapterSource> Sources { get; set; } = [];
public virtual ICollection<ChapterPage> Pages { get; set; } = [];
}
public class ChapterSource
{
public int MangaChapterId { get; set; }
public required MangaChapter Chapter { get; set; }
public int SourceId { get; set; }
public required Source Source { get; set; }
public required string Url { get; set; }
}
public class ChapterPage
{
public int ChapterPageId { get; set; }
public int MangaChapterId { get; set; }
public required MangaChapter MangaChapter { get; set; }
public int PageNumber { get; set; }
}
public class MangaContext(DbContextOptions options) : DbContext(options)
{
public DbSet<Manga> Mangas { get; set; }
public DbSet<MangaCover> MangaCovers { get; set; }
public DbSet<MangaTitle> MangaTitles { get; set; }
public DbSet<Source> Sources { get; set; }
public DbSet<MangaSource> MangaSources { get; set; }
public DbSet<Genre> Genres { get; set; }
public DbSet<MangaGenre> MangaGenres { get; set; }
public DbSet<MangaChapter> MangaChapters { get; set; }
public DbSet<ChapterSource> ChapterSources { get; set; }
public DbSet<ChapterPage> ChapterPages { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
ConfigureManga(modelBuilder);
ConfigureMangaCover(modelBuilder);
ConfigureMangaTitle(modelBuilder);
ConfigureSource(modelBuilder);
ConfigureMangaSource(modelBuilder);
ConfigureGenre(modelBuilder);
ConfigureMangaGenre(modelBuilder);
ConfigureMangaChapter(modelBuilder);
ConfigureChapterSource(modelBuilder);
ConfigureChapterPage(modelBuilder);
}
private static void ConfigureManga(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<Manga>()
.HasKey(x => x.MangaId);
modelBuilder.Entity<Manga>()
.HasIndex(x => x.Slug)
.IsUnique();
}
private static void ConfigureMangaCover(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaCover>()
.HasKey(x => x.MangaCoverId);
modelBuilder
.Entity<MangaCover>()
.HasOne(x => x.Manga)
.WithMany(x => x.Covers)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MangaCover>()
.HasIndex(x => x.Guid)
.IsUnique();
//modelBuilder
// .Entity<MangaCover>()
// .HasIndex(x => new { x.MangaId, x.IsPrimary })
// .IsUnique()
// .HasFilter("[IsPrimary] = 1"); // Enforce only one primary cover per manga
}
private static void ConfigureMangaTitle(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaTitle>()
.HasKey(mangaTitle => mangaTitle.MangaTitleId);
modelBuilder.Entity<MangaTitle>()
.HasIndex(mangaTitle => new { mangaTitle.MangaId, mangaTitle.TitleEntry })
.IsUnique();
modelBuilder
.Entity<MangaTitle>()
.HasIndex(mangaTitle => mangaTitle.TitleEntry);
modelBuilder
.Entity<MangaTitle>()
.HasOne(x => x.Manga)
.WithMany(x => x.Titles)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureSource(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<Source>()
.HasKey(x => x.SourceId);
modelBuilder
.Entity<Source>()
.HasIndex(x => x.Name)
.IsUnique(true);
}
private static void ConfigureMangaSource(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaSource>()
.HasKey(mangaSource => new { mangaSource.MangaId, mangaSource.SourceId });
modelBuilder.Entity<MangaSource>()
.HasIndex(x => x.Url)
.IsUnique();
modelBuilder
.Entity<MangaSource>()
.HasOne(x => x.Manga)
.WithMany(x => x.Sources)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureGenre(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<Genre>()
.HasKey(x => x.GenreId);
modelBuilder
.Entity<Genre>()
.HasIndex(x => x.Name)
.IsUnique(true);
}
private static void ConfigureMangaGenre(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaGenre>()
.HasKey(mangaGenre => new { mangaGenre.MangaId, mangaGenre.GenreId });
modelBuilder
.Entity<MangaGenre>()
.HasOne(x => x.Manga)
.WithMany(x => x.Genres)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureMangaChapter(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<MangaChapter>()
.HasKey(x => x.MangaChapterId);
modelBuilder
.Entity<MangaChapter>()
.HasOne(x => x.Manga)
.WithMany(x => x.Chapters)
.HasForeignKey(x => x.MangaId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureChapterSource(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<ChapterSource>()
.HasKey(chapterSource => new { chapterSource.MangaChapterId, chapterSource.SourceId });
modelBuilder
.Entity<ChapterSource>()
.HasOne(x => x.Chapter)
.WithMany(x => x.Sources)
.HasForeignKey(x => x.MangaChapterId)
.OnDelete(DeleteBehavior.Cascade);
}
private static void ConfigureChapterPage(ModelBuilder modelBuilder)
{
modelBuilder
.Entity<ChapterPage>()
.HasKey(chapterPage => chapterPage.ChapterPageId);
modelBuilder
.Entity<ChapterPage>()
.HasIndex(chapterPage => new { chapterPage.MangaChapterId, chapterPage.PageNumber })
.IsUnique(true);
modelBuilder
.Entity<ChapterPage>()
.HasOne(x => x.MangaChapter)
.WithMany(x => x.Pages)
.HasForeignKey(x => x.MangaChapterId)
.OnDelete(DeleteBehavior.Cascade);
}
}

View File

@@ -0,0 +1,6 @@
namespace MangaReader.Core.HttpService;
public class HttpService(HttpClient httpClient) : IHttpService
{
public Task<string> GetStringAsync(string url) => httpClient.GetStringAsync(url);
}

View File

@@ -0,0 +1,6 @@
namespace MangaReader.Core.HttpService;
public interface IHttpService
{
Task<string> GetStringAsync(string url);
}

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.12.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.5" />
</ItemGroup>
<ItemGroup>
<Folder Include="WebSearch\MangaDex\" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,6 @@
namespace MangaReader.Core.WebCrawlers;
public interface IMangaWebCrawler
{
MangaDTO GetManga(string url);
}

View File

@@ -0,0 +1,11 @@
namespace MangaReader.Core.WebCrawlers;
public class MangaChapterDTO
{
public int? Volume { get; set; }
public float? Number { get; set; }
public string? Name { get; set; }
public required string Url { get; set; }
public long? Views { get; set; }
public DateTime? UploadDate { get; set; }
}

View File

@@ -0,0 +1,16 @@
namespace MangaReader.Core.WebCrawlers;
public class MangaDTO
{
public required string Title { get; set; }
public string? Description { get; set; }
public List<string> AlternateTitles { get; set; } = [];
public List<string> Authors { get; set; } = [];
public MangaStatus Status { get; set; } = MangaStatus.Unknown;
public List<string> Genres { get; set; } = [];
public DateTime? UpdateDate { get; set; }
public long? Views { get; set; }
public float? RatingPercent { get; set; }
public int? Votes { get; set; }
public List<MangaChapterDTO> Chapters { get; set; } = [];
}

View File

@@ -0,0 +1,61 @@
using HtmlAgilityPack;
namespace MangaReader.Core.WebCrawlers.MangaNato;
public class MangaNatoMangaDocument
{
public HtmlNode StoryInfoNode { get; }
public HtmlNode TitleNode { get; }
public HtmlNode StoryInfoRightNode { get; }
public HtmlNode VariationsTableInfo { get; }
public HtmlNodeCollection VariationsTableValueNodes { get; }
public HtmlNode AlternateTitlesNode { get; }
public HtmlNode AuthorsNode { get; }
public HtmlNode StatusNode { get; }
public HtmlNode GenresNode { get; }
public HtmlNode StoryInfoRightExtentNode { get; }
public HtmlNodeCollection StoryInfoRightExtentValueNodes { get; }
public HtmlNode UpdateDateNode { get; }
public HtmlNode ViewsNode { get; }
public HtmlNode ReviewAggregateNode { get; }
public HtmlNode RatingNode { get; }
public HtmlNode AverageRatingNode { get; }
public HtmlNode BestRatingNode { get; }
public HtmlNode VotesNode { get; set; }
public HtmlNode StoryDescriptionNode { get; }
public List<HtmlNode> StoryDescriptionTextNodes { get; }
public HtmlNode StoryChapterListNode { get; }
public HtmlNodeCollection ChapterNodes { get; }
public MangaNatoMangaDocument(HtmlDocument document)
{
StoryInfoNode = document.DocumentNode.SelectSingleNode(".//div[@class='panel-story-info']");
TitleNode = StoryInfoNode.SelectSingleNode(".//h1");
StoryDescriptionNode = StoryInfoNode.SelectSingleNode(".//div[@class='panel-story-info-description']");
StoryDescriptionTextNodes = StoryDescriptionNode.ChildNodes.Skip(2).Take(StoryDescriptionNode.ChildNodes.Count - 2).ToList();
StoryInfoRightNode = StoryInfoNode.SelectSingleNode(".//div[@class='story-info-right']");
VariationsTableInfo = StoryInfoRightNode.SelectSingleNode(".//table[@class='variations-tableInfo']");
VariationsTableValueNodes = VariationsTableInfo.SelectNodes(".//td[@class='table-value']");
AlternateTitlesNode = VariationsTableValueNodes[0];
AuthorsNode = VariationsTableValueNodes[1];
StatusNode = VariationsTableValueNodes[2];
GenresNode = VariationsTableValueNodes[3];
StoryInfoRightExtentNode = StoryInfoRightNode.SelectSingleNode(".//div[@class='story-info-right-extent']");
StoryInfoRightExtentValueNodes = StoryInfoRightExtentNode.SelectNodes(".//span[@class='stre-value']");
UpdateDateNode = StoryInfoRightExtentValueNodes[0];
ViewsNode = StoryInfoRightExtentValueNodes[1];
// v:Review-aggregate
ReviewAggregateNode = StoryInfoRightNode.SelectSingleNode(".//em[@typeof='v:Review-aggregate']");
RatingNode = ReviewAggregateNode.SelectSingleNode(".//em[@typeof='v:Rating']");
AverageRatingNode = RatingNode.SelectSingleNode(".//em[@property='v:average']");
BestRatingNode = RatingNode.SelectSingleNode(".//em[@property='v:best']");
VotesNode = ReviewAggregateNode.SelectSingleNode(".//em[@property='v:votes']");
StoryChapterListNode = document.DocumentNode.SelectSingleNode(".//div[@class='panel-story-chapter-list']");
ChapterNodes = StoryChapterListNode.SelectNodes(".//li[@class='a-h']");
}
}

View File

@@ -0,0 +1,160 @@
using HtmlAgilityPack;
using System.Text;
using System.Web;
namespace MangaReader.Core.WebCrawlers.MangaNato;
public class MangaNatoWebCrawler : MangaWebCrawler
{
public override MangaDTO GetManga(string url)
{
HtmlDocument document = GetHtmlDocument(url);
MangaNatoMangaDocument node = new(document);
MangaDTO manga = new()
{
Title = node.TitleNode.InnerText,
AlternateTitles = GetAlternateTitles(node.AlternateTitlesNode),
Authors = GetAuthors(node.AuthorsNode),
Status = GetStatus(node.StatusNode),
Genres = GetGenres(node.GenresNode),
UpdateDate = GetUpdateDate(node.UpdateDateNode),
RatingPercent = GetRatingPercent(node.AverageRatingNode, node.BestRatingNode),
Votes = int.Parse(node.VotesNode.InnerText),
Views = GetViews(node.ViewsNode),
Description = GetTextFromNodes(node.StoryDescriptionTextNodes),
Chapters = GetChapters(node.ChapterNodes)
};
return manga;
}
private static List<string> GetAlternateTitles(HtmlNode node)
{
return node.InnerText.Split(';').Select(x => x.Trim()).ToList();
}
private static List<string> GetAuthors(HtmlNode node)
{
return node.InnerText.Split('-').Select(x => x.Trim()).ToList();
}
private static MangaStatus GetStatus(HtmlNode node)
{
return node.InnerText switch
{
"Ongoing" => MangaStatus.Ongoing,
"Completed" => MangaStatus.Complete,
_ => MangaStatus.Unknown,
};
}
private static List<string> GetGenres(HtmlNode node)
{
return node.InnerText.Split('-').Select(x => x.Trim()).ToList();
}
private static DateTime GetUpdateDate(HtmlNode node)
{
List<string> dateAndTime = node.InnerText.Split('-').Select(x => x.Trim()).ToList();
DateOnly date = DateOnly.Parse(dateAndTime[0]);
TimeOnly time = TimeOnly.Parse(dateAndTime[1]);
return date.ToDateTime(time);
}
private static long GetViews(HtmlNode node)
{
string text = node.InnerText;
if (int.TryParse(text, out int number))
return number;
ReadOnlySpan<char> shortText = text.AsSpan(0, text.Length - 1);
if (double.TryParse(shortText, out double formattedNumber) == false)
return 0;
char suffix = text[^1];
long multiplier = GetMultiplier(suffix);
return (int)(formattedNumber * multiplier);
}
private static long GetMultiplier(char c)
{
return c switch
{
'K' => 1_000,
'M' => 1_000_000,
'B' => 1_000_000_000,
'T' => 1_000_000_000_000,
_ => 0,
};
}
private static int GetRatingPercent(HtmlNode averageNode, HtmlNode bestNode)
{
double average = Convert.ToDouble(averageNode.InnerText);
double best = Convert.ToDouble(bestNode.InnerText);
return (int)Math.Round(average / best * 100);
}
private static List<MangaChapterDTO> GetChapters(HtmlNodeCollection chapterNodes)
{
List<MangaChapterDTO> chapters = [];
foreach (var node in chapterNodes)
{
HtmlNode chapterNameNode = node.SelectSingleNode(".//a[contains(@class, 'chapter-name')]");
HtmlNode chapterViewNode = node.SelectSingleNode(".//span[contains(@class, 'chapter-view')]");
HtmlNode chapterTimeNode = node.SelectSingleNode(".//span[contains(@class, 'chapter-time')]");
MangaChapterDTO chapter = new()
{
Number = GetChapterNumber(chapterNameNode),
Name = chapterNameNode.InnerText,
Url = chapterNameNode.Attributes["href"].Value,
Views = GetViews(chapterViewNode),
UploadDate = DateTime.Parse(chapterTimeNode.Attributes["title"].Value)
};
chapters.Add(chapter);
}
return chapters;
}
private static float GetChapterNumber(HtmlNode chapterNameNode)
{
string url = chapterNameNode.Attributes["href"].Value;
int index = url.IndexOf("/chapter-");
if (index == -1)
return 0;
string chapterNumber = url[(index + "/chapter-".Length)..];
return float.Parse(chapterNumber);
}
private static string GetTextFromNodes(List<HtmlNode> nodes)
{
StringBuilder stringBuilder = new();
foreach (HtmlNode node in nodes)
{
if (node.Name == "br")
{
stringBuilder.AppendLine();
}
else
{
stringBuilder.Append(HttpUtility.HtmlDecode(node.InnerText).Replace("\r\n", "").Trim());
}
}
return stringBuilder.ToString();
}
}

View File

@@ -0,0 +1,8 @@
namespace MangaReader.Core.WebCrawlers;
public enum MangaStatus
{
Unknown,
Ongoing,
Complete
}

View File

@@ -0,0 +1,18 @@
using HtmlAgilityPack;
namespace MangaReader.Core.WebCrawlers;
public abstract class MangaWebCrawler : IMangaWebCrawler
{
public abstract MangaDTO GetManga(string url);
protected virtual HtmlDocument GetHtmlDocument(string url)
{
HtmlWeb web = new()
{
UsingCacheIfExists = false
};
return web.Load(url);
}
}

View File

@@ -0,0 +1,21 @@
using HtmlAgilityPack;
namespace MangaReader.Core.WebCrawlers.NatoManga;
public class NatoMangaHtmlDocument
{
public HtmlNode? MangaInfoTextNode { get; }
public HtmlNode? TitleNode { get; }
public HtmlNode? GenresNode { get; }
public HtmlNode? ChapterListNode { get; }
public HtmlNodeCollection? ChapterNodes { get; }
public NatoMangaHtmlDocument(HtmlDocument document)
{
MangaInfoTextNode = document.DocumentNode.SelectSingleNode(".//ul[@class='manga-info-text']");
TitleNode = MangaInfoTextNode?.SelectSingleNode(".//li//h1");
GenresNode = MangaInfoTextNode?.SelectSingleNode(".//li[@class='genres']");
ChapterListNode = document.DocumentNode.SelectSingleNode(".//div[@class='chapter-list']");
ChapterNodes = ChapterListNode?.SelectNodes(".//div[@class='row']");
}
}

View File

@@ -0,0 +1,186 @@
using HtmlAgilityPack;
using System.Text;
using System.Web;
namespace MangaReader.Core.WebCrawlers.NatoManga;
public class NatoMangaWebCrawler : MangaWebCrawler
{
public override MangaDTO GetManga(string url)
{
HtmlDocument document = GetHtmlDocument(url);
NatoMangaHtmlDocument node = new(document);
MangaDTO manga = new()
{
Title = node.TitleNode?.InnerText ?? string.Empty,
//AlternateTitles = GetAlternateTitles(node.AlternateTitlesNode),
//Authors = GetAuthors(node.AuthorsNode),
//Status = GetStatus(node.StatusNode),
Genres = GetGenres(node.GenresNode),
//UpdateDate = GetUpdateDate(node.UpdateDateNode),
//RatingPercent = GetRatingPercent(node.AverageRatingNode, node.BestRatingNode),
//Votes = int.Parse(node.VotesNode.InnerText),
//Views = GetViews(node.ViewsNode),
//Description = GetTextFromNodes(node.StoryDescriptionTextNodes),
Chapters = GetChapters(node.ChapterNodes)
};
return manga;
}
private static List<string> GetAlternateTitles(HtmlNode node)
{
return node.InnerText.Split(';').Select(x => x.Trim()).ToList();
}
private static List<string> GetAuthors(HtmlNode node)
{
return node.InnerText.Split('-').Select(x => x.Trim()).ToList();
}
private static MangaStatus GetStatus(HtmlNode node)
{
return node.InnerText switch
{
"Ongoing" => MangaStatus.Ongoing,
"Completed" => MangaStatus.Complete,
_ => MangaStatus.Unknown,
};
}
private static List<string> GetGenres(HtmlNode? node)
{
if (node == null)
return [];
HtmlNodeCollection genreNodes = node.SelectNodes(".//a");
if (genreNodes == null)
return [];
return [.. genreNodes.Select(genreNode => genreNode.InnerText.Trim())];
}
private static DateTime GetUpdateDate(HtmlNode node)
{
List<string> dateAndTime = node.InnerText.Split('-').Select(x => x.Trim()).ToList();
DateOnly date = DateOnly.Parse(dateAndTime[0]);
TimeOnly time = TimeOnly.Parse(dateAndTime[1]);
return date.ToDateTime(time);
}
private static long GetViews(HtmlNode node)
{
string text = node.InnerText.Trim();
if (int.TryParse(text, out int number))
return number;
if (double.TryParse(text, out double doubleNumber))
return (int)doubleNumber;
ReadOnlySpan<char> shortText = text.AsSpan(0, text.Length - 1);
if (double.TryParse(shortText, out double formattedNumber) == false)
return 0;
char suffix = text[^1];
//if (char.GetNumericValue(suffix) > -1)
// return (int)formattedNumber;
long multiplier = GetMultiplier(suffix);
return (int)(formattedNumber * multiplier);
}
private static long GetMultiplier(char c)
{
return c switch
{
'K' => 1_000,
'M' => 1_000_000,
'B' => 1_000_000_000,
'T' => 1_000_000_000_000,
_ => 0,
};
}
private static int GetRatingPercent(HtmlNode averageNode, HtmlNode bestNode)
{
double average = Convert.ToDouble(averageNode.InnerText);
double best = Convert.ToDouble(bestNode.InnerText);
return (int)Math.Round(average / best * 100);
}
private static List<MangaChapterDTO> GetChapters(HtmlNodeCollection? chapterNodes)
{
List<MangaChapterDTO> chapters = [];
if (chapterNodes == null)
return chapters;
foreach (var node in chapterNodes)
{
HtmlNodeCollection? chapterPropertyNodes = node.SelectNodes(".//span");
if (chapterPropertyNodes == null || chapterPropertyNodes.Count < 3)
continue;
HtmlNode? chapterNameNode = chapterPropertyNodes[0].SelectSingleNode(".//a");
HtmlNode chapterViewNode = chapterPropertyNodes[1];
HtmlNode chapterTimeNode = chapterPropertyNodes[2];
if (chapterNameNode == null)
continue;
MangaChapterDTO chapter = new()
{
Number = GetChapterNumber(chapterNameNode),
Name = chapterNameNode.InnerText,
Url = chapterNameNode.Attributes["href"].Value,
Views = GetViews(chapterViewNode),
UploadDate = DateTime.Parse(chapterTimeNode.Attributes["title"].Value)
};
chapters.Add(chapter);
}
return chapters;
}
private static float GetChapterNumber(HtmlNode chapterNameNode)
{
string url = chapterNameNode.Attributes["href"].Value;
int index = url.IndexOf("/chapter-");
if (index == -1)
return 0;
string chapterNumber = url[(index + "/chapter-".Length)..].Replace('-', '.');
return float.Parse(chapterNumber);
}
private static string GetTextFromNodes(List<HtmlNode> nodes)
{
StringBuilder stringBuilder = new();
foreach (HtmlNode node in nodes)
{
if (node.Name == "br")
{
stringBuilder.AppendLine();
}
else
{
stringBuilder.Append(HttpUtility.HtmlDecode(node.InnerText).Replace("\r\n", "").Trim());
}
}
return stringBuilder.ToString();
}
}

View File

@@ -0,0 +1,11 @@
namespace MangaReader.Core.WebSearch;
public interface IMangaWebSearch<T>
{
Task<MangaSearchResult[]> SearchAsync(string keyword);
}
//public class MangaDexWebSearch : IMangaWebSearch
//{
// // https://api.mangadex.org/manga?title=gal can't be&limit=5
//}

View File

@@ -0,0 +1,9 @@
namespace MangaReader.Core.WebSearch;
public record MangaSearchResult
{
public required string Url { get; init; }
public required string Title { get; init; }
public string? Author { get; init; }
public string? Description { get; init; }
}

View File

@@ -0,0 +1,33 @@
using MangaReader.Core.HttpService;
using System.Text.Json;
namespace MangaReader.Core.WebSearch;
public abstract class MangaWebSearchBase<T>(IHttpService httpService) : IMangaWebSearch<T>
{
private static JsonSerializerOptions _jsonSerializerOptions = new()
{
PropertyNameCaseInsensitive = true
};
public async Task<MangaSearchResult[]> SearchAsync(string keyword)
{
T? searchResult = await GetSearchResultAsync(keyword);
if (searchResult == null)
return [];
return GetSearchResult(searchResult);
}
private async Task<T?> GetSearchResultAsync(string keyword)
{
string url = GetSearchUrl(keyword);
string response = await httpService.GetStringAsync(url);
return JsonSerializer.Deserialize<T>(response, _jsonSerializerOptions);
}
protected abstract string GetSearchUrl(string keyword);
protected abstract MangaSearchResult[] GetSearchResult(T searchResult);
}

View File

@@ -0,0 +1,12 @@
namespace MangaReader.Core.WebSearch.NatoManga;
public record NatoMangaSearchResult
{
public int Id { get; init; }
public string? Author { get; init; }
public required string Name { get; init; }
public string? ChapterLatest { get; init; }
public required string Url { get; init; }
public string? Thumb { get; init; }
public string? Slug { get; init; }
}

View File

@@ -0,0 +1,25 @@
using MangaReader.Core.HttpService;
namespace MangaReader.Core.WebSearch.NatoManga;
public class NatoMangaWebSearch(IHttpService httpService) : MangaWebSearchBase<NatoMangaSearchResult[]>(httpService)
{
// https://www.natomanga.com/home/search/json?searchword=gal_can_t_be_kind
protected override string GetSearchUrl(string keyword)
{
return $"https://www.natomanga.com/home/search/json?searchword={keyword}";
}
protected override MangaSearchResult[] GetSearchResult(NatoMangaSearchResult[] searchResult)
{
IEnumerable<MangaSearchResult> mangaSearchResults = searchResult.Select(searchResult =>
new MangaSearchResult()
{
Title = searchResult.Name,
Url = searchResult.Url
});
return [.. mangaSearchResults];
}
}

View File

@@ -0,0 +1,53 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<None Remove="WebCrawlers\Samples\MangaNato - Please Go Home, Akutsu-San!.htm" />
<None Remove="WebSearch\NatoManga\SampleSearchResult.json" />
</ItemGroup>
<ItemGroup>
<Content Include="WebCrawlers\NatoManga\SampleMangaPage.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="WebCrawlers\Samples\MangaNato - Please Go Home, Akutsu-San!.htm">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="WebSearch\NatoManga\SampleSearchResult.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="Shouldly" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\MangaReader.Core\MangaReader.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,23 @@
using System.Reflection;
using System.Text;
namespace MangaReader.Tests.Utilities;
public static class ResourceHelper
{
/// <summary>
/// Reads an embedded JSON resource from the calling assembly.
/// </summary>
/// <param name="resourceName">The full resource name, e.g. "MyNamespace.Folder.sample.json".</param>
public static async Task<string> ReadJsonResourceAsync(string resourceName)
{
Assembly assmbly = Assembly.GetExecutingAssembly();
using Stream? stream = assmbly.GetManifestResourceStream(resourceName)
?? throw new FileNotFoundException($"Resource '{resourceName}' not found.");
using StreamReader reader = new(stream, Encoding.UTF8);
return await reader.ReadToEndAsync();
}
}

View File

@@ -0,0 +1,52 @@
using MangaReader.Core.WebCrawlers.NatoManga;
using Shouldly;
namespace MangaReader.Tests.WebCrawlers.NatoManga;
public class NatoMangaWebCrawlerTests
{
[Fact]
public void Get_Manga()
{
string sampleFilePath = Path.Combine(AppContext.BaseDirectory, "WebCrawlers", "NatoManga", "SampleMangaPage.html");
var webCrawler = new NatoMangaWebCrawler();
var manga = webCrawler.GetManga(sampleFilePath);
manga.ShouldNotBeNull();
manga.Title.ShouldBe("Gal Cant Be Kind to Otaku!?");
//manga.AlternateTitles.ShouldBe([
// "Kaette kudasai! Akutsu-san",
// "Yankee Musume",
// "ヤンキー娘",
// "帰ってください! 阿久津さん"]);
//manga.Authors.ShouldBe(["Nagaoka Taichi"]);
//manga.Status.ShouldBe(MangaStatus.Ongoing);
manga.Genres.ShouldBe(["Comedy", "Harem", "Romance", "School life", "Seinen", "Slice of life"]);
//manga.UpdateDate.ShouldBe(new DateTime(2024, 9, 26, 0, 12, 0));
//manga.Views.ShouldBe(93_300_000);
//manga.RatingPercent.ShouldBe(97);
//manga.Votes.ShouldBe(15979);
////manga.Description.ShouldStartWith("Ooyama-kun normally doesnt get involved with Akutsu-san, a delinquent girl in his class");
//manga.Description.ShouldStartWith("Ooyama-kun normally doesnt get involved with Akutsu-san, a delinquent girl in his class");
//manga.Description.ShouldEndWith("Artist's Pixiv: https://www.pixiv.net/member.php?id=133935");
manga.Chapters.Count.ShouldBe(83);
manga.Chapters[0].Url.ShouldBe("https://www.natomanga.com/manga/gal-cant-be-kind-to-otaku/chapter-69");
manga.Chapters[0].Number.ShouldBe(69);
manga.Chapters[0].Name.ShouldBe("Chapter 69");
manga.Chapters[0].Views.ShouldBe(8146);
//manga.Chapters[0].UploadDate.ShouldBe(new DateTime(2025, 4, 23, 17, 17, 0));
//manga.Chapters[235].URL.ShouldBe("https://chapmanganato.to/manga-hf984788/chapter-0.1");
//manga.Chapters[235].Number.ShouldBe(0.1f);
//manga.Chapters[235].Name.ShouldBe("Vol.0 Chapter : Oneshot");
//manga.Chapters[235].Views.ShouldBe(232_200);
//manga.Chapters[235].UploadDate.ShouldBe(new DateTime(2021, 8, 24, 1, 8, 0));
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,60 @@
using MangaReader.Core.WebCrawlers;
using MangaReader.Core.WebCrawlers.MangaNato;
using Shouldly;
namespace MangaReader.Tests.WebCrawlers;
public class UnitTest1
{
private readonly string samplesPath;
private readonly string mangaNatoSampleFilePath;
public UnitTest1()
{
samplesPath = Path.Combine(AppContext.BaseDirectory, "WebCrawlers", "Samples");
mangaNatoSampleFilePath = Path.Combine(samplesPath, "MangaNato - Please Go Home, Akutsu-San!.htm");
}
[Fact]
public void Get_Manga()
{
var webCrawler = new MangaNatoWebCrawler();
var manga = webCrawler.GetManga(mangaNatoSampleFilePath);
manga.ShouldNotBeNull();
manga.Title.ShouldBe("Please Go Home, Akutsu-San!");
manga.AlternateTitles.ShouldBe([
"Kaette kudasai! Akutsu-san",
"Yankee Musume",
"ヤンキー娘",
"帰ってください! 阿久津さん"]);
manga.Authors.ShouldBe(["Nagaoka Taichi"]);
manga.Status.ShouldBe(MangaStatus.Ongoing);
manga.Genres.ShouldBe(["Comedy", "Romance", "School life"]);
manga.UpdateDate.ShouldBe(new DateTime(2024, 9, 26, 0, 12, 0));
manga.Views.ShouldBe(93_300_000);
manga.RatingPercent.ShouldBe(97);
manga.Votes.ShouldBe(15979);
//manga.Description.ShouldStartWith("Ooyama-kun normally doesnt get involved with Akutsu-san, a delinquent girl in his class");
manga.Description.ShouldStartWith("Ooyama-kun normally doesnt get involved with Akutsu-san, a delinquent girl in his class");
manga.Description.ShouldEndWith("Artist's Pixiv: https://www.pixiv.net/member.php?id=133935");
manga.Chapters.Count.ShouldBe(236);
manga.Chapters[0].Url.ShouldBe("https://chapmanganato.to/manga-hf984788/chapter-186");
manga.Chapters[0].Number.ShouldBe(186);
manga.Chapters[0].Name.ShouldBe("Chapter 186");
manga.Chapters[0].Views.ShouldBe(37_900);
manga.Chapters[0].UploadDate.ShouldBe(new DateTime(2024, 9, 26, 0, 9, 0));
manga.Chapters[235].Url.ShouldBe("https://chapmanganato.to/manga-hf984788/chapter-0.1");
manga.Chapters[235].Number.ShouldBe(0.1f);
manga.Chapters[235].Name.ShouldBe("Vol.0 Chapter : Oneshot");
manga.Chapters[235].Views.ShouldBe(232_200);
manga.Chapters[235].UploadDate.ShouldBe(new DateTime(2021, 8, 24, 1, 8, 0));
}
}

View File

@@ -0,0 +1,32 @@
using MangaReader.Core.HttpService;
using MangaReader.Core.WebSearch;
using MangaReader.Core.WebSearch.NatoManga;
using MangaReader.Tests.Utilities;
using NSubstitute;
using Shouldly;
namespace MangaReader.Tests.WebSearch.NatoManga;
public class NatoMangaWebSearchTests
{
[Fact]
public async Task Get_Search_Result()
{
string resourceName = "MangaReader.Tests.WebSearch.NatoManga.SampleSearchResult.json";
string searchResultJson = await ResourceHelper.ReadJsonResourceAsync(resourceName);
IHttpService httpService = Substitute.For<IHttpService>();
httpService.GetStringAsync(Arg.Any<string>())
.Returns(Task.FromResult(searchResultJson));
NatoMangaWebSearch webSearch = new(httpService);
MangaSearchResult[] searchResult = await webSearch.SearchAsync("Gals Can't Be Kind");
searchResult.Length.ShouldBe(2);
searchResult[0].Title.ShouldBe("Gal Can't Be Kind to Otaku!");
searchResult[1].Title.ShouldBe("Gal Cant Be Kind to Otaku!?");
searchResult[0].Url.ShouldBe("https://www.natomanga.com/manga/gal-can-t-be-kind-to-otaku");
searchResult[1].Url.ShouldBe("https://www.natomanga.com/manga/gal-cant-be-kind-to-otaku");
}
}

View File

@@ -0,0 +1,20 @@
[
{
"id": 51811,
"author": "Norishiro-chan, Sakana Uozumi",
"name": "Gal Can't Be Kind to Otaku!",
"chapterLatest": "Chapter 69",
"url": "https:\/\/www.natomanga.com\/manga\/gal-can-t-be-kind-to-otaku",
"thumb": "https:\/\/img-r1.2xstorage.com\/thumb\/gal-can-t-be-kind-to-otaku.webp",
"slug": "gal-can-t-be-kind-to-otaku"
},
{
"id": 38065,
"author": "Norishiro-chan,Sakana Uozimi",
"name": "Gal Can\u2019t Be Kind to Otaku!?",
"chapterLatest": "Chapter 69",
"url": "https:\/\/www.natomanga.com\/manga\/gal-cant-be-kind-to-otaku",
"thumb": "https:\/\/img-r1.2xstorage.com\/thumb\/gal-cant-be-kind-to-otaku.webp",
"slug": "gal-cant-be-kind-to-otaku"
}
]

31
MangaReader.sln Normal file
View File

@@ -0,0 +1,31 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.11.35303.130
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MangaReader.Core", "MangaReader.Core\MangaReader.Core.csproj", "{D1C7616F-8794-4F52-9694-6814BD368F0B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MangaReader.Tests", "MangaReader.Tests\MangaReader.Tests.csproj", "{D86F1282-485A-4FF2-A75A-AB8102F3C853}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D1C7616F-8794-4F52-9694-6814BD368F0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D1C7616F-8794-4F52-9694-6814BD368F0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D1C7616F-8794-4F52-9694-6814BD368F0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D1C7616F-8794-4F52-9694-6814BD368F0B}.Release|Any CPU.Build.0 = Release|Any CPU
{D86F1282-485A-4FF2-A75A-AB8102F3C853}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D86F1282-485A-4FF2-A75A-AB8102F3C853}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D86F1282-485A-4FF2-A75A-AB8102F3C853}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D86F1282-485A-4FF2-A75A-AB8102F3C853}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {6E11B5CA-67EA-463B-B6AE-1D2065A04F76}
EndGlobalSection
EndGlobal