From e6d6b629dbef8f834d62139fa19d8da091891da0 Mon Sep 17 00:00:00 2001 From: littlefoot Date: Fri, 15 Jul 2022 12:21:37 -0400 Subject: [PATCH] db changes and build pipeline --- .drone.yml | 32 +++ ...0715040739_AddLastContentFetch.Designer.cs | 253 +++++++++++++++++ .../20220715040739_AddLastContentFetch.cs | 27 ++ ...e some chapter fields optional.Designer.cs | 253 +++++++++++++++++ ...35707_make some chapter fields optional.cs | 70 +++++ ...220715143230_add id for novels.Designer.cs | 257 ++++++++++++++++++ .../20220715143230_add id for novels.cs | 27 ++ .../Migrations/AppDbContextModelSnapshot.cs | 25 +- DBConnection/Models/Chapter.cs | 5 +- DBConnection/Models/Novel.cs | 2 + DBConnection/Repositories/BaseRepository.cs | 5 + .../Interfaces/INovelRepository.cs | 2 +- .../Repositories/Interfaces/IRepository.cs | 1 + DBConnection/Repositories/NovelRepository.cs | 29 +- Shared/AccessLayers/ApiAccessLayer.cs | 15 +- .../AccessLayers/WebApiAccessLayer.cs | 28 ++ WebNovelPortal/Pages/Index.razor | 53 +++- WebNovelPortal/Pages/NovelDetails.razor | 47 ++++ WebNovelPortal/Program.cs | 2 + .../Components/Layout/NavMenu.razor.css | 28 +- .../Shared/Layouts/MainLayout.razor.css | 22 +- WebNovelPortal/WebNovelPortal.csproj | 11 +- WebNovelPortal/appsettings.json | 3 +- .../Controllers/NovelController.cs | 13 + .../Extensions/DBUpdateExtensions.cs | 17 ++ .../Extensions/ScraperExtensions.cs | 2 +- WebNovelPortalAPI/Program.cs | 2 +- WebNovelPortalAPI/Scrapers/AbstractScraper.cs | 113 ++++++++ WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs | 92 +------ WebNovelPortalAPI/Scrapers/SyosetuScraper.cs | 28 ++ 30 files changed, 1332 insertions(+), 132 deletions(-) create mode 100644 .drone.yml create mode 100644 DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs create mode 100644 DBConnection/Migrations/20220715040739_AddLastContentFetch.cs create mode 100644 DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs create mode 100644 DBConnection/Migrations/20220715135707_make some chapter fields optional.cs create mode 100644 DBConnection/Migrations/20220715143230_add id for novels.Designer.cs create mode 100644 DBConnection/Migrations/20220715143230_add id for novels.cs create mode 100644 WebNovelPortal/AccessLayers/WebApiAccessLayer.cs create mode 100644 WebNovelPortal/Pages/NovelDetails.razor create mode 100644 WebNovelPortalAPI/Extensions/DBUpdateExtensions.cs create mode 100644 WebNovelPortalAPI/Scrapers/AbstractScraper.cs create mode 100644 WebNovelPortalAPI/Scrapers/SyosetuScraper.cs diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..1587f77 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,32 @@ +kind: pipeline +name: default + +steps: +- name: build + image: mcr.microsoft.com/dotnet/sdk:6.0 + + commands: + - dotnet restore *.sln + - mkdir build + - mkdir publish + - mkdir publish/api + - mkdir publish/frontend + - mkdir dist + - dotnet build *.sln -c Release -o build + - dotnet publish WebNovelPortalAPI/*.csproj -c Release -o publish/api + - dotnet publish WebNovelPortal/*.csproj -c Release -o publish/frontend + - tar -czvf dist/API.tar.gz publish/api/* + - tar -czvf dist/Frontend.tar.gz publish/frontend/* + + + +- name: gitea_release + image: plugins/gitea-release + settings: + api_key: + from_secret: gitea-api-key + base_url: https://git.orfl.xyz + files: + - dist/* + when: + event: tag \ No newline at end of file diff --git a/DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs b/DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs new file mode 100644 index 0000000..b1b50a7 --- /dev/null +++ b/DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs @@ -0,0 +1,253 @@ +// +using System; +using DBConnection; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DBConnection.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20220715040739_AddLastContentFetch")] + partial class AddLastContentFetch + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("ChapterNumber") + .HasColumnType("INTEGER"); + + b.Property("Content") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("DateUpdated") + .HasColumnType("TEXT"); + + b.Property("LastContentFetch") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("RawContent") + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("NovelUrl"); + + b.ToTable("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("AuthorUrl") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("AuthorUrl"); + + b.ToTable("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Tag", b => + { + b.Property("TagValue") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.HasKey("TagValue"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Email") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.Property("LastChapterRead") + .HasColumnType("INTEGER"); + + b.HasKey("NovelUrl", "UserId"); + + b.HasIndex("UserId"); + + b.ToTable("UserNovels"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany("Chapters") + .HasForeignKey("NovelUrl"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.HasOne("DBConnection.Models.Author", "Author") + .WithMany("Novels") + .HasForeignKey("AuthorUrl"); + + b.Navigation("Author"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.HasOne("DBConnection.Models.Novel", "Novel") + .WithMany() + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.User", "User") + .WithMany("WatchedNovels") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Novel"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Navigation("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Navigation("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Navigation("WatchedNovels"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs b/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs new file mode 100644 index 0000000..9e45e0d --- /dev/null +++ b/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs @@ -0,0 +1,27 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DBConnection.Migrations +{ + public partial class AddLastContentFetch : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "LastContentFetch", + table: "Chapters", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "LastContentFetch", + table: "Chapters"); + } + } +} diff --git a/DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs b/DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs new file mode 100644 index 0000000..f1d5f11 --- /dev/null +++ b/DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs @@ -0,0 +1,253 @@ +// +using System; +using DBConnection; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DBConnection.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20220715135707_make some chapter fields optional")] + partial class makesomechapterfieldsoptional + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("ChapterNumber") + .HasColumnType("INTEGER"); + + b.Property("Content") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("DateUpdated") + .HasColumnType("TEXT"); + + b.Property("LastContentFetch") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("RawContent") + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("NovelUrl"); + + b.ToTable("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("AuthorUrl") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("AuthorUrl"); + + b.ToTable("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Tag", b => + { + b.Property("TagValue") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.HasKey("TagValue"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Email") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.Property("LastChapterRead") + .HasColumnType("INTEGER"); + + b.HasKey("NovelUrl", "UserId"); + + b.HasIndex("UserId"); + + b.ToTable("UserNovels"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany("Chapters") + .HasForeignKey("NovelUrl"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.HasOne("DBConnection.Models.Author", "Author") + .WithMany("Novels") + .HasForeignKey("AuthorUrl"); + + b.Navigation("Author"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.HasOne("DBConnection.Models.Novel", "Novel") + .WithMany() + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.User", "User") + .WithMany("WatchedNovels") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Novel"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Navigation("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Navigation("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Navigation("WatchedNovels"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs b/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs new file mode 100644 index 0000000..51ea2f6 --- /dev/null +++ b/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs @@ -0,0 +1,70 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DBConnection.Migrations +{ + public partial class makesomechapterfieldsoptional : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "LastContentFetch", + table: "Chapters", + type: "TEXT", + nullable: true, + oldClrType: typeof(DateTime), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "DateUpdated", + table: "Chapters", + type: "TEXT", + nullable: true, + oldClrType: typeof(DateTime), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "DatePosted", + table: "Chapters", + type: "TEXT", + nullable: true, + oldClrType: typeof(DateTime), + oldType: "TEXT"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "LastContentFetch", + table: "Chapters", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + oldClrType: typeof(DateTime), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "DateUpdated", + table: "Chapters", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + oldClrType: typeof(DateTime), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "DatePosted", + table: "Chapters", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + oldClrType: typeof(DateTime), + oldType: "TEXT", + oldNullable: true); + } + } +} diff --git a/DBConnection/Migrations/20220715143230_add id for novels.Designer.cs b/DBConnection/Migrations/20220715143230_add id for novels.Designer.cs new file mode 100644 index 0000000..ca6d10b --- /dev/null +++ b/DBConnection/Migrations/20220715143230_add id for novels.Designer.cs @@ -0,0 +1,257 @@ +// +using System; +using DBConnection; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DBConnection.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20220715143230_add id for novels")] + partial class addidfornovels + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("ChapterNumber") + .HasColumnType("INTEGER"); + + b.Property("Content") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("DateUpdated") + .HasColumnType("TEXT"); + + b.Property("LastContentFetch") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("RawContent") + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("NovelUrl"); + + b.ToTable("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Property("Url") + .HasColumnType("TEXT"); + + b.Property("AuthorUrl") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("DatePosted") + .HasColumnType("TEXT"); + + b.Property("Guid") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Url"); + + b.HasIndex("AuthorUrl"); + + b.ToTable("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Tag", b => + { + b.Property("TagValue") + .HasColumnType("TEXT"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.HasKey("TagValue"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DateCreated") + .HasColumnType("TEXT"); + + b.Property("DateModified") + .HasColumnType("TEXT"); + + b.Property("Email") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.Property("NovelUrl") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.Property("LastChapterRead") + .HasColumnType("INTEGER"); + + b.HasKey("NovelUrl", "UserId"); + + b.HasIndex("UserId"); + + b.ToTable("UserNovels"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("DBConnection.Models.Chapter", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany("Chapters") + .HasForeignKey("NovelUrl"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.HasOne("DBConnection.Models.Author", "Author") + .WithMany("Novels") + .HasForeignKey("AuthorUrl"); + + b.Navigation("Author"); + }); + + modelBuilder.Entity("DBConnection.Models.UserNovel", b => + { + b.HasOne("DBConnection.Models.Novel", "Novel") + .WithMany() + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.User", "User") + .WithMany("WatchedNovels") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Novel"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.HasOne("DBConnection.Models.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DBConnection.Models.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DBConnection.Models.Author", b => + { + b.Navigation("Novels"); + }); + + modelBuilder.Entity("DBConnection.Models.Novel", b => + { + b.Navigation("Chapters"); + }); + + modelBuilder.Entity("DBConnection.Models.User", b => + { + b.Navigation("WatchedNovels"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DBConnection/Migrations/20220715143230_add id for novels.cs b/DBConnection/Migrations/20220715143230_add id for novels.cs new file mode 100644 index 0000000..a228c04 --- /dev/null +++ b/DBConnection/Migrations/20220715143230_add id for novels.cs @@ -0,0 +1,27 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DBConnection.Migrations +{ + public partial class addidfornovels : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "Guid", + table: "Novels", + type: "TEXT", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "Guid", + table: "Novels"); + } + } +} diff --git a/DBConnection/Migrations/AppDbContextModelSnapshot.cs b/DBConnection/Migrations/AppDbContextModelSnapshot.cs index 3c23927..c300c69 100644 --- a/DBConnection/Migrations/AppDbContextModelSnapshot.cs +++ b/DBConnection/Migrations/AppDbContextModelSnapshot.cs @@ -34,7 +34,7 @@ namespace DBConnection.Migrations b.HasKey("Url"); - b.ToTable("Authors"); + b.ToTable("Authors", (string)null); }); modelBuilder.Entity("DBConnection.Models.Chapter", b => @@ -54,10 +54,13 @@ namespace DBConnection.Migrations b.Property("DateModified") .HasColumnType("TEXT"); - b.Property("DatePosted") + b.Property("DatePosted") .HasColumnType("TEXT"); - b.Property("DateUpdated") + b.Property("DateUpdated") + .HasColumnType("TEXT"); + + b.Property("LastContentFetch") .HasColumnType("TEXT"); b.Property("Name") @@ -74,7 +77,7 @@ namespace DBConnection.Migrations b.HasIndex("NovelUrl"); - b.ToTable("Chapters"); + b.ToTable("Chapters", (string)null); }); modelBuilder.Entity("DBConnection.Models.Novel", b => @@ -94,6 +97,10 @@ namespace DBConnection.Migrations b.Property("DatePosted") .HasColumnType("TEXT"); + b.Property("Guid") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + b.Property("LastUpdated") .HasColumnType("TEXT"); @@ -105,7 +112,7 @@ namespace DBConnection.Migrations b.HasIndex("AuthorUrl"); - b.ToTable("Novels"); + b.ToTable("Novels", (string)null); }); modelBuilder.Entity("DBConnection.Models.Tag", b => @@ -121,7 +128,7 @@ namespace DBConnection.Migrations b.HasKey("TagValue"); - b.ToTable("Tags"); + b.ToTable("Tags", (string)null); }); modelBuilder.Entity("DBConnection.Models.User", b => @@ -142,7 +149,7 @@ namespace DBConnection.Migrations b.HasKey("Id"); - b.ToTable("Users"); + b.ToTable("Users", (string)null); }); modelBuilder.Entity("DBConnection.Models.UserNovel", b => @@ -160,7 +167,7 @@ namespace DBConnection.Migrations b.HasIndex("UserId"); - b.ToTable("UserNovels"); + b.ToTable("UserNovels", (string)null); }); modelBuilder.Entity("NovelTag", b => @@ -175,7 +182,7 @@ namespace DBConnection.Migrations b.HasIndex("TagsTagValue"); - b.ToTable("NovelTag"); + b.ToTable("NovelTag", (string)null); }); modelBuilder.Entity("DBConnection.Models.Chapter", b => diff --git a/DBConnection/Models/Chapter.cs b/DBConnection/Models/Chapter.cs index 38698a4..c379e08 100644 --- a/DBConnection/Models/Chapter.cs +++ b/DBConnection/Models/Chapter.cs @@ -10,6 +10,7 @@ public class Chapter : BaseEntity public string? RawContent { get; set; } [Key] public string Url { get; set; } - public DateTime DatePosted { get; set; } - public DateTime DateUpdated { get; set; } + public DateTime? DatePosted { get; set; } + public DateTime? DateUpdated { get; set; } + public DateTime? LastContentFetch { get; set; } } \ No newline at end of file diff --git a/DBConnection/Models/Novel.cs b/DBConnection/Models/Novel.cs index f0cd5cb..39c14e0 100644 --- a/DBConnection/Models/Novel.cs +++ b/DBConnection/Models/Novel.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; namespace DBConnection.Models; @@ -6,6 +7,7 @@ public class Novel : BaseEntity { [Key] public string Url { get; set; } + public Guid Guid { get; set; } public string Title { get; set; } public Author Author { get; set; } public List Tags { get; set; } diff --git a/DBConnection/Repositories/BaseRepository.cs b/DBConnection/Repositories/BaseRepository.cs index 71ed45d..92ac496 100644 --- a/DBConnection/Repositories/BaseRepository.cs +++ b/DBConnection/Repositories/BaseRepository.cs @@ -47,6 +47,11 @@ public abstract class BaseRepository : IRepository whe return entity; } + public virtual async Task> GetAllIncluded() + { + return await GetWhereIncluded(i => true); + } + public virtual async Task GetIncluded(TEntityType entity) { return await GetIncluded(dbEntity => GetPrimaryKey(dbEntity).SequenceEqual(GetPrimaryKey(entity))); diff --git a/DBConnection/Repositories/Interfaces/INovelRepository.cs b/DBConnection/Repositories/Interfaces/INovelRepository.cs index 06dea71..f638715 100644 --- a/DBConnection/Repositories/Interfaces/INovelRepository.cs +++ b/DBConnection/Repositories/Interfaces/INovelRepository.cs @@ -4,5 +4,5 @@ namespace DBConnection.Repositories.Interfaces; public interface INovelRepository : IRepository { - + Task GetNovel(Guid guid); } \ No newline at end of file diff --git a/DBConnection/Repositories/Interfaces/IRepository.cs b/DBConnection/Repositories/Interfaces/IRepository.cs index 2d220f2..902c953 100644 --- a/DBConnection/Repositories/Interfaces/IRepository.cs +++ b/DBConnection/Repositories/Interfaces/IRepository.cs @@ -14,4 +14,5 @@ public interface IRepository : IRepository where TEntityType : Base Task GetIncluded(TEntityType entity); Task GetIncluded(Func predicate); Task> GetWhereIncluded(Func predicate); + Task> GetAllIncluded(); } \ No newline at end of file diff --git a/DBConnection/Repositories/NovelRepository.cs b/DBConnection/Repositories/NovelRepository.cs index bfa2548..2fe79fd 100644 --- a/DBConnection/Repositories/NovelRepository.cs +++ b/DBConnection/Repositories/NovelRepository.cs @@ -17,16 +17,38 @@ public class NovelRepository : BaseRepository, INovelRepository public override async Task Upsert(Novel entity) { var dbEntity = await GetIncluded(entity) ?? entity; + // Author dbEntity.Author = await _authorRepository.GetIncluded(entity.Author) ?? entity.Author; + //Tags List newTags = new List(); - foreach (var tag in dbEntity.Tags) + foreach (var tag in entity.Tags) { newTags.Add(await _tagRepository.GetIncluded(tag) ?? tag); } dbEntity.Tags.Clear(); dbEntity.Tags = newTags; + //chapters + var newChapters = new List(); + foreach (var chapter in entity.Chapters.ToList()) + { + var existingChapter = await DbContext.Chapters.FindAsync(chapter.Url); + if (existingChapter == null) + { + newChapters.Add(chapter); + } + else + { + existingChapter.Name = chapter.Name; + existingChapter.DateUpdated = chapter.DateUpdated; + newChapters.Add(existingChapter); + } + } + dbEntity.Chapters.Clear(); + dbEntity.Chapters = newChapters; + // update in db if (DbContext.Entry(dbEntity).State == EntityState.Detached) { + dbEntity.Guid = Guid.NewGuid(); DbContext.Add(dbEntity); } @@ -42,6 +64,11 @@ public class NovelRepository : BaseRepository, INovelRepository .Include(i => i.Tags); } + public async Task GetNovel(Guid guid) + { + return await GetIncluded(i => i.Guid == guid); + } + public async Task GetNovel(string url) { return await GetIncluded(i => i.Url == url); diff --git a/Shared/AccessLayers/ApiAccessLayer.cs b/Shared/AccessLayers/ApiAccessLayer.cs index 4410178..daeb105 100644 --- a/Shared/AccessLayers/ApiAccessLayer.cs +++ b/Shared/AccessLayers/ApiAccessLayer.cs @@ -1,3 +1,5 @@ +using System.Net.Mime; +using System.Text; using Microsoft.AspNetCore.WebUtilities; using Newtonsoft.Json; using Shared.Models; @@ -10,8 +12,12 @@ public abstract class ApiAccessLayer protected ApiAccessLayer(string apiBaseUrl) { - _httpClient = new HttpClient(); - _httpClient.BaseAddress = new Uri(apiBaseUrl); + var handler = new HttpClientHandler() + { + ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator + }; + _httpClient = new HttpClient(handler); + _httpClient.BaseAddress = new Uri(apiBaseUrl, UriKind.Absolute); } private async Task SendRequest(HttpRequestMessage message) @@ -53,12 +59,11 @@ public abstract class ApiAccessLayer { uri = QueryHelpers.AddQueryString(endpoint, queryParams); } - - message.RequestUri = new Uri(uri); + message.RequestUri = new Uri(uri, UriKind.Relative); message.Method = method; if (data != null) { - message.Content = new StringContent(JsonConvert.SerializeObject(data)); + message.Content = new StringContent(JsonConvert.SerializeObject(data), Encoding.UTF8, MediaTypeNames.Application.Json); } return message; diff --git a/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs b/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs new file mode 100644 index 0000000..6d0b058 --- /dev/null +++ b/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs @@ -0,0 +1,28 @@ +using DBConnection.Models; +using Shared.AccessLayers; +using WebNovelPortalAPI.DTO; + +namespace WebNovelPortal.AccessLayers; + +public class WebApiAccessLayer : ApiAccessLayer +{ + public WebApiAccessLayer(string apiBaseUrl) : base(apiBaseUrl) + { + } + + public async Task?> GetNovels() + { + return (await SendRequest>("novel", HttpMethod.Get)).ResponseObject; + } + + public async Task RequestNovelScrape(string url) + { + return (await SendRequest("novel/scrapeNovel", HttpMethod.Post, null, + new ScrapeNovelRequest {NovelUrl = url})).ResponseObject; + } + + public async Task GetNovel(string guid) + { + return (await SendRequest($"novel/{guid}", HttpMethod.Get)).ResponseObject; + } +} \ No newline at end of file diff --git a/WebNovelPortal/Pages/Index.razor b/WebNovelPortal/Pages/Index.razor index 800daa8..ec56c1a 100644 --- a/WebNovelPortal/Pages/Index.razor +++ b/WebNovelPortal/Pages/Index.razor @@ -1,7 +1,56 @@ @page "/" +@using DBConnection.Models +@using WebNovelPortal.AccessLayers Index +

Novels

+ + + + + + + + @foreach (var novel in novels) + { + + + + + + } +
TitleAuthorChapter Count
+ @novel.Title + + @novel.Author.Name + + @novel.Chapters.Count +
-

Hello, world!

-Welcome to your new app. \ No newline at end of file +@code { + [Inject] + WebApiAccessLayer api { get; set; } + string NovelUrl { get; set; } + List novels = new List(); + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + await RefreshNovels(); + } + } + + async Task RequestNovelScrape(string url) + { + await api.RequestNovelScrape(url); + await RefreshNovels(); + } + + async Task RefreshNovels() + { + novels = await api.GetNovels(); + StateHasChanged(); + } + +} \ No newline at end of file diff --git a/WebNovelPortal/Pages/NovelDetails.razor b/WebNovelPortal/Pages/NovelDetails.razor new file mode 100644 index 0000000..a64f7e0 --- /dev/null +++ b/WebNovelPortal/Pages/NovelDetails.razor @@ -0,0 +1,47 @@ +@page "/novel/{NovelId}" +@using WebNovelPortal.AccessLayers +@using DBConnection.Models +@if (Novel == null) +{ +

Loading...

+} +else +{ +

@(Novel.Title)

+

Author: (@Novel.Author.Name)

+

Date Posted: @Novel.DatePosted

+

Date Updated: @Novel.LastUpdated

+

Tags

+
    + @foreach (var tag in Novel.Tags) + { +
  • @tag.TagValue
    • + } +
+

Chapters

+
    + @foreach (var chapter in Novel.Chapters) + { +
  1. @chapter.Name
    2. + } +
+} + +@code { + [Parameter] + public string? NovelId { get; set; } + [Inject] + public WebApiAccessLayer api { get; set; } + Novel? Novel { get; set; } + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + Novel = await api.GetNovel(NovelId); + StateHasChanged(); + } + await base.OnAfterRenderAsync(firstRender); + } + +} \ No newline at end of file diff --git a/WebNovelPortal/Program.cs b/WebNovelPortal/Program.cs index 545e7b4..ca5ba2d 100644 --- a/WebNovelPortal/Program.cs +++ b/WebNovelPortal/Program.cs @@ -1,9 +1,11 @@ using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; +using WebNovelPortal.AccessLayers; var builder = WebApplication.CreateBuilder(args); // Add services to the container. +builder.Services.AddScoped(fac => new WebApiAccessLayer(builder.Configuration["WebAPIUrl"])); builder.Services.AddRazorPages(); builder.Services.AddServerSideBlazor(); diff --git a/WebNovelPortal/Shared/Components/Layout/NavMenu.razor.css b/WebNovelPortal/Shared/Components/Layout/NavMenu.razor.css index acc5f9f..4c3379c 100644 --- a/WebNovelPortal/Shared/Components/Layout/NavMenu.razor.css +++ b/WebNovelPortal/Shared/Components/Layout/NavMenu.razor.css @@ -23,22 +23,22 @@ padding-bottom: 0.5rem; } - .nav-item:first-of-type { - padding-top: 1rem; - } +.nav-item:first-of-type { + padding-top: 1rem; +} - .nav-item:last-of-type { - padding-bottom: 1rem; - } +.nav-item:last-of-type { + padding-bottom: 1rem; +} - .nav-item ::deep a { - color: #d7d7d7; - border-radius: 4px; - height: 3rem; - display: flex; - align-items: center; - line-height: 3rem; - } +.nav-item ::deep a { + color: #d7d7d7; + border-radius: 4px; + height: 3rem; + display: flex; + align-items: center; + line-height: 3rem; +} .nav-item ::deep a.active { background-color: rgba(255,255,255,0.25); diff --git a/WebNovelPortal/Shared/Layouts/MainLayout.razor.css b/WebNovelPortal/Shared/Layouts/MainLayout.razor.css index 551e4b2..955e64c 100644 --- a/WebNovelPortal/Shared/Layouts/MainLayout.razor.css +++ b/WebNovelPortal/Shared/Layouts/MainLayout.razor.css @@ -1,10 +1,10 @@ -.page { +.page { position: relative; display: flex; flex-direction: column; } -main { +.main { flex: 1; } @@ -21,15 +21,15 @@ main { align-items: center; } - .top-row ::deep a, .top-row .btn-link { - white-space: nowrap; - margin-left: 1.5rem; - } +.top-row ::deep a, .top-row .btn-link { + white-space: nowrap; + margin-left: 1.5rem; +} - .top-row a:first-child { - overflow: hidden; - text-overflow: ellipsis; - } +.top-row a:first-child { + overflow: hidden; + text-overflow: ellipsis; +} @media (max-width: 640.98px) { .top-row:not(.auth) { @@ -63,7 +63,7 @@ main { z-index: 1; } - .top-row, article { + .main > div { padding-left: 2rem !important; padding-right: 1.5rem !important; } diff --git a/WebNovelPortal/WebNovelPortal.csproj b/WebNovelPortal/WebNovelPortal.csproj index d152de7..011d7b8 100644 --- a/WebNovelPortal/WebNovelPortal.csproj +++ b/WebNovelPortal/WebNovelPortal.csproj @@ -7,13 +7,14 @@ Linux - - - - - + + + + + + diff --git a/WebNovelPortal/appsettings.json b/WebNovelPortal/appsettings.json index 10f68b8..9b02a3a 100644 --- a/WebNovelPortal/appsettings.json +++ b/WebNovelPortal/appsettings.json @@ -5,5 +5,6 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "WebAPIUrl": "https://localhost:7137/api/" } diff --git a/WebNovelPortalAPI/Controllers/NovelController.cs b/WebNovelPortalAPI/Controllers/NovelController.cs index 2ae8eff..8343566 100644 --- a/WebNovelPortalAPI/Controllers/NovelController.cs +++ b/WebNovelPortalAPI/Controllers/NovelController.cs @@ -31,6 +31,19 @@ namespace WebNovelPortalAPI.Controllers return _scrapers.FirstOrDefault(i => i.MatchesUrl(novelUrl)); } + [HttpGet] + [Route("{guid:guid}")] + public async Task GetNovel(Guid guid) + { + return await _novelRepository.GetNovel(guid); + } + + [HttpGet] + public async Task> GetNovels() + { + return (await _novelRepository.GetAllIncluded()).ToList(); + } + [HttpPost] [Route("scrapeNovel")] public async Task ScrapeNovel(ScrapeNovelRequest request) diff --git a/WebNovelPortalAPI/Extensions/DBUpdateExtensions.cs b/WebNovelPortalAPI/Extensions/DBUpdateExtensions.cs new file mode 100644 index 0000000..a4d8c1e --- /dev/null +++ b/WebNovelPortalAPI/Extensions/DBUpdateExtensions.cs @@ -0,0 +1,17 @@ +using Microsoft.EntityFrameworkCore; + +namespace WebNovelPortalAPI.Extensions; + +public static class DBUpdateExtensions +{ + public static void UpdateDatabase(this IApplicationBuilder app) where T : DbContext + { + using var serviceScope = app.ApplicationServices.CreateScope(); + using var context = serviceScope.ServiceProvider.GetService(); + + if (context.Database.GetPendingMigrations().Any()) + { + context.Database.Migrate(); + } + } +} \ No newline at end of file diff --git a/WebNovelPortalAPI/Extensions/ScraperExtensions.cs b/WebNovelPortalAPI/Extensions/ScraperExtensions.cs index bec3e06..8a9af5f 100644 --- a/WebNovelPortalAPI/Extensions/ScraperExtensions.cs +++ b/WebNovelPortalAPI/Extensions/ScraperExtensions.cs @@ -8,7 +8,7 @@ public static class ScraperExtensions public static void AddScrapers(this IServiceCollection services) { Type[] types = Assembly.GetExecutingAssembly().GetTypes().Where(t => - t.IsClass && typeof(IScraper).IsAssignableFrom(t) && (t.Namespace?.Contains(nameof(Scrapers)) ?? false)) + t.IsClass && typeof(IScraper).IsAssignableFrom(t) && !t.IsAbstract && (t.Namespace?.Contains(nameof(Scrapers)) ?? false)) .ToArray(); foreach (var t in types) { diff --git a/WebNovelPortalAPI/Program.cs b/WebNovelPortalAPI/Program.cs index 559d52f..d17286c 100644 --- a/WebNovelPortalAPI/Program.cs +++ b/WebNovelPortalAPI/Program.cs @@ -19,7 +19,7 @@ builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); var app = builder.Build(); - +app.UpdateDatabase(); // Configure the HTTP request pipeline. if (app.Environment.IsDevelopment()) { diff --git a/WebNovelPortalAPI/Scrapers/AbstractScraper.cs b/WebNovelPortalAPI/Scrapers/AbstractScraper.cs new file mode 100644 index 0000000..3fd9a58 --- /dev/null +++ b/WebNovelPortalAPI/Scrapers/AbstractScraper.cs @@ -0,0 +1,113 @@ +using System.Text.RegularExpressions; +using DBConnection.Models; +using HtmlAgilityPack; + +namespace WebNovelPortalAPI.Scrapers; + +public abstract class AbstractScraper : IScraper +{ + protected abstract string UrlMatchPattern { get; } + protected abstract string BaseUrlPattern { get; } + protected virtual string? WorkTitlePattern { get; } + protected virtual string? AuthorNamePattern { get; } + protected virtual string? AuthorLinkPattern { get; } + protected virtual string? ChapterUrlPattern { get; } + protected virtual string? ChapterNamePattern { get; } + protected virtual string? ChapterPostedPattern { get; } + protected virtual string? ChapterUpdatedPattern { get; } + protected virtual string? TagPattern { get; } + protected virtual string? DatePostedPattern { get; } + protected virtual string? DateUpdatedPattern { get; } + + public virtual bool MatchesUrl(string url) + { + var regex = new Regex(UrlMatchPattern, RegexOptions.IgnoreCase); + return regex.IsMatch(url); + } + + protected virtual string GetNovelTitle(HtmlDocument document) + { + var xpath = WorkTitlePattern; + return document.DocumentNode.SelectSingleNode(xpath).InnerText; + } + + protected virtual Author GetAuthor(HtmlDocument document, string baseUrl) + { + var nameXPath = AuthorNamePattern; + var urlXPath = AuthorLinkPattern; + var authorName = document.DocumentNode.SelectSingleNode(nameXPath).InnerText; + var authorUrl = document.DocumentNode.SelectSingleNode(urlXPath).Attributes["href"].Value; + Author author = new Author + { + Name = authorName, + Url = $"{baseUrl + authorUrl}" + }; + return author; + + } + + protected virtual List GetChapters(HtmlDocument document, string baseUrl) + { + var urlxpath = ChapterUrlPattern; + var namexpath = ChapterNamePattern; + var urlnodes = document.DocumentNode.SelectNodes(urlxpath); + var chapters = urlnodes.Select((node, i) => new Chapter + { + ChapterNumber = i + 1, + Url = $"{baseUrl}{node.Attributes["href"].Value}", + Name = node.SelectSingleNode(namexpath).InnerText + }); + + return chapters.ToList(); + } + + protected virtual List GetTags(HtmlDocument document) + { + var xpath = TagPattern; + var nodes = document.DocumentNode.SelectNodes(xpath); + return nodes.Select(node => new Tag + { + TagValue = node.InnerText + }).ToList(); + } + + protected virtual DateTime GetPostedDate(HtmlDocument document) + { + var xpath = DatePostedPattern; + return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); + } + + protected virtual DateTime GetLastUpdatedDate(HtmlDocument document) + { + var xpath = DateUpdatedPattern; + return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); + } + + public Novel ScrapeNovel(string url) + { + var web = new HtmlWeb(); + var doc = web.Load(url); + if (doc == null) + { + throw new Exception("Error parsing document"); + } + + var baseUrl = new Regex(BaseUrlPattern).Match(url).Value; + var novelUrl = new Regex(UrlMatchPattern).Match(url).Value; + return new Novel + { + Author = GetAuthor(doc, baseUrl), + Chapters = GetChapters(doc, baseUrl), + DatePosted = GetPostedDate(doc), + LastUpdated = GetLastUpdatedDate(doc), + Tags = GetTags(doc), + Title = GetNovelTitle(doc), + Url = novelUrl + }; + } + + public string? ScrapeChapterContent(string chapterUrl) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs b/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs index 6249b54..1eccb0c 100644 --- a/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs +++ b/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs @@ -5,95 +5,29 @@ using HtmlAgilityPack; namespace WebNovelPortalAPI.Scrapers; -public class KakuyomuScraper : IScraper +public class KakuyomuScraper : AbstractScraper { - private const string UrlPattern = @"https?:\/\/kakuyomu\.jp\/works\/\d+\/?"; - private const string BaseUrl = "https://kakuyomu.jp"; - public bool MatchesUrl(string url) - { - var regex = new Regex(UrlPattern, RegexOptions.IgnoreCase); - return regex.IsMatch(url); - } + protected override string UrlMatchPattern => @"https?:\/\/kakuyomu\.jp\/works\/\d+\/?"; - private string GetNovelTitle(HtmlDocument document) - { - var xpath = @"//*[@id='workTitle']/a"; - return document.DocumentNode.SelectSingleNode(xpath).InnerText; - } + protected override string BaseUrlPattern => @"https?:\/\/kakuyomu\.jp"; - private Author GetAuthor(HtmlDocument document) - { - var nameXPath = @"//*[@id='workAuthor-activityName']/a"; - var urlXPath = @"//*[@id='workAuthor-activityName']/a"; - var authorName = document.DocumentNode.SelectSingleNode(nameXPath).InnerText; - var authorUrl = document.DocumentNode.SelectSingleNode(urlXPath).Attributes["href"].Value; - Author author = new Author - { - Name = authorName, - Url = $"{BaseUrl + authorUrl}" - }; - return author; + protected override string? WorkTitlePattern => @"//*[@id='workTitle']/a"; + protected override string? AuthorNamePattern => @"//*[@id='workAuthor-activityName']/a"; + protected override string? AuthorLinkPattern => @"//*[@id='workAuthor-activityName']/a"; - } + protected override string? ChapterUrlPattern => @"//a[@class='widget-toc-episode-episodeTitle']"; - private List GetChapters(HtmlDocument document) - { - var urlxpath = @"//a[@class='widget-toc-episode-episodeTitle']"; - var namexpath = @"span"; - var urlnodes = document.DocumentNode.SelectNodes(urlxpath); - var chapters = urlnodes.Select((node, i) => new Chapter - { - ChapterNumber = i + 1, - Url = $"{BaseUrl}{node.Attributes["href"].Value}", - Name = node.SelectSingleNode(namexpath).InnerText - }); + protected override string? ChapterNamePattern => @"span"; - return chapters.ToList(); - } + protected override string? ChapterPostedPattern => base.ChapterPostedPattern; - private List GetTags(HtmlDocument document) - { - var xpath = @"//span[@itemprop='keywords']/a"; - var nodes = document.DocumentNode.SelectNodes(xpath); - return nodes.Select(node => new Tag - { - TagValue = node.InnerText - }).ToList(); - } + protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern; - private DateTime GetPostedDate(HtmlDocument document) - { - var xpath = @"//time[@itemprop='datePublished']"; - return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); - } + protected override string? TagPattern => @"//span[@itemprop='keywords']/a"; - private DateTime GetLastUpdatedDate(HtmlDocument document) - { - var xpath = @"//time[@itemprop='dateModified']"; - return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); - } + protected override string? DatePostedPattern => @"//time[@itemprop='datePublished']"; - public Novel ScrapeNovel(string url) - { - Novel novel = new Novel(); - var web = new HtmlWeb(); - var doc = web.Load(url); - if (doc == null) - { - throw new Exception("Error parsing document"); - } - - return new Novel - { - Author = GetAuthor(doc), - Chapters = GetChapters(doc), - DatePosted = GetPostedDate(doc), - LastUpdated = GetLastUpdatedDate(doc), - Tags = GetTags(doc), - Title = GetNovelTitle(doc), - Url = url - }; - } + protected override string? DateUpdatedPattern => @"//time[@itemprop='dateModified']"; public string? ScrapeChapterContent(string chapterUrl) { diff --git a/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs b/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs new file mode 100644 index 0000000..c0decde --- /dev/null +++ b/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs @@ -0,0 +1,28 @@ +namespace WebNovelPortalAPI.Scrapers; + +public class SyosetuScraper : AbstractScraper +{ + protected override string UrlMatchPattern => @"https?:\/\/\w+\.syosetu\.com\/\w+\/?"; + + protected override string BaseUrlPattern => @"https?:\/\/\w+\.syosetu\.com\/?"; + + protected override string? WorkTitlePattern => @"//p[@class='novel_title']"; + + protected override string? AuthorNamePattern => @"//div[@class='novel_writername']/a | //div[@class='novel_writername']"; + + protected override string? AuthorLinkPattern => @"//div[@class='novel_writername']/a"; + + protected override string? ChapterUrlPattern => @"//dl[@class='novel_sublist2']//a"; + + protected override string? ChapterNamePattern => @"//dl[@class='novel_sublist2']//a"; + + protected override string? ChapterPostedPattern => base.ChapterPostedPattern; + + protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern; + + protected override string? TagPattern => base.TagPattern; + + protected override string? DatePostedPattern => base.DatePostedPattern; + + protected override string? DateUpdatedPattern => base.DateUpdatedPattern; +} \ No newline at end of file