From d98324c11e85a6443cfadb00425c18766df07fff Mon Sep 17 00:00:00 2001 From: littlefoot Date: Sat, 16 Jul 2022 17:17:43 -0400 Subject: [PATCH] Updated lots of stuff, got multi scrape working, need to test not-nullable chapter novel ids with our current model, now supports sqlite and postgres concurrently (and easy add more), need to get it deployed/do auth --- .drone.yml | 1 + DBConnection/{ => Contexts}/AppDbContext.cs | 15 +- .../Contexts/PostgresSqlAppDbContext.cs | 21 ++ DBConnection/Contexts/SqliteAppDbContext.cs | 21 ++ DBConnection/DBConnection.csproj | 6 + DBConnection/Enums/NovelStatus.cs | 9 - DBConnection/Extensions/BuilderExtensions.cs | 22 +- .../20220715040739_AddLastContentFetch.cs | 27 -- ...35707_make some chapter fields optional.cs | 70 ----- .../20220715143230_add id for novels.cs | 27 -- .../20220716211121_Initial.Designer.cs} | 179 ++++++------ .../PostgresSql/20220716211121_Initial.cs | 200 +++++++++++++ ...11513_Add index on novel guid.Designer.cs} | 184 ++++++------ .../20220716211513_Add index on novel guid.cs | 24 ++ .../PostgresSqlAppDbContextModelSnapshot.cs | 269 ++++++++++++++++++ .../20220716210907_Initial.Designer.cs} | 101 +++---- .../20220716210907_Initial.cs} | 14 +- ...11435_Add index on novel guid.Designer.cs} | 105 ++++--- .../20220716211435_Add index on novel guid.cs | 24 ++ .../SqliteAppDbContextModelSnapshot.cs} | 113 ++++---- .../ModelBuilders/UserNovelBuilder.cs | 2 +- DBConnection/Models/Author.cs | 13 - DBConnection/Models/BaseEntity.cs | 7 - DBConnection/Models/Chapter.cs | 16 -- DBConnection/Models/Novel.cs | 17 -- DBConnection/Models/Tag.cs | 12 - DBConnection/Models/User.cs | 11 - DBConnection/Models/UserNovel.cs | 14 - DBConnection/Readme.md | 13 + DBConnection/Repositories/AuthorRepository.cs | 3 +- DBConnection/Repositories/BaseRepository.cs | 30 +- .../Repositories/ChapterRepository.cs | 18 ++ .../Interfaces/IAuthorRepository.cs | 2 +- .../Interfaces/IChapterRepository.cs | 8 + .../Interfaces/INovelRepository.cs | 2 +- .../Repositories/Interfaces/IRepository.cs | 5 +- .../Repositories/Interfaces/ITagRepository.cs | 3 +- DBConnection/Repositories/NovelRepository.cs | 54 ++-- DBConnection/Repositories/TagRepository.cs | 3 +- .../AccessLayers/ApiAccessLayer.cs | 4 +- Treestar.Shared/Models/DBDomain/Author.cs | 32 +++ Treestar.Shared/Models/DBDomain/BaseEntity.cs | 8 + Treestar.Shared/Models/DBDomain/Chapter.cs | 37 +++ Treestar.Shared/Models/DBDomain/Novel.cs | 21 ++ Treestar.Shared/Models/DBDomain/Tag.cs | 31 ++ Treestar.Shared/Models/DBDomain/User.cs | 12 + Treestar.Shared/Models/DBDomain/UserNovel.cs | 14 + .../DTO/Requests}/ScrapeNovelRequest.cs | 2 +- .../DTO/Requests/ScrapeNovelsRequest.cs | 9 + .../DTO/Responses/ScrapeNovelsResponse.cs | 9 + Treestar.Shared/Models/Enums/NovelStatus.cs | 9 + .../Models/HttpResponseWrapper.cs | 2 +- .../Treestar.Shared.csproj | 1 + WebNovelPortal.sln | 2 +- .../AccessLayers/WebApiAccessLayer.cs | 14 +- WebNovelPortal/Pages/Index.razor | 64 +++-- WebNovelPortal/Pages/NovelDetails.razor | 6 +- .../Components/Display/LoadingDisplay.razor | 5 + .../Display/LoadingDisplay.razor.css | 7 + .../Shared/Components/Display/NovelList.razor | 33 +++ .../Shared/Layouts/MainLayout.razor.css | 10 +- WebNovelPortal/WebNovelPortal.csproj | 3 +- WebNovelPortal/_Imports.razor | 4 +- WebNovelPortal/wwwroot/css/site.css | 1 + .../Controllers/NovelController.cs | 66 ++++- .../Exceptions/NoMatchingScraperException.cs | 10 + WebNovelPortalAPI/Program.cs | 1 + WebNovelPortalAPI/Scrapers/AbstractScraper.cs | 67 +++-- WebNovelPortalAPI/Scrapers/IScraper.cs | 2 +- WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs | 9 +- WebNovelPortalAPI/Scrapers/SyosetuScraper.cs | 105 ++++++- WebNovelPortalAPI/WebNovelPortalAPI.csproj | 2 +- WebNovelPortalAPI/appsettings.json | 4 +- 73 files changed, 1591 insertions(+), 680 deletions(-) rename DBConnection/{ => Contexts}/AppDbContext.cs (78%) create mode 100644 DBConnection/Contexts/PostgresSqlAppDbContext.cs create mode 100644 DBConnection/Contexts/SqliteAppDbContext.cs delete mode 100644 DBConnection/Enums/NovelStatus.cs delete mode 100644 DBConnection/Migrations/20220715040739_AddLastContentFetch.cs delete mode 100644 DBConnection/Migrations/20220715135707_make some chapter fields optional.cs delete mode 100644 DBConnection/Migrations/20220715143230_add id for novels.cs rename DBConnection/Migrations/{20220715030913_Initial.Designer.cs => PostgresSql/20220716211121_Initial.Designer.cs} (52%) create mode 100644 DBConnection/Migrations/PostgresSql/20220716211121_Initial.cs rename DBConnection/Migrations/{20220715040739_AddLastContentFetch.Designer.cs => PostgresSql/20220716211513_Add index on novel guid.Designer.cs} (51%) create mode 100644 DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.cs create mode 100644 DBConnection/Migrations/PostgresSql/PostgresSqlAppDbContextModelSnapshot.cs rename DBConnection/Migrations/{20220715143230_add id for novels.Designer.cs => Sqlite/20220716210907_Initial.Designer.cs} (78%) rename DBConnection/Migrations/{20220715030913_Initial.cs => Sqlite/20220716210907_Initial.cs} (94%) rename DBConnection/Migrations/{20220715135707_make some chapter fields optional.Designer.cs => Sqlite/20220716211435_Add index on novel guid.Designer.cs} (76%) create mode 100644 DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.cs rename DBConnection/Migrations/{AppDbContextModelSnapshot.cs => Sqlite/SqliteAppDbContextModelSnapshot.cs} (74%) delete mode 100644 DBConnection/Models/Author.cs delete mode 100644 DBConnection/Models/BaseEntity.cs delete mode 100644 DBConnection/Models/Chapter.cs delete mode 100644 DBConnection/Models/Novel.cs delete mode 100644 DBConnection/Models/Tag.cs delete mode 100644 DBConnection/Models/User.cs delete mode 100644 DBConnection/Models/UserNovel.cs create mode 100644 DBConnection/Readme.md create mode 100644 DBConnection/Repositories/ChapterRepository.cs create mode 100644 DBConnection/Repositories/Interfaces/IChapterRepository.cs rename {Shared => Treestar.Shared}/AccessLayers/ApiAccessLayer.cs (97%) create mode 100644 Treestar.Shared/Models/DBDomain/Author.cs create mode 100644 Treestar.Shared/Models/DBDomain/BaseEntity.cs create mode 100644 Treestar.Shared/Models/DBDomain/Chapter.cs create mode 100644 Treestar.Shared/Models/DBDomain/Novel.cs create mode 100644 Treestar.Shared/Models/DBDomain/Tag.cs create mode 100644 Treestar.Shared/Models/DBDomain/User.cs create mode 100644 Treestar.Shared/Models/DBDomain/UserNovel.cs rename {Shared/Models/DTO => Treestar.Shared/Models/DTO/Requests}/ScrapeNovelRequest.cs (62%) create mode 100644 Treestar.Shared/Models/DTO/Requests/ScrapeNovelsRequest.cs create mode 100644 Treestar.Shared/Models/DTO/Responses/ScrapeNovelsResponse.cs create mode 100644 Treestar.Shared/Models/Enums/NovelStatus.cs rename {Shared => Treestar.Shared}/Models/HttpResponseWrapper.cs (85%) rename Shared/Shared.csproj => Treestar.Shared/Treestar.Shared.csproj (84%) create mode 100644 WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor create mode 100644 WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor.css create mode 100644 WebNovelPortal/Shared/Components/Display/NovelList.razor create mode 100644 WebNovelPortalAPI/Exceptions/NoMatchingScraperException.cs diff --git a/.drone.yml b/.drone.yml index 81366f0..e3f364f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,4 +1,5 @@ kind: pipeline +type: docker name: default steps: diff --git a/DBConnection/AppDbContext.cs b/DBConnection/Contexts/AppDbContext.cs similarity index 78% rename from DBConnection/AppDbContext.cs rename to DBConnection/Contexts/AppDbContext.cs index 35baae0..dede958 100644 --- a/DBConnection/AppDbContext.cs +++ b/DBConnection/Contexts/AppDbContext.cs @@ -1,12 +1,13 @@ using System.Reflection; using DBConnection.ModelBuilders; -using DBConnection.Models; using DBConnection.Seeders; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Treestar.Shared.Models.DBDomain; -namespace DBConnection; +namespace DBConnection.Contexts; -public class AppDbContext : DbContext +public abstract class AppDbContext : DbContext { public DbSet Novels { get; set; } public DbSet Chapters { get; set; } @@ -14,6 +15,8 @@ public class AppDbContext : DbContext public DbSet Users { get; set; } public DbSet Tags { get; set; } public DbSet UserNovels { get; set; } + protected IConfiguration Configuration { get; set; } + protected abstract string ConnectionStringName { get; } private readonly IEnumerable _seeders = from t in Assembly.GetExecutingAssembly().GetTypes() @@ -25,8 +28,9 @@ public class AppDbContext : DbContext where t.IsClass && (t.Namespace?.Contains(nameof(DBConnection.ModelBuilders)) ?? false) && typeof(IModelBuilder).IsAssignableFrom(t) select (IModelBuilder) Activator.CreateInstance(t); - public AppDbContext(DbContextOptions options) : base(options) + protected AppDbContext(DbContextOptions options, IConfiguration configuration) : base(options) { + Configuration = configuration; } public override Task SaveChangesAsync(CancellationToken cancellationToken = new CancellationToken()) @@ -64,5 +68,8 @@ public class AppDbContext : DbContext protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { base.OnConfiguring(optionsBuilder); + UseSqlConnection(optionsBuilder, Configuration.GetConnectionString(ConnectionStringName)); } + + protected abstract void UseSqlConnection(DbContextOptionsBuilder builder, string connectionString); } \ No newline at end of file diff --git a/DBConnection/Contexts/PostgresSqlAppDbContext.cs b/DBConnection/Contexts/PostgresSqlAppDbContext.cs new file mode 100644 index 0000000..0e004c5 --- /dev/null +++ b/DBConnection/Contexts/PostgresSqlAppDbContext.cs @@ -0,0 +1,21 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; + +namespace DBConnection.Contexts; + +/// +/// Pulls connection string from 'PostgresSql' and selected with provider 'PostgresSql' +/// +public class PostgresSqlAppDbContext : AppDbContext +{ + public PostgresSqlAppDbContext(DbContextOptions options, IConfiguration configuration) : base(options, configuration) + { + } + + protected override string ConnectionStringName => "PostgresSql"; + + protected override void UseSqlConnection(DbContextOptionsBuilder builder, string connectionString) + { + builder.UseNpgsql(connectionString); + } +} \ No newline at end of file diff --git a/DBConnection/Contexts/SqliteAppDbContext.cs b/DBConnection/Contexts/SqliteAppDbContext.cs new file mode 100644 index 0000000..413a22f --- /dev/null +++ b/DBConnection/Contexts/SqliteAppDbContext.cs @@ -0,0 +1,21 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; + +namespace DBConnection.Contexts; + +/// +/// Pulls connection string from 'Sqlite' and selected with provider 'Sqlite' +/// +public class SqliteAppDbContext : AppDbContext +{ + public SqliteAppDbContext(DbContextOptions options, IConfiguration configuration) : base(options, configuration) + { + } + + + protected override string ConnectionStringName => "Sqlite"; + protected override void UseSqlConnection(DbContextOptionsBuilder builder, string connectionString) + { + builder.UseSqlite(connectionString); + } +} \ No newline at end of file diff --git a/DBConnection/DBConnection.csproj b/DBConnection/DBConnection.csproj index 9fd009f..f0c4ff4 100644 --- a/DBConnection/DBConnection.csproj +++ b/DBConnection/DBConnection.csproj @@ -19,6 +19,12 @@ + + + + + + diff --git a/DBConnection/Enums/NovelStatus.cs b/DBConnection/Enums/NovelStatus.cs deleted file mode 100644 index 5042ce8..0000000 --- a/DBConnection/Enums/NovelStatus.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace DBConnection.Enums; - -public enum NovelStatus -{ - InProgress, - Completed, - Hiatus, - Unknown -} \ No newline at end of file diff --git a/DBConnection/Extensions/BuilderExtensions.cs b/DBConnection/Extensions/BuilderExtensions.cs index 40265c0..460cf19 100644 --- a/DBConnection/Extensions/BuilderExtensions.cs +++ b/DBConnection/Extensions/BuilderExtensions.cs @@ -1,4 +1,5 @@ using System.Reflection; +using DBConnection.Contexts; using DBConnection.Repositories.Interfaces; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -15,11 +16,22 @@ public static class BuilderExtensions /// configuration public static void AddDbServices(this IServiceCollection collection, IConfiguration config) { - string dbConnectionString = config.GetConnectionString("DefaultConnection"); - collection.AddDbContext(opt => - { - opt.UseSqlite(dbConnectionString); - }); + // Add appropriate DbContext + // Contexts are linked to providers by trimming the 'AppDbContext' portion of their name. + // So 'PostgresSqlAppDbContext' is selected with provider 'PostgresSql' + var providerTypes = Assembly.GetExecutingAssembly().GetTypes() + .Where(t => (t.Namespace?.Contains(nameof(Contexts)) ?? false) && typeof(AppDbContext).IsAssignableFrom(t) && !t.IsAbstract && t.Name.EndsWith(nameof(AppDbContext))); + var providers = providerTypes.ToDictionary(t => t.Name.Replace(nameof(AppDbContext), ""), t => t); + var selectedProvider = config["DatabaseProvider"]; + + //add dboptions + collection.AddSingleton(new DbContextOptions()); + collection.AddSingleton(p => p.GetRequiredService>()); + + // add our provider dbcontext + collection.AddScoped(typeof(AppDbContext), providers[selectedProvider]); + + // Add db repositories Type[] repositories = Assembly.GetExecutingAssembly().GetTypes() .Where(t => t.IsClass && !t.IsAbstract && (t.Namespace?.Contains(nameof(DBConnection.Repositories)) ?? false) && typeof(IRepository).IsAssignableFrom(t)).ToArray(); foreach (var repo in repositories) diff --git a/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs b/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs deleted file mode 100644 index 9e45e0d..0000000 --- a/DBConnection/Migrations/20220715040739_AddLastContentFetch.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DBConnection.Migrations -{ - public partial class AddLastContentFetch : Migration - { - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AddColumn( - name: "LastContentFetch", - table: "Chapters", - type: "TEXT", - nullable: false, - defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); - } - - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropColumn( - name: "LastContentFetch", - table: "Chapters"); - } - } -} diff --git a/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs b/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs deleted file mode 100644 index 51ea2f6..0000000 --- a/DBConnection/Migrations/20220715135707_make some chapter fields optional.cs +++ /dev/null @@ -1,70 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DBConnection.Migrations -{ - public partial class makesomechapterfieldsoptional : Migration - { - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "LastContentFetch", - table: "Chapters", - type: "TEXT", - nullable: true, - oldClrType: typeof(DateTime), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "DateUpdated", - table: "Chapters", - type: "TEXT", - nullable: true, - oldClrType: typeof(DateTime), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "DatePosted", - table: "Chapters", - type: "TEXT", - nullable: true, - oldClrType: typeof(DateTime), - oldType: "TEXT"); - } - - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "LastContentFetch", - table: "Chapters", - type: "TEXT", - nullable: false, - defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), - oldClrType: typeof(DateTime), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "DateUpdated", - table: "Chapters", - type: "TEXT", - nullable: false, - defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), - oldClrType: typeof(DateTime), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "DatePosted", - table: "Chapters", - type: "TEXT", - nullable: false, - defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), - oldClrType: typeof(DateTime), - oldType: "TEXT", - oldNullable: true); - } - } -} diff --git a/DBConnection/Migrations/20220715143230_add id for novels.cs b/DBConnection/Migrations/20220715143230_add id for novels.cs deleted file mode 100644 index a228c04..0000000 --- a/DBConnection/Migrations/20220715143230_add id for novels.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DBConnection.Migrations -{ - public partial class addidfornovels : Migration - { - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AddColumn( - name: "Guid", - table: "Novels", - type: "TEXT", - nullable: false, - defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); - } - - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropColumn( - name: "Guid", - table: "Novels"); - } - } -} diff --git a/DBConnection/Migrations/20220715030913_Initial.Designer.cs b/DBConnection/Migrations/PostgresSql/20220716211121_Initial.Designer.cs similarity index 52% rename from DBConnection/Migrations/20220715030913_Initial.Designer.cs rename to DBConnection/Migrations/PostgresSql/20220716211121_Initial.Designer.cs index 1afa21b..4aa3a4c 100644 --- a/DBConnection/Migrations/20220715030913_Initial.Designer.cs +++ b/DBConnection/Migrations/PostgresSql/20220716211121_Initial.Designer.cs @@ -1,76 +1,100 @@ // using System; -using DBConnection; +using DBConnection.Contexts; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.PostgresSql { - [DbContext(typeof(AppDbContext))] - [Migration("20220715030913_Initial")] + [DbContext(typeof(PostgresSqlAppDbContext))] + [Migration("20220716211121_Initial")] partial class Initial { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + modelBuilder + .HasAnnotation("ProductVersion", "6.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); - modelBuilder.Entity("DBConnection.Models.Author", b => + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("text"); + + b.Property("TagsTagValue") + .HasColumnType("text"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("Name") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); b.ToTable("Authors"); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("ChapterNumber") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.Property("Content") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); - b.Property("DatePosted") - .HasColumnType("TEXT"); + b.Property("DatePosted") + .HasColumnType("timestamp with time zone"); - b.Property("DateUpdated") - .HasColumnType("TEXT"); + b.Property("DateUpdated") + .HasColumnType("timestamp with time zone"); + + b.Property("LastContentFetch") + .HasColumnType("timestamp with time zone"); b.Property("Name") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("NovelUrl") - .HasColumnType("TEXT"); + .IsRequired() + .HasColumnType("text"); b.Property("RawContent") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); @@ -79,29 +103,35 @@ namespace DBConnection.Migrations b.ToTable("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("AuthorUrl") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DatePosted") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); + + b.Property("Guid") + .HasColumnType("uuid"); b.Property("LastUpdated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .HasColumnType("integer"); b.Property("Title") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); @@ -110,53 +140,55 @@ namespace DBConnection.Migrations b.ToTable("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Tag", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => { b.Property("TagValue") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.HasKey("TagValue"); b.ToTable("Tags"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Property("Id") .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("Email") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Id"); b.ToTable("Users"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { b.Property("NovelUrl") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("UserId") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.Property("LastChapterRead") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.HasKey("NovelUrl", "UserId"); @@ -167,44 +199,46 @@ namespace DBConnection.Migrations modelBuilder.Entity("NovelTag", b => { - b.Property("NovelsUrl") - .HasColumnType("TEXT"); + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); - b.Property("TagsTagValue") - .HasColumnType("TEXT"); - - b.HasKey("NovelsUrl", "TagsTagValue"); - - b.HasIndex("TagsTagValue"); - - b.ToTable("NovelTag"); + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { - b.HasOne("DBConnection.Models.Novel", null) + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) .WithMany("Chapters") - .HasForeignKey("NovelUrl"); + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { - b.HasOne("DBConnection.Models.Author", "Author") + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") .WithMany("Novels") .HasForeignKey("AuthorUrl"); b.Navigation("Author"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { - b.HasOne("DBConnection.Models.Novel", "Novel") + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") .WithMany() .HasForeignKey("NovelUrl") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DBConnection.Models.User", "User") + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") .WithMany("WatchedNovels") .HasForeignKey("UserId") .OnDelete(DeleteBehavior.Cascade) @@ -215,32 +249,17 @@ namespace DBConnection.Migrations b.Navigation("User"); }); - modelBuilder.Entity("NovelTag", b => - { - b.HasOne("DBConnection.Models.Novel", null) - .WithMany() - .HasForeignKey("NovelsUrl") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DBConnection.Models.Tag", null) - .WithMany() - .HasForeignKey("TagsTagValue") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); - - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Navigation("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Navigation("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Navigation("WatchedNovels"); }); diff --git a/DBConnection/Migrations/PostgresSql/20220716211121_Initial.cs b/DBConnection/Migrations/PostgresSql/20220716211121_Initial.cs new file mode 100644 index 0000000..ba1814c --- /dev/null +++ b/DBConnection/Migrations/PostgresSql/20220716211121_Initial.cs @@ -0,0 +1,200 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DBConnection.Migrations.PostgresSql +{ + public partial class Initial : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "Authors", + columns: table => new + { + Url = table.Column(type: "text", nullable: false), + Name = table.Column(type: "text", nullable: false), + DateCreated = table.Column(type: "timestamp with time zone", nullable: false), + DateModified = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Authors", x => x.Url); + }); + + migrationBuilder.CreateTable( + name: "Tags", + columns: table => new + { + TagValue = table.Column(type: "text", nullable: false), + DateCreated = table.Column(type: "timestamp with time zone", nullable: false), + DateModified = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Tags", x => x.TagValue); + }); + + migrationBuilder.CreateTable( + name: "Users", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + Email = table.Column(type: "text", nullable: false), + DateCreated = table.Column(type: "timestamp with time zone", nullable: false), + DateModified = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Users", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Novels", + columns: table => new + { + Url = table.Column(type: "text", nullable: false), + Guid = table.Column(type: "uuid", nullable: false), + Title = table.Column(type: "text", nullable: false), + AuthorUrl = table.Column(type: "text", nullable: true), + Status = table.Column(type: "integer", nullable: false), + LastUpdated = table.Column(type: "timestamp with time zone", nullable: false), + DatePosted = table.Column(type: "timestamp with time zone", nullable: false), + DateCreated = table.Column(type: "timestamp with time zone", nullable: false), + DateModified = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Novels", x => x.Url); + table.ForeignKey( + name: "FK_Novels_Authors_AuthorUrl", + column: x => x.AuthorUrl, + principalTable: "Authors", + principalColumn: "Url"); + }); + + migrationBuilder.CreateTable( + name: "Chapters", + columns: table => new + { + Url = table.Column(type: "text", nullable: false), + ChapterNumber = table.Column(type: "integer", nullable: false), + Name = table.Column(type: "text", nullable: false), + Content = table.Column(type: "text", nullable: true), + RawContent = table.Column(type: "text", nullable: true), + DatePosted = table.Column(type: "timestamp with time zone", nullable: true), + DateUpdated = table.Column(type: "timestamp with time zone", nullable: true), + LastContentFetch = table.Column(type: "timestamp with time zone", nullable: true), + NovelUrl = table.Column(type: "text", nullable: false), + DateCreated = table.Column(type: "timestamp with time zone", nullable: false), + DateModified = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Chapters", x => x.Url); + table.ForeignKey( + name: "FK_Chapters_Novels_NovelUrl", + column: x => x.NovelUrl, + principalTable: "Novels", + principalColumn: "Url", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "NovelTag", + columns: table => new + { + NovelsUrl = table.Column(type: "text", nullable: false), + TagsTagValue = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_NovelTag", x => new { x.NovelsUrl, x.TagsTagValue }); + table.ForeignKey( + name: "FK_NovelTag_Novels_NovelsUrl", + column: x => x.NovelsUrl, + principalTable: "Novels", + principalColumn: "Url", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_NovelTag_Tags_TagsTagValue", + column: x => x.TagsTagValue, + principalTable: "Tags", + principalColumn: "TagValue", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "UserNovels", + columns: table => new + { + UserId = table.Column(type: "integer", nullable: false), + NovelUrl = table.Column(type: "text", nullable: false), + LastChapterRead = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_UserNovels", x => new { x.NovelUrl, x.UserId }); + table.ForeignKey( + name: "FK_UserNovels_Novels_NovelUrl", + column: x => x.NovelUrl, + principalTable: "Novels", + principalColumn: "Url", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_UserNovels_Users_UserId", + column: x => x.UserId, + principalTable: "Users", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_Chapters_NovelUrl", + table: "Chapters", + column: "NovelUrl"); + + migrationBuilder.CreateIndex( + name: "IX_Novels_AuthorUrl", + table: "Novels", + column: "AuthorUrl"); + + migrationBuilder.CreateIndex( + name: "IX_NovelTag_TagsTagValue", + table: "NovelTag", + column: "TagsTagValue"); + + migrationBuilder.CreateIndex( + name: "IX_UserNovels_UserId", + table: "UserNovels", + column: "UserId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "Chapters"); + + migrationBuilder.DropTable( + name: "NovelTag"); + + migrationBuilder.DropTable( + name: "UserNovels"); + + migrationBuilder.DropTable( + name: "Tags"); + + migrationBuilder.DropTable( + name: "Novels"); + + migrationBuilder.DropTable( + name: "Users"); + + migrationBuilder.DropTable( + name: "Authors"); + } + } +} diff --git a/DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs b/DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.Designer.cs similarity index 51% rename from DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs rename to DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.Designer.cs index b1b50a7..be9ae51 100644 --- a/DBConnection/Migrations/20220715040739_AddLastContentFetch.Designer.cs +++ b/DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.Designer.cs @@ -1,79 +1,100 @@ // using System; -using DBConnection; +using DBConnection.Contexts; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.PostgresSql { - [DbContext(typeof(AppDbContext))] - [Migration("20220715040739_AddLastContentFetch")] - partial class AddLastContentFetch + [DbContext(typeof(PostgresSqlAppDbContext))] + [Migration("20220716211513_Add index on novel guid")] + partial class Addindexonnovelguid { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + modelBuilder + .HasAnnotation("ProductVersion", "6.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); - modelBuilder.Entity("DBConnection.Models.Author", b => + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("text"); + + b.Property("TagsTagValue") + .HasColumnType("text"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("Name") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); b.ToTable("Authors"); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("ChapterNumber") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.Property("Content") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); - b.Property("DatePosted") - .HasColumnType("TEXT"); + b.Property("DatePosted") + .HasColumnType("timestamp with time zone"); - b.Property("DateUpdated") - .HasColumnType("TEXT"); + b.Property("DateUpdated") + .HasColumnType("timestamp with time zone"); - b.Property("LastContentFetch") - .HasColumnType("TEXT"); + b.Property("LastContentFetch") + .HasColumnType("timestamp with time zone"); b.Property("Name") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("NovelUrl") - .HasColumnType("TEXT"); + .IsRequired() + .HasColumnType("text"); b.Property("RawContent") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); @@ -82,84 +103,94 @@ namespace DBConnection.Migrations b.ToTable("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Property("Url") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("AuthorUrl") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DatePosted") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); + + b.Property("Guid") + .HasColumnType("uuid"); b.Property("LastUpdated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .HasColumnType("integer"); b.Property("Title") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Url"); b.HasIndex("AuthorUrl"); + b.HasIndex("Guid"); + b.ToTable("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Tag", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => { b.Property("TagValue") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.HasKey("TagValue"); b.ToTable("Tags"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Property("Id") .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); b.Property("DateCreated") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("DateModified") - .HasColumnType("TEXT"); + .HasColumnType("timestamp with time zone"); b.Property("Email") .IsRequired() - .HasColumnType("TEXT"); + .HasColumnType("text"); b.HasKey("Id"); b.ToTable("Users"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { b.Property("NovelUrl") - .HasColumnType("TEXT"); + .HasColumnType("text"); b.Property("UserId") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.Property("LastChapterRead") - .HasColumnType("INTEGER"); + .HasColumnType("integer"); b.HasKey("NovelUrl", "UserId"); @@ -170,44 +201,46 @@ namespace DBConnection.Migrations modelBuilder.Entity("NovelTag", b => { - b.Property("NovelsUrl") - .HasColumnType("TEXT"); + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); - b.Property("TagsTagValue") - .HasColumnType("TEXT"); - - b.HasKey("NovelsUrl", "TagsTagValue"); - - b.HasIndex("TagsTagValue"); - - b.ToTable("NovelTag"); + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { - b.HasOne("DBConnection.Models.Novel", null) + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) .WithMany("Chapters") - .HasForeignKey("NovelUrl"); + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { - b.HasOne("DBConnection.Models.Author", "Author") + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") .WithMany("Novels") .HasForeignKey("AuthorUrl"); b.Navigation("Author"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { - b.HasOne("DBConnection.Models.Novel", "Novel") + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") .WithMany() .HasForeignKey("NovelUrl") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DBConnection.Models.User", "User") + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") .WithMany("WatchedNovels") .HasForeignKey("UserId") .OnDelete(DeleteBehavior.Cascade) @@ -218,32 +251,17 @@ namespace DBConnection.Migrations b.Navigation("User"); }); - modelBuilder.Entity("NovelTag", b => - { - b.HasOne("DBConnection.Models.Novel", null) - .WithMany() - .HasForeignKey("NovelsUrl") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DBConnection.Models.Tag", null) - .WithMany() - .HasForeignKey("TagsTagValue") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); - - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Navigation("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Navigation("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Navigation("WatchedNovels"); }); diff --git a/DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.cs b/DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.cs new file mode 100644 index 0000000..12c9d00 --- /dev/null +++ b/DBConnection/Migrations/PostgresSql/20220716211513_Add index on novel guid.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DBConnection.Migrations.PostgresSql +{ + public partial class Addindexonnovelguid : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_Novels_Guid", + table: "Novels", + column: "Guid"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Novels_Guid", + table: "Novels"); + } + } +} diff --git a/DBConnection/Migrations/PostgresSql/PostgresSqlAppDbContextModelSnapshot.cs b/DBConnection/Migrations/PostgresSql/PostgresSqlAppDbContextModelSnapshot.cs new file mode 100644 index 0000000..f219fe5 --- /dev/null +++ b/DBConnection/Migrations/PostgresSql/PostgresSqlAppDbContextModelSnapshot.cs @@ -0,0 +1,269 @@ +// +using System; +using DBConnection.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DBConnection.Migrations.PostgresSql +{ + [DbContext(typeof(PostgresSqlAppDbContext))] + partial class PostgresSqlAppDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("text"); + + b.Property("TagsTagValue") + .HasColumnType("text"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => + { + b.Property("Url") + .HasColumnType("text"); + + b.Property("DateCreated") + .HasColumnType("timestamp with time zone"); + + b.Property("DateModified") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Url"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => + { + b.Property("Url") + .HasColumnType("text"); + + b.Property("ChapterNumber") + .HasColumnType("integer"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("DateCreated") + .HasColumnType("timestamp with time zone"); + + b.Property("DateModified") + .HasColumnType("timestamp with time zone"); + + b.Property("DatePosted") + .HasColumnType("timestamp with time zone"); + + b.Property("DateUpdated") + .HasColumnType("timestamp with time zone"); + + b.Property("LastContentFetch") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("NovelUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("RawContent") + .HasColumnType("text"); + + b.HasKey("Url"); + + b.HasIndex("NovelUrl"); + + b.ToTable("Chapters"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => + { + b.Property("Url") + .HasColumnType("text"); + + b.Property("AuthorUrl") + .HasColumnType("text"); + + b.Property("DateCreated") + .HasColumnType("timestamp with time zone"); + + b.Property("DateModified") + .HasColumnType("timestamp with time zone"); + + b.Property("DatePosted") + .HasColumnType("timestamp with time zone"); + + b.Property("Guid") + .HasColumnType("uuid"); + + b.Property("LastUpdated") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Url"); + + b.HasIndex("AuthorUrl"); + + b.HasIndex("Guid"); + + b.ToTable("Novels"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => + { + b.Property("TagValue") + .HasColumnType("text"); + + b.Property("DateCreated") + .HasColumnType("timestamp with time zone"); + + b.Property("DateModified") + .HasColumnType("timestamp with time zone"); + + b.HasKey("TagValue"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DateCreated") + .HasColumnType("timestamp with time zone"); + + b.Property("DateModified") + .HasColumnType("timestamp with time zone"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => + { + b.Property("NovelUrl") + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.Property("LastChapterRead") + .HasColumnType("integer"); + + b.HasKey("NovelUrl", "UserId"); + + b.HasIndex("UserId"); + + b.ToTable("UserNovels"); + }); + + modelBuilder.Entity("NovelTag", b => + { + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => + { + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany("Chapters") + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => + { + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") + .WithMany("Novels") + .HasForeignKey("AuthorUrl"); + + b.Navigation("Author"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => + { + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") + .WithMany() + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") + .WithMany("WatchedNovels") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Novel"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => + { + b.Navigation("Novels"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => + { + b.Navigation("Chapters"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => + { + b.Navigation("WatchedNovels"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DBConnection/Migrations/20220715143230_add id for novels.Designer.cs b/DBConnection/Migrations/Sqlite/20220716210907_Initial.Designer.cs similarity index 78% rename from DBConnection/Migrations/20220715143230_add id for novels.Designer.cs rename to DBConnection/Migrations/Sqlite/20220716210907_Initial.Designer.cs index ca6d10b..2eee76e 100644 --- a/DBConnection/Migrations/20220715143230_add id for novels.Designer.cs +++ b/DBConnection/Migrations/Sqlite/20220716210907_Initial.Designer.cs @@ -1,6 +1,6 @@ // using System; -using DBConnection; +using DBConnection.Contexts; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; @@ -8,18 +8,33 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.Sqlite { - [DbContext(typeof(AppDbContext))] - [Migration("20220715143230_add id for novels")] - partial class addidfornovels + [DbContext(typeof(SqliteAppDbContext))] + [Migration("20220716210907_Initial")] + partial class Initial { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -39,7 +54,7 @@ namespace DBConnection.Migrations b.ToTable("Authors"); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -70,6 +85,7 @@ namespace DBConnection.Migrations .HasColumnType("TEXT"); b.Property("NovelUrl") + .IsRequired() .HasColumnType("TEXT"); b.Property("RawContent") @@ -82,7 +98,7 @@ namespace DBConnection.Migrations b.ToTable("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -100,12 +116,14 @@ namespace DBConnection.Migrations .HasColumnType("TEXT"); b.Property("Guid") - .ValueGeneratedOnAdd() .HasColumnType("TEXT"); b.Property("LastUpdated") .HasColumnType("TEXT"); + b.Property("Status") + .HasColumnType("INTEGER"); + b.Property("Title") .IsRequired() .HasColumnType("TEXT"); @@ -117,7 +135,7 @@ namespace DBConnection.Migrations b.ToTable("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Tag", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => { b.Property("TagValue") .HasColumnType("TEXT"); @@ -133,7 +151,7 @@ namespace DBConnection.Migrations b.ToTable("Tags"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -154,7 +172,7 @@ namespace DBConnection.Migrations b.ToTable("Users"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { b.Property("NovelUrl") .HasColumnType("TEXT"); @@ -174,44 +192,46 @@ namespace DBConnection.Migrations modelBuilder.Entity("NovelTag", b => { - b.Property("NovelsUrl") - .HasColumnType("TEXT"); + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); - b.Property("TagsTagValue") - .HasColumnType("TEXT"); - - b.HasKey("NovelsUrl", "TagsTagValue"); - - b.HasIndex("TagsTagValue"); - - b.ToTable("NovelTag"); + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { - b.HasOne("DBConnection.Models.Novel", null) + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) .WithMany("Chapters") - .HasForeignKey("NovelUrl"); + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { - b.HasOne("DBConnection.Models.Author", "Author") + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") .WithMany("Novels") .HasForeignKey("AuthorUrl"); b.Navigation("Author"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { - b.HasOne("DBConnection.Models.Novel", "Novel") + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") .WithMany() .HasForeignKey("NovelUrl") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DBConnection.Models.User", "User") + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") .WithMany("WatchedNovels") .HasForeignKey("UserId") .OnDelete(DeleteBehavior.Cascade) @@ -222,32 +242,17 @@ namespace DBConnection.Migrations b.Navigation("User"); }); - modelBuilder.Entity("NovelTag", b => - { - b.HasOne("DBConnection.Models.Novel", null) - .WithMany() - .HasForeignKey("NovelsUrl") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DBConnection.Models.Tag", null) - .WithMany() - .HasForeignKey("TagsTagValue") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); - - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Navigation("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Navigation("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Navigation("WatchedNovels"); }); diff --git a/DBConnection/Migrations/20220715030913_Initial.cs b/DBConnection/Migrations/Sqlite/20220716210907_Initial.cs similarity index 94% rename from DBConnection/Migrations/20220715030913_Initial.cs rename to DBConnection/Migrations/Sqlite/20220716210907_Initial.cs index 99b53f4..1846009 100644 --- a/DBConnection/Migrations/20220715030913_Initial.cs +++ b/DBConnection/Migrations/Sqlite/20220716210907_Initial.cs @@ -3,7 +3,7 @@ using Microsoft.EntityFrameworkCore.Migrations; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.Sqlite { public partial class Initial : Migration { @@ -56,8 +56,10 @@ namespace DBConnection.Migrations columns: table => new { Url = table.Column(type: "TEXT", nullable: false), + Guid = table.Column(type: "TEXT", nullable: false), Title = table.Column(type: "TEXT", nullable: false), AuthorUrl = table.Column(type: "TEXT", nullable: true), + Status = table.Column(type: "INTEGER", nullable: false), LastUpdated = table.Column(type: "TEXT", nullable: false), DatePosted = table.Column(type: "TEXT", nullable: false), DateCreated = table.Column(type: "TEXT", nullable: false), @@ -82,9 +84,10 @@ namespace DBConnection.Migrations Name = table.Column(type: "TEXT", nullable: false), Content = table.Column(type: "TEXT", nullable: true), RawContent = table.Column(type: "TEXT", nullable: true), - DatePosted = table.Column(type: "TEXT", nullable: false), - DateUpdated = table.Column(type: "TEXT", nullable: false), - NovelUrl = table.Column(type: "TEXT", nullable: true), + DatePosted = table.Column(type: "TEXT", nullable: true), + DateUpdated = table.Column(type: "TEXT", nullable: true), + LastContentFetch = table.Column(type: "TEXT", nullable: true), + NovelUrl = table.Column(type: "TEXT", nullable: false), DateCreated = table.Column(type: "TEXT", nullable: false), DateModified = table.Column(type: "TEXT", nullable: false) }, @@ -95,7 +98,8 @@ namespace DBConnection.Migrations name: "FK_Chapters_Novels_NovelUrl", column: x => x.NovelUrl, principalTable: "Novels", - principalColumn: "Url"); + principalColumn: "Url", + onDelete: ReferentialAction.Cascade); }); migrationBuilder.CreateTable( diff --git a/DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs b/DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.Designer.cs similarity index 76% rename from DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs rename to DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.Designer.cs index f1d5f11..ba6ff0b 100644 --- a/DBConnection/Migrations/20220715135707_make some chapter fields optional.Designer.cs +++ b/DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.Designer.cs @@ -1,6 +1,6 @@ // using System; -using DBConnection; +using DBConnection.Contexts; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; @@ -8,18 +8,33 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.Sqlite { - [DbContext(typeof(AppDbContext))] - [Migration("20220715135707_make some chapter fields optional")] - partial class makesomechapterfieldsoptional + [DbContext(typeof(SqliteAppDbContext))] + [Migration("20220716211435_Add index on novel guid")] + partial class Addindexonnovelguid { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -39,7 +54,7 @@ namespace DBConnection.Migrations b.ToTable("Authors"); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -70,6 +85,7 @@ namespace DBConnection.Migrations .HasColumnType("TEXT"); b.Property("NovelUrl") + .IsRequired() .HasColumnType("TEXT"); b.Property("RawContent") @@ -82,7 +98,7 @@ namespace DBConnection.Migrations b.ToTable("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -99,9 +115,15 @@ namespace DBConnection.Migrations b.Property("DatePosted") .HasColumnType("TEXT"); + b.Property("Guid") + .HasColumnType("TEXT"); + b.Property("LastUpdated") .HasColumnType("TEXT"); + b.Property("Status") + .HasColumnType("INTEGER"); + b.Property("Title") .IsRequired() .HasColumnType("TEXT"); @@ -110,10 +132,12 @@ namespace DBConnection.Migrations b.HasIndex("AuthorUrl"); + b.HasIndex("Guid"); + b.ToTable("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Tag", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => { b.Property("TagValue") .HasColumnType("TEXT"); @@ -129,7 +153,7 @@ namespace DBConnection.Migrations b.ToTable("Tags"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -150,7 +174,7 @@ namespace DBConnection.Migrations b.ToTable("Users"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { b.Property("NovelUrl") .HasColumnType("TEXT"); @@ -170,44 +194,46 @@ namespace DBConnection.Migrations modelBuilder.Entity("NovelTag", b => { - b.Property("NovelsUrl") - .HasColumnType("TEXT"); + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); - b.Property("TagsTagValue") - .HasColumnType("TEXT"); - - b.HasKey("NovelsUrl", "TagsTagValue"); - - b.HasIndex("TagsTagValue"); - - b.ToTable("NovelTag"); + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { - b.HasOne("DBConnection.Models.Novel", null) + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) .WithMany("Chapters") - .HasForeignKey("NovelUrl"); + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { - b.HasOne("DBConnection.Models.Author", "Author") + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") .WithMany("Novels") .HasForeignKey("AuthorUrl"); b.Navigation("Author"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { - b.HasOne("DBConnection.Models.Novel", "Novel") + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") .WithMany() .HasForeignKey("NovelUrl") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DBConnection.Models.User", "User") + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") .WithMany("WatchedNovels") .HasForeignKey("UserId") .OnDelete(DeleteBehavior.Cascade) @@ -218,32 +244,17 @@ namespace DBConnection.Migrations b.Navigation("User"); }); - modelBuilder.Entity("NovelTag", b => - { - b.HasOne("DBConnection.Models.Novel", null) - .WithMany() - .HasForeignKey("NovelsUrl") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DBConnection.Models.Tag", null) - .WithMany() - .HasForeignKey("TagsTagValue") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); - - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Navigation("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Navigation("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Navigation("WatchedNovels"); }); diff --git a/DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.cs b/DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.cs new file mode 100644 index 0000000..4f9e3a4 --- /dev/null +++ b/DBConnection/Migrations/Sqlite/20220716211435_Add index on novel guid.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DBConnection.Migrations.Sqlite +{ + public partial class Addindexonnovelguid : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_Novels_Guid", + table: "Novels", + column: "Guid"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Novels_Guid", + table: "Novels"); + } + } +} diff --git a/DBConnection/Migrations/AppDbContextModelSnapshot.cs b/DBConnection/Migrations/Sqlite/SqliteAppDbContextModelSnapshot.cs similarity index 74% rename from DBConnection/Migrations/AppDbContextModelSnapshot.cs rename to DBConnection/Migrations/Sqlite/SqliteAppDbContextModelSnapshot.cs index c300c69..220fecc 100644 --- a/DBConnection/Migrations/AppDbContextModelSnapshot.cs +++ b/DBConnection/Migrations/Sqlite/SqliteAppDbContextModelSnapshot.cs @@ -1,23 +1,38 @@ // using System; -using DBConnection; +using DBConnection.Contexts; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; #nullable disable -namespace DBConnection.Migrations +namespace DBConnection.Migrations.Sqlite { - [DbContext(typeof(AppDbContext))] - partial class AppDbContextModelSnapshot : ModelSnapshot + [DbContext(typeof(SqliteAppDbContext))] + partial class SqliteAppDbContextModelSnapshot : ModelSnapshot { protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("NovelTag", b => + { + b.Property("NovelsUrl") + .HasColumnType("TEXT"); + + b.Property("TagsTagValue") + .HasColumnType("TEXT"); + + b.HasKey("NovelsUrl", "TagsTagValue"); + + b.HasIndex("TagsTagValue"); + + b.ToTable("NovelTag"); + }); + + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -34,10 +49,10 @@ namespace DBConnection.Migrations b.HasKey("Url"); - b.ToTable("Authors", (string)null); + b.ToTable("Authors"); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -68,6 +83,7 @@ namespace DBConnection.Migrations .HasColumnType("TEXT"); b.Property("NovelUrl") + .IsRequired() .HasColumnType("TEXT"); b.Property("RawContent") @@ -77,10 +93,10 @@ namespace DBConnection.Migrations b.HasIndex("NovelUrl"); - b.ToTable("Chapters", (string)null); + b.ToTable("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Property("Url") .HasColumnType("TEXT"); @@ -98,12 +114,14 @@ namespace DBConnection.Migrations .HasColumnType("TEXT"); b.Property("Guid") - .ValueGeneratedOnAdd() .HasColumnType("TEXT"); b.Property("LastUpdated") .HasColumnType("TEXT"); + b.Property("Status") + .HasColumnType("INTEGER"); + b.Property("Title") .IsRequired() .HasColumnType("TEXT"); @@ -112,10 +130,12 @@ namespace DBConnection.Migrations b.HasIndex("AuthorUrl"); - b.ToTable("Novels", (string)null); + b.HasIndex("Guid"); + + b.ToTable("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Tag", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Tag", b => { b.Property("TagValue") .HasColumnType("TEXT"); @@ -128,10 +148,10 @@ namespace DBConnection.Migrations b.HasKey("TagValue"); - b.ToTable("Tags", (string)null); + b.ToTable("Tags"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -149,10 +169,10 @@ namespace DBConnection.Migrations b.HasKey("Id"); - b.ToTable("Users", (string)null); + b.ToTable("Users"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { b.Property("NovelUrl") .HasColumnType("TEXT"); @@ -167,49 +187,51 @@ namespace DBConnection.Migrations b.HasIndex("UserId"); - b.ToTable("UserNovels", (string)null); + b.ToTable("UserNovels"); }); modelBuilder.Entity("NovelTag", b => { - b.Property("NovelsUrl") - .HasColumnType("TEXT"); + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) + .WithMany() + .HasForeignKey("NovelsUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); - b.Property("TagsTagValue") - .HasColumnType("TEXT"); - - b.HasKey("NovelsUrl", "TagsTagValue"); - - b.HasIndex("TagsTagValue"); - - b.ToTable("NovelTag", (string)null); + b.HasOne("Treestar.Shared.Models.DBDomain.Tag", null) + .WithMany() + .HasForeignKey("TagsTagValue") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Chapter", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Chapter", b => { - b.HasOne("DBConnection.Models.Novel", null) + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", null) .WithMany("Chapters") - .HasForeignKey("NovelUrl"); + .HasForeignKey("NovelUrl") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { - b.HasOne("DBConnection.Models.Author", "Author") + b.HasOne("Treestar.Shared.Models.DBDomain.Author", "Author") .WithMany("Novels") .HasForeignKey("AuthorUrl"); b.Navigation("Author"); }); - modelBuilder.Entity("DBConnection.Models.UserNovel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.UserNovel", b => { - b.HasOne("DBConnection.Models.Novel", "Novel") + b.HasOne("Treestar.Shared.Models.DBDomain.Novel", "Novel") .WithMany() .HasForeignKey("NovelUrl") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DBConnection.Models.User", "User") + b.HasOne("Treestar.Shared.Models.DBDomain.User", "User") .WithMany("WatchedNovels") .HasForeignKey("UserId") .OnDelete(DeleteBehavior.Cascade) @@ -220,32 +242,17 @@ namespace DBConnection.Migrations b.Navigation("User"); }); - modelBuilder.Entity("NovelTag", b => - { - b.HasOne("DBConnection.Models.Novel", null) - .WithMany() - .HasForeignKey("NovelsUrl") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DBConnection.Models.Tag", null) - .WithMany() - .HasForeignKey("TagsTagValue") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); - - modelBuilder.Entity("DBConnection.Models.Author", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Author", b => { b.Navigation("Novels"); }); - modelBuilder.Entity("DBConnection.Models.Novel", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.Novel", b => { b.Navigation("Chapters"); }); - modelBuilder.Entity("DBConnection.Models.User", b => + modelBuilder.Entity("Treestar.Shared.Models.DBDomain.User", b => { b.Navigation("WatchedNovels"); }); diff --git a/DBConnection/ModelBuilders/UserNovelBuilder.cs b/DBConnection/ModelBuilders/UserNovelBuilder.cs index cafa5ed..a246885 100644 --- a/DBConnection/ModelBuilders/UserNovelBuilder.cs +++ b/DBConnection/ModelBuilders/UserNovelBuilder.cs @@ -1,5 +1,5 @@ -using DBConnection.Models; using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.ModelBuilders; diff --git a/DBConnection/Models/Author.cs b/DBConnection/Models/Author.cs deleted file mode 100644 index 792cdb6..0000000 --- a/DBConnection/Models/Author.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using Newtonsoft.Json; - -namespace DBConnection.Models; - -public class Author : BaseEntity -{ - [Key] - public string Url { get; set; } - public string Name { get; set; } - [JsonIgnore] - public List Novels { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/BaseEntity.cs b/DBConnection/Models/BaseEntity.cs deleted file mode 100644 index 8293f8f..0000000 --- a/DBConnection/Models/BaseEntity.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace DBConnection.Models; - -public abstract class BaseEntity -{ - public DateTime DateCreated { get; set; } - public DateTime DateModified { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/Chapter.cs b/DBConnection/Models/Chapter.cs deleted file mode 100644 index c379e08..0000000 --- a/DBConnection/Models/Chapter.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace DBConnection.Models; - -public class Chapter : BaseEntity -{ - public int ChapterNumber { get; set; } - public string Name { get; set; } - public string? Content { get; set; } - public string? RawContent { get; set; } - [Key] - public string Url { get; set; } - public DateTime? DatePosted { get; set; } - public DateTime? DateUpdated { get; set; } - public DateTime? LastContentFetch { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/Novel.cs b/DBConnection/Models/Novel.cs deleted file mode 100644 index 39c14e0..0000000 --- a/DBConnection/Models/Novel.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using System.ComponentModel.DataAnnotations.Schema; - -namespace DBConnection.Models; - -public class Novel : BaseEntity -{ - [Key] - public string Url { get; set; } - public Guid Guid { get; set; } - public string Title { get; set; } - public Author Author { get; set; } - public List Tags { get; set; } - public List Chapters { get; set; } - public DateTime LastUpdated { get; set; } - public DateTime DatePosted { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/Tag.cs b/DBConnection/Models/Tag.cs deleted file mode 100644 index f148a79..0000000 --- a/DBConnection/Models/Tag.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using Newtonsoft.Json; - -namespace DBConnection.Models; - -public class Tag : BaseEntity -{ - [Key] - public string TagValue { get; set; } - [JsonIgnore] - public List Novels { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/User.cs b/DBConnection/Models/User.cs deleted file mode 100644 index cf13dda..0000000 --- a/DBConnection/Models/User.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace DBConnection.Models; - -public class User : BaseEntity -{ - [Key] - public int Id { get; set; } - public string Email { get; set; } - public List WatchedNovels { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Models/UserNovel.cs b/DBConnection/Models/UserNovel.cs deleted file mode 100644 index 81d9fec..0000000 --- a/DBConnection/Models/UserNovel.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using Newtonsoft.Json; - -namespace DBConnection.Models; - -public class UserNovel -{ - [JsonIgnore] - public int UserId { get; set; } - public string NovelUrl { get; set; } - public Novel Novel { get; set; } - public User User { get; set; } - public int LastChapterRead { get; set; } -} \ No newline at end of file diff --git a/DBConnection/Readme.md b/DBConnection/Readme.md new file mode 100644 index 0000000..2c22787 --- /dev/null +++ b/DBConnection/Readme.md @@ -0,0 +1,13 @@ +# DBConnection +## Providers +Currently AppDbContext can support: +* Sqlite (default) +* PostgresSql + +The startup project should specify a 'DatabaseProvider' configuration key and provide an appropriate connection string +## Repositories +Repositories added into the DBConnection.Repositories namespace and assignable (implementing) IRepository will be dependency injected. + +Repositories extending a non-generic interface that also implements IRepository (or some descendant interface) will be DI'd as an implementation of that interface. + +As an example, AuthorRepository implements IAuthorRepository which implements IRepository which implements IRepository. Therefore, Author will get added as an implementation of IAuthorRepository. \ No newline at end of file diff --git a/DBConnection/Repositories/AuthorRepository.cs b/DBConnection/Repositories/AuthorRepository.cs index 6f55f21..5ef3009 100644 --- a/DBConnection/Repositories/AuthorRepository.cs +++ b/DBConnection/Repositories/AuthorRepository.cs @@ -1,6 +1,7 @@ -using DBConnection.Models; +using DBConnection.Contexts; using DBConnection.Repositories.Interfaces; using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories; diff --git a/DBConnection/Repositories/BaseRepository.cs b/DBConnection/Repositories/BaseRepository.cs index 92ac496..7bf6296 100644 --- a/DBConnection/Repositories/BaseRepository.cs +++ b/DBConnection/Repositories/BaseRepository.cs @@ -1,8 +1,9 @@ using System.Reflection; -using DBConnection.Models; +using DBConnection.Contexts; using DBConnection.Repositories.Interfaces; using Microsoft.EntityFrameworkCore; using NuGet.Configuration; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories; @@ -30,7 +31,22 @@ public abstract class BaseRepository : IRepository whe return entity; } - public virtual async Task Upsert(TEntityType entity) + public virtual async Task> UpsertMany(IEnumerable entities, bool saveAfter=true) + { + var newEntities = new List(); + foreach (var entity in entities) + { + newEntities.Add(await Upsert(entity, false)); + } + + if (saveAfter) + { + await DbContext.SaveChangesAsync(); + } + return newEntities; + } + + public virtual async Task Upsert(TEntityType entity, bool saveAfter=true) { bool exists = await DbContext.Set().ContainsAsync(entity); if (!exists) @@ -40,10 +56,16 @@ public abstract class BaseRepository : IRepository whe else { var dbEntry = await GetIncluded(entity); - DbContext.Entry(dbEntry).CurrentValues.SetValues(entity); + entity.DateCreated = dbEntry.DateCreated; + var entry = DbContext.Entry(dbEntry); + entry.CurrentValues.SetValues(entity); + entity = dbEntry; } - await DbContext.SaveChangesAsync(); + if (saveAfter) + { + await DbContext.SaveChangesAsync(); + } return entity; } diff --git a/DBConnection/Repositories/ChapterRepository.cs b/DBConnection/Repositories/ChapterRepository.cs new file mode 100644 index 0000000..d070029 --- /dev/null +++ b/DBConnection/Repositories/ChapterRepository.cs @@ -0,0 +1,18 @@ +using DBConnection.Contexts; +using DBConnection.Repositories.Interfaces; +using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.DBDomain; + +namespace DBConnection.Repositories; + +public class ChapterRepository : BaseRepository, IChapterRepository +{ + public ChapterRepository(AppDbContext dbContext) : base(dbContext) + { + } + + protected override IQueryable GetAllIncludedQueryable() + { + return DbContext.Chapters.AsQueryable(); + } +} \ No newline at end of file diff --git a/DBConnection/Repositories/Interfaces/IAuthorRepository.cs b/DBConnection/Repositories/Interfaces/IAuthorRepository.cs index 6ed1c77..5737d81 100644 --- a/DBConnection/Repositories/Interfaces/IAuthorRepository.cs +++ b/DBConnection/Repositories/Interfaces/IAuthorRepository.cs @@ -1,4 +1,4 @@ -using DBConnection.Models; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories.Interfaces; diff --git a/DBConnection/Repositories/Interfaces/IChapterRepository.cs b/DBConnection/Repositories/Interfaces/IChapterRepository.cs new file mode 100644 index 0000000..a2d7a2d --- /dev/null +++ b/DBConnection/Repositories/Interfaces/IChapterRepository.cs @@ -0,0 +1,8 @@ +using Treestar.Shared.Models.DBDomain; + +namespace DBConnection.Repositories.Interfaces; + +public interface IChapterRepository : IRepository +{ + +} \ No newline at end of file diff --git a/DBConnection/Repositories/Interfaces/INovelRepository.cs b/DBConnection/Repositories/Interfaces/INovelRepository.cs index f638715..dd9f2e2 100644 --- a/DBConnection/Repositories/Interfaces/INovelRepository.cs +++ b/DBConnection/Repositories/Interfaces/INovelRepository.cs @@ -1,4 +1,4 @@ -using DBConnection.Models; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories.Interfaces; diff --git a/DBConnection/Repositories/Interfaces/IRepository.cs b/DBConnection/Repositories/Interfaces/IRepository.cs index 902c953..733a46b 100644 --- a/DBConnection/Repositories/Interfaces/IRepository.cs +++ b/DBConnection/Repositories/Interfaces/IRepository.cs @@ -1,4 +1,4 @@ -using DBConnection.Models; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories.Interfaces; @@ -10,9 +10,10 @@ public interface IRepository public interface IRepository : IRepository where TEntityType : BaseEntity { TEntityType Delete(TEntityType entity); - Task Upsert(TEntityType entity); + Task Upsert(TEntityType entity, bool saveAfter=true); Task GetIncluded(TEntityType entity); Task GetIncluded(Func predicate); Task> GetWhereIncluded(Func predicate); Task> GetAllIncluded(); + Task> UpsertMany(IEnumerable entities, bool saveAfter=true); } \ No newline at end of file diff --git a/DBConnection/Repositories/Interfaces/ITagRepository.cs b/DBConnection/Repositories/Interfaces/ITagRepository.cs index 5f0e521..8b779c9 100644 --- a/DBConnection/Repositories/Interfaces/ITagRepository.cs +++ b/DBConnection/Repositories/Interfaces/ITagRepository.cs @@ -1,8 +1,7 @@ -using DBConnection.Models; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories.Interfaces; public interface ITagRepository : IRepository { - } \ No newline at end of file diff --git a/DBConnection/Repositories/NovelRepository.cs b/DBConnection/Repositories/NovelRepository.cs index 2fe79fd..a80d6f5 100644 --- a/DBConnection/Repositories/NovelRepository.cs +++ b/DBConnection/Repositories/NovelRepository.cs @@ -1,6 +1,7 @@ -using DBConnection.Models; +using DBConnection.Contexts; using DBConnection.Repositories.Interfaces; using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories; @@ -8,51 +9,44 @@ public class NovelRepository : BaseRepository, INovelRepository { private readonly IAuthorRepository _authorRepository; private readonly ITagRepository _tagRepository; - public NovelRepository(AppDbContext dbContext, IAuthorRepository authorRepository, ITagRepository tagRepository) : base(dbContext) + private readonly IChapterRepository _chapterRepository; + public NovelRepository(AppDbContext dbContext, IAuthorRepository authorRepository, ITagRepository tagRepository, IChapterRepository chapterRepository) : base(dbContext) { _authorRepository = authorRepository; _tagRepository = tagRepository; + _chapterRepository = chapterRepository; } - public override async Task Upsert(Novel entity) + public override async Task Upsert(Novel entity, bool saveAfter=true) { - var dbEntity = await GetIncluded(entity) ?? entity; // Author - dbEntity.Author = await _authorRepository.GetIncluded(entity.Author) ?? entity.Author; + if (entity.Author != null) + { + entity.Author = await _authorRepository.Upsert(entity.Author, saveAfter); + } + //Tags - List newTags = new List(); - foreach (var tag in entity.Tags) - { - newTags.Add(await _tagRepository.GetIncluded(tag) ?? tag); - } - dbEntity.Tags.Clear(); - dbEntity.Tags = newTags; + var newTags = await _tagRepository.UpsertMany(entity.Tags, false); + entity.Tags.Clear(); + entity.Tags = newTags.ToList(); //chapters - var newChapters = new List(); - foreach (var chapter in entity.Chapters.ToList()) - { - var existingChapter = await DbContext.Chapters.FindAsync(chapter.Url); - if (existingChapter == null) - { - newChapters.Add(chapter); - } - else - { - existingChapter.Name = chapter.Name; - existingChapter.DateUpdated = chapter.DateUpdated; - newChapters.Add(existingChapter); - } - } - dbEntity.Chapters.Clear(); - dbEntity.Chapters = newChapters; + var newChapters = await _chapterRepository.UpsertMany(entity.Chapters, false); + entity.Chapters.Clear(); + entity.Chapters = newChapters.ToList(); // update in db + var dbEntity = await GetIncluded(entity) ?? entity; + entity.Guid = dbEntity.Guid; + DbContext.Entry(dbEntity).CurrentValues.SetValues(entity); if (DbContext.Entry(dbEntity).State == EntityState.Detached) { dbEntity.Guid = Guid.NewGuid(); DbContext.Add(dbEntity); } - await DbContext.SaveChangesAsync(); + if (saveAfter) + { + await DbContext.SaveChangesAsync(); + } return dbEntity; } diff --git a/DBConnection/Repositories/TagRepository.cs b/DBConnection/Repositories/TagRepository.cs index 5a71f99..c287403 100644 --- a/DBConnection/Repositories/TagRepository.cs +++ b/DBConnection/Repositories/TagRepository.cs @@ -1,6 +1,7 @@ -using DBConnection.Models; +using DBConnection.Contexts; using DBConnection.Repositories.Interfaces; using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.DBDomain; namespace DBConnection.Repositories; diff --git a/Shared/AccessLayers/ApiAccessLayer.cs b/Treestar.Shared/AccessLayers/ApiAccessLayer.cs similarity index 97% rename from Shared/AccessLayers/ApiAccessLayer.cs rename to Treestar.Shared/AccessLayers/ApiAccessLayer.cs index daeb105..ee2dd8e 100644 --- a/Shared/AccessLayers/ApiAccessLayer.cs +++ b/Treestar.Shared/AccessLayers/ApiAccessLayer.cs @@ -2,9 +2,9 @@ using System.Net.Mime; using System.Text; using Microsoft.AspNetCore.WebUtilities; using Newtonsoft.Json; -using Shared.Models; +using Treestar.Shared.Models; -namespace Shared.AccessLayers; +namespace Treestar.Shared.AccessLayers; public abstract class ApiAccessLayer { diff --git a/Treestar.Shared/Models/DBDomain/Author.cs b/Treestar.Shared/Models/DBDomain/Author.cs new file mode 100644 index 0000000..307a32c --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/Author.cs @@ -0,0 +1,32 @@ +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Treestar.Shared.Models.DBDomain +{ + public class Author : BaseEntity + { + [Key] + public string Url { get; set; } + public string Name { get; set; } + [JsonIgnore] + public List Novels { get; set; } + + protected bool Equals(Author other) + { + return Url == other.Url; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((Author) obj); + } + + public override int GetHashCode() + { + return Url.GetHashCode(); + } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/BaseEntity.cs b/Treestar.Shared/Models/DBDomain/BaseEntity.cs new file mode 100644 index 0000000..27837f4 --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/BaseEntity.cs @@ -0,0 +1,8 @@ +namespace Treestar.Shared.Models.DBDomain +{ + public abstract class BaseEntity + { + public DateTime DateCreated { get; set; } + public DateTime DateModified { get; set; } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/Chapter.cs b/Treestar.Shared/Models/DBDomain/Chapter.cs new file mode 100644 index 0000000..4f4d242 --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/Chapter.cs @@ -0,0 +1,37 @@ +using System.ComponentModel.DataAnnotations; + +namespace Treestar.Shared.Models.DBDomain +{ + public class Chapter : BaseEntity + { + public int ChapterNumber { get; set; } + public string Name { get; set; } + public string? Content { get; set; } + public string? RawContent { get; set; } + [Key] + public string Url { get; set; } + public DateTime? DatePosted { get; set; } + public DateTime? DateUpdated { get; set; } + public DateTime? LastContentFetch { get; set; } + [Required] + public string NovelUrl { get; set; } + + protected bool Equals(Chapter other) + { + return Url == other.Url; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((Chapter) obj); + } + + public override int GetHashCode() + { + return Url.GetHashCode(); + } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/Novel.cs b/Treestar.Shared/Models/DBDomain/Novel.cs new file mode 100644 index 0000000..8c5c9ab --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/Novel.cs @@ -0,0 +1,21 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.EntityFrameworkCore; +using Treestar.Shared.Models.Enums; + +namespace Treestar.Shared.Models.DBDomain +{ + [Index(nameof(Guid))] + public class Novel : BaseEntity + { + [Key] + public string Url { get; set; } + public Guid Guid { get; set; } + public string Title { get; set; } + public Author? Author { get; set; } + public List Tags { get; set; } + public List Chapters { get; set; } + public NovelStatus Status { get; set; } + public DateTime LastUpdated { get; set; } + public DateTime DatePosted { get; set; } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/Tag.cs b/Treestar.Shared/Models/DBDomain/Tag.cs new file mode 100644 index 0000000..d2ee45c --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/Tag.cs @@ -0,0 +1,31 @@ +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Treestar.Shared.Models.DBDomain +{ + public class Tag : BaseEntity + { + [Key] + public string TagValue { get; set; } + [JsonIgnore] + public List Novels { get; set; } + + protected bool Equals(Tag other) + { + return TagValue == other.TagValue; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((Tag) obj); + } + + public override int GetHashCode() + { + return TagValue.GetHashCode(); + } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/User.cs b/Treestar.Shared/Models/DBDomain/User.cs new file mode 100644 index 0000000..bf49392 --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/User.cs @@ -0,0 +1,12 @@ +using System.ComponentModel.DataAnnotations; + +namespace Treestar.Shared.Models.DBDomain +{ + public class User : BaseEntity + { + [Key] + public int Id { get; set; } + public string Email { get; set; } + public List WatchedNovels { get; set; } + } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DBDomain/UserNovel.cs b/Treestar.Shared/Models/DBDomain/UserNovel.cs new file mode 100644 index 0000000..aad0d31 --- /dev/null +++ b/Treestar.Shared/Models/DBDomain/UserNovel.cs @@ -0,0 +1,14 @@ +using Newtonsoft.Json; + +namespace Treestar.Shared.Models.DBDomain +{ + public class UserNovel + { + [JsonIgnore] + public int UserId { get; set; } + public string NovelUrl { get; set; } + public Novel Novel { get; set; } + public User User { get; set; } + public int LastChapterRead { get; set; } + } +} \ No newline at end of file diff --git a/Shared/Models/DTO/ScrapeNovelRequest.cs b/Treestar.Shared/Models/DTO/Requests/ScrapeNovelRequest.cs similarity index 62% rename from Shared/Models/DTO/ScrapeNovelRequest.cs rename to Treestar.Shared/Models/DTO/Requests/ScrapeNovelRequest.cs index b5cd979..1b1b89c 100644 --- a/Shared/Models/DTO/ScrapeNovelRequest.cs +++ b/Treestar.Shared/Models/DTO/Requests/ScrapeNovelRequest.cs @@ -1,4 +1,4 @@ -namespace Shared.Models.DTO; +namespace Treestar.Shared.Models.DTO.Requests; public class ScrapeNovelRequest { diff --git a/Treestar.Shared/Models/DTO/Requests/ScrapeNovelsRequest.cs b/Treestar.Shared/Models/DTO/Requests/ScrapeNovelsRequest.cs new file mode 100644 index 0000000..60c79c1 --- /dev/null +++ b/Treestar.Shared/Models/DTO/Requests/ScrapeNovelsRequest.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Treestar.Shared.Models.DTO.Requests; + +public class ScrapeNovelsRequest +{ + [JsonProperty("novelUrls")] + public List NovelUrls { get; set; } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/DTO/Responses/ScrapeNovelsResponse.cs b/Treestar.Shared/Models/DTO/Responses/ScrapeNovelsResponse.cs new file mode 100644 index 0000000..74c9628 --- /dev/null +++ b/Treestar.Shared/Models/DTO/Responses/ScrapeNovelsResponse.cs @@ -0,0 +1,9 @@ +using Treestar.Shared.Models.DBDomain; + +namespace Treestar.Shared.Models.DTO.Responses; + +public class ScrapeNovelsResponse +{ + public List SuccessfulNovels { get; set; } + public Dictionary Failures { get; set; } +} \ No newline at end of file diff --git a/Treestar.Shared/Models/Enums/NovelStatus.cs b/Treestar.Shared/Models/Enums/NovelStatus.cs new file mode 100644 index 0000000..dfa44ea --- /dev/null +++ b/Treestar.Shared/Models/Enums/NovelStatus.cs @@ -0,0 +1,9 @@ +namespace Treestar.Shared.Models.Enums; + +public enum NovelStatus +{ + Unknown, + InProgress, + Completed, + Hiatus +} \ No newline at end of file diff --git a/Shared/Models/HttpResponseWrapper.cs b/Treestar.Shared/Models/HttpResponseWrapper.cs similarity index 85% rename from Shared/Models/HttpResponseWrapper.cs rename to Treestar.Shared/Models/HttpResponseWrapper.cs index d7d1c19..595b953 100644 --- a/Shared/Models/HttpResponseWrapper.cs +++ b/Treestar.Shared/Models/HttpResponseWrapper.cs @@ -1,4 +1,4 @@ -namespace Shared.Models; +namespace Treestar.Shared.Models; public class HttpResponseWrapper : HttpResponseWrapper { diff --git a/Shared/Shared.csproj b/Treestar.Shared/Treestar.Shared.csproj similarity index 84% rename from Shared/Shared.csproj rename to Treestar.Shared/Treestar.Shared.csproj index 72d6e25..07592e3 100644 --- a/Shared/Shared.csproj +++ b/Treestar.Shared/Treestar.Shared.csproj @@ -13,6 +13,7 @@ + diff --git a/WebNovelPortal.sln b/WebNovelPortal.sln index 39bdc08..98be864 100644 --- a/WebNovelPortal.sln +++ b/WebNovelPortal.sln @@ -4,7 +4,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebNovelPortal", "WebNovelP EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebNovelPortalAPI", "WebNovelPortalAPI\WebNovelPortalAPI.csproj", "{D24E3BBA-EAA1-4515-9060-56E673CC7FAA}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{639F52AF-9D62-4341-BEE6-0E9243020FC5}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Treestar.Shared", "Treestar.Shared\Treestar.Shared.csproj", "{639F52AF-9D62-4341-BEE6-0E9243020FC5}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DBConnection", "DBConnection\DBConnection.csproj", "{CD895518-DA05-4886-BE14-3E04D62FA2F7}" EndProject diff --git a/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs b/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs index f862f43..e4bcba0 100644 --- a/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs +++ b/WebNovelPortal/AccessLayers/WebApiAccessLayer.cs @@ -1,6 +1,8 @@ -using DBConnection.Models; -using Shared.AccessLayers; -using Shared.Models.DTO; +using Treestar.Shared.AccessLayers; +using Treestar.Shared.Models.DBDomain; +using Treestar.Shared.Models.DTO; +using Treestar.Shared.Models.DTO.Requests; +using Treestar.Shared.Models.DTO.Responses; namespace WebNovelPortal.AccessLayers; @@ -25,4 +27,10 @@ public class WebApiAccessLayer : ApiAccessLayer { return (await SendRequest($"novel/{guid}", HttpMethod.Get)).ResponseObject; } + + public async Task ScrapeNovels(List novelUrls) + { + return (await SendRequest("novel/scrapeNovels", HttpMethod.Post, null, + new ScrapeNovelsRequest {NovelUrls = novelUrls})).ResponseObject; + } } \ No newline at end of file diff --git a/WebNovelPortal/Pages/Index.razor b/WebNovelPortal/Pages/Index.razor index ec56c1a..89da7ce 100644 --- a/WebNovelPortal/Pages/Index.razor +++ b/WebNovelPortal/Pages/Index.razor @@ -1,38 +1,23 @@ @page "/" -@using DBConnection.Models @using WebNovelPortal.AccessLayers Index -

Novels

- - - - - - - - @foreach (var novel in novels) - { - - - - - - } -
TitleAuthorChapter Count
- @novel.Title - - @novel.Author.Name - - @novel.Chapters.Count -
- +@if (loading) +{ + + return; +} +
+ + @code { [Inject] WebApiAccessLayer api { get; set; } string NovelUrl { get; set; } List novels = new List(); + protected bool loading = true; + protected bool awaitingRequest = false; protected override async Task OnAfterRenderAsync(bool firstRender) { if (firstRender) @@ -43,13 +28,42 @@ async Task RequestNovelScrape(string url) { + SetAwaitingRequest(true); await api.RequestNovelScrape(url); + SetAwaitingRequest(false); await RefreshNovels(); } async Task RefreshNovels() { + SetLoading(true); novels = await api.GetNovels(); + SetLoading(false); + StateHasChanged(); + } + + async Task UpdateNovels() + { + SetAwaitingRequest(true); + var res = await api.ScrapeNovels(novels.Select(i => i.Url).ToList()); + SetAwaitingRequest(false); + await RefreshNovels(); + } + + bool IsDisabled() + { + return awaitingRequest; + } + + void SetLoading(bool enabled) + { + loading = enabled; + StateHasChanged(); + } + + void SetAwaitingRequest(bool enabled) + { + awaitingRequest = enabled; StateHasChanged(); } diff --git a/WebNovelPortal/Pages/NovelDetails.razor b/WebNovelPortal/Pages/NovelDetails.razor index a64f7e0..32b025d 100644 --- a/WebNovelPortal/Pages/NovelDetails.razor +++ b/WebNovelPortal/Pages/NovelDetails.razor @@ -1,6 +1,6 @@ @page "/novel/{NovelId}" @using WebNovelPortal.AccessLayers -@using DBConnection.Models + @if (Novel == null) {

Loading...

@@ -8,7 +8,7 @@ else {

@(Novel.Title)

-

Author: (@Novel.Author.Name)

+

Author: @(Novel.Author?.Name ?? "Anonymous")

Date Posted: @Novel.DatePosted

Date Updated: @Novel.LastUpdated

Tags

@@ -20,7 +20,7 @@ else

Chapters

    - @foreach (var chapter in Novel.Chapters) + @foreach (var chapter in Novel.Chapters.OrderBy(i => i.ChapterNumber)) {
  1. @chapter.Name
    2. } diff --git a/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor b/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor new file mode 100644 index 0000000..ce7b358 --- /dev/null +++ b/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor @@ -0,0 +1,5 @@ +
    Loading...
    + +@code { + +} \ No newline at end of file diff --git a/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor.css b/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor.css new file mode 100644 index 0000000..2f697a2 --- /dev/null +++ b/WebNovelPortal/Shared/Components/Display/LoadingDisplay.razor.css @@ -0,0 +1,7 @@ +.loading-display { + display: flex; + justify-content: center; + align-content: center; + align-items: center; + height: 100%; +} \ No newline at end of file diff --git a/WebNovelPortal/Shared/Components/Display/NovelList.razor b/WebNovelPortal/Shared/Components/Display/NovelList.razor new file mode 100644 index 0000000..2049fb0 --- /dev/null +++ b/WebNovelPortal/Shared/Components/Display/NovelList.razor @@ -0,0 +1,33 @@ +@using Microsoft.AspNetCore.Components + +

    Novels

    + + + + + + + + @foreach (var novel in Novels) + { + + + + + + + } +
    TitleAuthorChapter CountLast Updated
    + @novel.Title + + @(novel.Author?.Name ?? "Anonymous") + + @novel.Chapters.Count + + @novel.LastUpdated +
    + +@code { + [Parameter] + public List Novels { get; set; } +} \ No newline at end of file diff --git a/WebNovelPortal/Shared/Layouts/MainLayout.razor.css b/WebNovelPortal/Shared/Layouts/MainLayout.razor.css index 955e64c..ff2d2f4 100644 --- a/WebNovelPortal/Shared/Layouts/MainLayout.razor.css +++ b/WebNovelPortal/Shared/Layouts/MainLayout.razor.css @@ -2,10 +2,17 @@ position: relative; display: flex; flex-direction: column; + width: 100%; } -.main { +article { + flex: 1 1 auto; +} + +main { flex: 1; + display: flex; + flex-flow: column; } .sidebar { @@ -19,6 +26,7 @@ height: 3.5rem; display: flex; align-items: center; + width: 100% } .top-row ::deep a, .top-row .btn-link { diff --git a/WebNovelPortal/WebNovelPortal.csproj b/WebNovelPortal/WebNovelPortal.csproj index 7feead2..69b0f64 100644 --- a/WebNovelPortal/WebNovelPortal.csproj +++ b/WebNovelPortal/WebNovelPortal.csproj @@ -12,8 +12,7 @@ - - + diff --git a/WebNovelPortal/_Imports.razor b/WebNovelPortal/_Imports.razor index f97f31a..5572ab2 100644 --- a/WebNovelPortal/_Imports.razor +++ b/WebNovelPortal/_Imports.razor @@ -9,4 +9,6 @@ @using WebNovelPortal @using WebNovelPortal.Shared @using WebNovelPortal.Shared.Layouts -@using WebNovelPortal.Shared.Components.Layout \ No newline at end of file +@using WebNovelPortal.Shared.Components.Layout +@using WebNovelPortal.Shared.Components.Display +@using Treestar.Shared.Models.DBDomain \ No newline at end of file diff --git a/WebNovelPortal/wwwroot/css/site.css b/WebNovelPortal/wwwroot/css/site.css index 1f4b8cf..57f69b3 100644 --- a/WebNovelPortal/wwwroot/css/site.css +++ b/WebNovelPortal/wwwroot/css/site.css @@ -2,6 +2,7 @@ html, body { font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; + height: 100%; } h1:focus { diff --git a/WebNovelPortalAPI/Controllers/NovelController.cs b/WebNovelPortalAPI/Controllers/NovelController.cs index f5f80ae..e782fe2 100644 --- a/WebNovelPortalAPI/Controllers/NovelController.cs +++ b/WebNovelPortalAPI/Controllers/NovelController.cs @@ -3,12 +3,15 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using DBConnection; -using DBConnection.Models; using DBConnection.Repositories; using DBConnection.Repositories.Interfaces; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Shared.Models.DTO; +using Treestar.Shared.Models.DBDomain; +using Treestar.Shared.Models.DTO; +using Treestar.Shared.Models.DTO.Requests; +using Treestar.Shared.Models.DTO.Responses; +using WebNovelPortalAPI.Exceptions; using WebNovelPortalAPI.Scrapers; namespace WebNovelPortalAPI.Controllers @@ -26,6 +29,17 @@ namespace WebNovelPortalAPI.Controllers _novelRepository = novelRepository; } + private async Task ScrapeNovel(string url) + { + var scraper = MatchScraper(url); + if (scraper == null) + { + throw new NoMatchingScraperException(url); + } + var novel = scraper.ScrapeNovel(url); + return novel; + } + private IScraper? MatchScraper(string novelUrl) { return _scrapers.FirstOrDefault(i => i.MatchesUrl(novelUrl)); @@ -45,27 +59,57 @@ namespace WebNovelPortalAPI.Controllers } [HttpPost] - [Route("scrapeNovel")] - public async Task ScrapeNovel(ScrapeNovelRequest request) + [Route("scrapeNovels")] + public async Task ScrapeNovels(ScrapeNovelsRequest request) { - var scraper = MatchScraper(request.NovelUrl); - if (scraper == null) + var successfulScrapes = new List(); + var failures = new Dictionary(); + foreach (var novelUrl in request.NovelUrls) { - return BadRequest("Invalid url, no valid scraper configured"); + try + { + successfulScrapes.Add(await ScrapeNovel(novelUrl)); + } + catch (Exception e) + { + failures[novelUrl] = e; + } } - Novel novel; + IEnumerable successfulUploads; try { - novel = scraper.ScrapeNovel(request.NovelUrl); + successfulUploads = await _novelRepository.UpsertMany(successfulScrapes); } catch (Exception e) { return StatusCode(500, e); } + return Ok(new ScrapeNovelsResponse + { + Failures = failures, + SuccessfulNovels = successfulScrapes + }); + } - var novelUpload = await _novelRepository.Upsert(novel); - return Ok(novelUpload); + [HttpPost] + [Route("scrapeNovel")] + public async Task ScrapeNovel(ScrapeNovelRequest request) + { + try + { + var novel = await ScrapeNovel(request.NovelUrl); + var dbNovel = await _novelRepository.Upsert(novel); + return Ok(dbNovel); + } + catch (NoMatchingScraperException e) + { + return BadRequest("Invalid url, no valid scraper configured"); + } + catch (Exception e) + { + return StatusCode(500, e); + } } } } diff --git a/WebNovelPortalAPI/Exceptions/NoMatchingScraperException.cs b/WebNovelPortalAPI/Exceptions/NoMatchingScraperException.cs new file mode 100644 index 0000000..e0963c3 --- /dev/null +++ b/WebNovelPortalAPI/Exceptions/NoMatchingScraperException.cs @@ -0,0 +1,10 @@ +namespace WebNovelPortalAPI.Exceptions; + +public class NoMatchingScraperException: Exception +{ + public NoMatchingScraperException(string novelUrl) : base($"Novel URL {novelUrl} did not match any registered web scraper.") + { + } + + +} \ No newline at end of file diff --git a/WebNovelPortalAPI/Program.cs b/WebNovelPortalAPI/Program.cs index d17286c..096cb28 100644 --- a/WebNovelPortalAPI/Program.cs +++ b/WebNovelPortalAPI/Program.cs @@ -1,4 +1,5 @@ using DBConnection; +using DBConnection.Contexts; using DBConnection.Extensions; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; diff --git a/WebNovelPortalAPI/Scrapers/AbstractScraper.cs b/WebNovelPortalAPI/Scrapers/AbstractScraper.cs index 3fd9a58..8a0d625 100644 --- a/WebNovelPortalAPI/Scrapers/AbstractScraper.cs +++ b/WebNovelPortalAPI/Scrapers/AbstractScraper.cs @@ -1,6 +1,6 @@ using System.Text.RegularExpressions; -using DBConnection.Models; using HtmlAgilityPack; +using Treestar.Shared.Models.DBDomain; namespace WebNovelPortalAPI.Scrapers; @@ -18,6 +18,12 @@ public abstract class AbstractScraper : IScraper protected virtual string? TagPattern { get; } protected virtual string? DatePostedPattern { get; } protected virtual string? DateUpdatedPattern { get; } + + protected virtual (DateTime? Posted, DateTime? Updated) GetDateTimeForChapter(HtmlNode linkNode, HtmlNode baseNode, + string baseUrl, string novelUrl) + { + return (null, null); + } public virtual bool MatchesUrl(string url) { @@ -25,43 +31,56 @@ public abstract class AbstractScraper : IScraper return regex.IsMatch(url); } - protected virtual string GetNovelTitle(HtmlDocument document) + protected virtual string GetNovelTitle(HtmlDocument document, string baseUrl, string novelUrl) { var xpath = WorkTitlePattern; return document.DocumentNode.SelectSingleNode(xpath).InnerText; } - protected virtual Author GetAuthor(HtmlDocument document, string baseUrl) + protected virtual Author GetAuthor(HtmlDocument document, string baseUrl, string novelUrl) { var nameXPath = AuthorNamePattern; var urlXPath = AuthorLinkPattern; - var authorName = document.DocumentNode.SelectSingleNode(nameXPath).InnerText; - var authorUrl = document.DocumentNode.SelectSingleNode(urlXPath).Attributes["href"].Value; - Author author = new Author + try { - Name = authorName, - Url = $"{baseUrl + authorUrl}" - }; - return author; + var authorName = document.DocumentNode.SelectSingleNode(nameXPath).InnerText; + var authorUrl = document.DocumentNode.SelectSingleNode(urlXPath).Attributes["href"].Value; + Author author = new Author + { + Name = authorName, + Url = $"{baseUrl + authorUrl}" + }; + return author; + } + catch (Exception e) + { + return null; + } } - protected virtual List GetChapters(HtmlDocument document, string baseUrl) + protected virtual List GetChapters(HtmlDocument document, string baseUrl, string novelUrl) { var urlxpath = ChapterUrlPattern; var namexpath = ChapterNamePattern; var urlnodes = document.DocumentNode.SelectNodes(urlxpath); - var chapters = urlnodes.Select((node, i) => new Chapter + var chapters = urlnodes.Select((node, i) => { - ChapterNumber = i + 1, - Url = $"{baseUrl}{node.Attributes["href"].Value}", - Name = node.SelectSingleNode(namexpath).InnerText + var dates = GetDateTimeForChapter(node, document.DocumentNode, baseUrl, novelUrl); + return new Chapter + { + ChapterNumber = i + 1, + Url = $"{baseUrl}{node.Attributes["href"].Value}", + Name = node.SelectSingleNode(namexpath).InnerText, + DatePosted = dates.Posted, + DateUpdated = dates.Updated + }; }); return chapters.ToList(); } - protected virtual List GetTags(HtmlDocument document) + protected virtual List GetTags(HtmlDocument document, string baseUrl, string novelUrl) { var xpath = TagPattern; var nodes = document.DocumentNode.SelectNodes(xpath); @@ -71,13 +90,13 @@ public abstract class AbstractScraper : IScraper }).ToList(); } - protected virtual DateTime GetPostedDate(HtmlDocument document) + protected virtual DateTime GetPostedDate(HtmlDocument document, string baseUrl, string novelUrl) { var xpath = DatePostedPattern; return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); } - protected virtual DateTime GetLastUpdatedDate(HtmlDocument document) + protected virtual DateTime GetLastUpdatedDate(HtmlDocument document, string baseUrl, string novelUrl) { var xpath = DateUpdatedPattern; return DateTime.Parse(document.DocumentNode.SelectSingleNode(xpath).InnerText); @@ -96,12 +115,12 @@ public abstract class AbstractScraper : IScraper var novelUrl = new Regex(UrlMatchPattern).Match(url).Value; return new Novel { - Author = GetAuthor(doc, baseUrl), - Chapters = GetChapters(doc, baseUrl), - DatePosted = GetPostedDate(doc), - LastUpdated = GetLastUpdatedDate(doc), - Tags = GetTags(doc), - Title = GetNovelTitle(doc), + Author = GetAuthor(doc, baseUrl, novelUrl), + Chapters = GetChapters(doc, baseUrl, novelUrl), + DatePosted = GetPostedDate(doc, baseUrl, novelUrl), + LastUpdated = GetLastUpdatedDate(doc, baseUrl, novelUrl), + Tags = GetTags(doc, baseUrl, novelUrl), + Title = GetNovelTitle(doc, baseUrl, novelUrl), Url = novelUrl }; } diff --git a/WebNovelPortalAPI/Scrapers/IScraper.cs b/WebNovelPortalAPI/Scrapers/IScraper.cs index f0b8ce6..6bc719d 100644 --- a/WebNovelPortalAPI/Scrapers/IScraper.cs +++ b/WebNovelPortalAPI/Scrapers/IScraper.cs @@ -1,4 +1,4 @@ -using DBConnection.Models; +using Treestar.Shared.Models.DBDomain; namespace WebNovelPortalAPI.Scrapers; diff --git a/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs b/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs index 1eccb0c..e7792f0 100644 --- a/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs +++ b/WebNovelPortalAPI/Scrapers/KakuyomuScraper.cs @@ -1,6 +1,5 @@ using System.Reflection.Metadata; using System.Text.RegularExpressions; -using DBConnection.Models; using HtmlAgilityPack; namespace WebNovelPortalAPI.Scrapers; @@ -19,7 +18,7 @@ public class KakuyomuScraper : AbstractScraper protected override string? ChapterNamePattern => @"span"; - protected override string? ChapterPostedPattern => base.ChapterPostedPattern; + protected override string? ChapterPostedPattern => @"time"; protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern; @@ -29,8 +28,10 @@ public class KakuyomuScraper : AbstractScraper protected override string? DateUpdatedPattern => @"//time[@itemprop='dateModified']"; - public string? ScrapeChapterContent(string chapterUrl) + protected override (DateTime? Posted, DateTime? Updated) GetDateTimeForChapter(HtmlNode linkNode, HtmlNode baseNode, string baseUrl, + string novelUrl) { - throw new NotImplementedException(); + var datePosted = linkNode.SelectSingleNode(ChapterPostedPattern).Attributes["datetime"].Value; + return (DateTime.Parse(datePosted), null); } } \ No newline at end of file diff --git a/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs b/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs index c0decde..a5a8b5b 100644 --- a/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs +++ b/WebNovelPortalAPI/Scrapers/SyosetuScraper.cs @@ -1,10 +1,15 @@ +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Treestar.Shared.Models.DBDomain; + namespace WebNovelPortalAPI.Scrapers; public class SyosetuScraper : AbstractScraper { + protected override string UrlMatchPattern => @"https?:\/\/\w+\.syosetu\.com\/\w+\/?"; - protected override string BaseUrlPattern => @"https?:\/\/\w+\.syosetu\.com\/?"; + protected override string BaseUrlPattern => @"https?:\/\/\w+\.syosetu\.com"; protected override string? WorkTitlePattern => @"//p[@class='novel_title']"; @@ -14,15 +19,101 @@ public class SyosetuScraper : AbstractScraper protected override string? ChapterUrlPattern => @"//dl[@class='novel_sublist2']//a"; - protected override string? ChapterNamePattern => @"//dl[@class='novel_sublist2']//a"; + protected override string? ChapterPostedPattern => @"following-sibling::dt[@class='long_update']"; - protected override string? ChapterPostedPattern => base.ChapterPostedPattern; + protected override string? ChapterUpdatedPattern => @"span"; - protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern; + protected override string? TagPattern => @"//th[text()='キーワード']/following-sibling::td"; - protected override string? TagPattern => base.TagPattern; + protected override string? DatePostedPattern => @"//th[text()='掲載日']/following-sibling::td"; - protected override string? DatePostedPattern => base.DatePostedPattern; + protected override string? DateUpdatedPattern => @"//th[contains(text(),'掲載日')]/following-sibling::td"; - protected override string? DateUpdatedPattern => base.DateUpdatedPattern; + private HtmlDocument? GetInfoPage(string baseUrl, string novelUrl) + { + string novelInfoBase = $"/novelview/infotop/ncode/"; + string novelRegex = @"https?:\/\/\w+\.syosetu\.com\/(\w+)\/?"; + string novelCode = new Regex(novelRegex).Match(novelUrl).Groups[1].Value; + string novelInfoPage = $"{baseUrl}{novelInfoBase}{novelCode}"; + var web = new HtmlWeb(); + return web.Load(novelInfoPage); + } + + protected override List GetChapters(HtmlDocument document, string baseUrl, string novelUrl) + { + string dateUpdatedRegex = @"\d\d\d\d\/\d\d\/\d\d \d\d:\d\d"; + var nodes = document.DocumentNode.SelectNodes(ChapterUrlPattern); + return nodes.Select((node,i) => + { + var datePostedNode = node.ParentNode.SelectSingleNode(ChapterPostedPattern); + var datePosted = DateTime.Parse(new Regex(dateUpdatedRegex).Match(datePostedNode.InnerText).Value); + var dateUpdatedNode = datePostedNode.SelectSingleNode(ChapterUpdatedPattern); + DateTime dateUpdated; + if (dateUpdatedNode == null) + { + dateUpdated = datePosted; + } + else + { + dateUpdated = DateTime.Parse(new Regex(dateUpdatedRegex).Match(dateUpdatedNode.Attributes["title"].Value).Value); + } + return new Chapter + { + Name = node.InnerText, + Url = baseUrl + node.Attributes["href"].Value, + ChapterNumber = i+1, + DatePosted = datePosted, + DateUpdated = dateUpdated + }; + }).ToList(); + } + + protected override Author GetAuthor(HtmlDocument document, string baseUrl, string novelUrl) + { + var authorLink = document.DocumentNode.SelectSingleNode(AuthorLinkPattern)?.Attributes["href"].Value ?? null; + if (string.IsNullOrEmpty(authorLink)) + { + return null; + } + var authorName = document.DocumentNode.SelectSingleNode(AuthorNamePattern).InnerText.Replace("\n", ""); + return new Author + { + Name = authorName, + Url = authorLink + }; + } + + protected override DateTime GetPostedDate(HtmlDocument document, string baseUrl, string novelUrl) + { + var doc = GetInfoPage(baseUrl, novelUrl); + if (doc == null) + { + return DateTime.MinValue; + } + + var node = doc.DocumentNode.SelectSingleNode(DatePostedPattern); + return DateTime.Parse(node.InnerText); + } + + protected override DateTime GetLastUpdatedDate(HtmlDocument document, string baseUrl, string novelUrl) + { + var doc = GetInfoPage(baseUrl, novelUrl); + if (doc == null) + { + return DateTime.MinValue; + } + return DateTime.Parse(doc.DocumentNode.SelectNodes(DateUpdatedPattern)[1].InnerText); + } + + protected override List GetTags(HtmlDocument document, string baseUrl, string novelUrl) + { + var doc = GetInfoPage(baseUrl, novelUrl); + if (doc == null) + { + return new List(); + } + + var tags = doc.DocumentNode.SelectSingleNode(TagPattern).InnerText.Replace("\n", "").Replace(" ", " ").Split(' '); + return tags.Select(i => new Tag {TagValue = i}).ToList(); + } } \ No newline at end of file diff --git a/WebNovelPortalAPI/WebNovelPortalAPI.csproj b/WebNovelPortalAPI/WebNovelPortalAPI.csproj index 9b32535..78850ae 100644 --- a/WebNovelPortalAPI/WebNovelPortalAPI.csproj +++ b/WebNovelPortalAPI/WebNovelPortalAPI.csproj @@ -22,7 +22,7 @@ - + diff --git a/WebNovelPortalAPI/appsettings.json b/WebNovelPortalAPI/appsettings.json index 84cc7d0..71c6c22 100644 --- a/WebNovelPortalAPI/appsettings.json +++ b/WebNovelPortalAPI/appsettings.json @@ -6,7 +6,9 @@ } }, "ConnectionStrings": { - "DefaultConnection": "Data Source=test_db" + "Sqlite": "Data Source=test_db", + "PostgresSql": "placeholder" }, + "DatabaseProvider": "Sqlite", "AllowedHosts": "*" }