Refactor and novel18 support (added cookie support in general to AbstractScraper.cs

Common/AccessLayers/ApiAccessLayer.cs

@@ -0,0 +1,93 @@
+using System.Net.Mime;
+using System.Text;
+using Common.Models;
+using Microsoft.AspNetCore.WebUtilities;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace Common.AccessLayers;
+
+public abstract class ApiAccessLayer
+{
+    private readonly HttpClient _httpClient;
+    private readonly IAccessLayerAuthenticationProvider _authenticationProvider;
+    protected readonly ILogger Logger;
+
+    protected ApiAccessLayer(string apiBaseUrl, IAccessLayerAuthenticationProvider authenticationProvider, ILogger logger)
+    {
+        _authenticationProvider = authenticationProvider;
+        Logger = logger;
+        var handler = new HttpClientHandler() 
+        {
+            ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator
+        };
+        _httpClient = new HttpClient(handler);
+        _httpClient.BaseAddress = new Uri(apiBaseUrl, UriKind.Absolute);
+    }
+
+    private async Task<HttpResponseWrapper> SendRequest(HttpRequestMessage message)
+    {
+        await _authenticationProvider.AddAuthentication(message);
+        var response = await _httpClient.SendAsync(message);
+        if (!response.IsSuccessStatusCode)
+        {
+            Logger.LogError("Response returned status code {statusCode} with reason {reason} and content {content}", response.StatusCode, response.ReasonPhrase, await response.Content.ReadAsStringAsync());
+        }
+        return new HttpResponseWrapper()
+        {
+            HttpResponseMessage = response
+        };
+    }
+
+    private async Task<HttpResponseWrapper<T>> SendRequest<T>(HttpRequestMessage message)
+    {
+        var wrapper = await SendRequest(message);
+        if (wrapper.HttpResponseMessage.IsSuccessStatusCode)
+        {
+            var parsedJson =
+                JsonConvert.DeserializeObject<T>(await wrapper.HttpResponseMessage.Content.ReadAsStringAsync());
+            return new HttpResponseWrapper<T>
+            {
+                HttpResponseMessage = wrapper.HttpResponseMessage,
+                ResponseObject = parsedJson
+            };
+        }
+        return new HttpResponseWrapper<T>
+        {
+            HttpResponseMessage = wrapper.HttpResponseMessage,
+            ResponseObject = default(T)
+        };
+
+    }
+
+    protected HttpRequestMessage CreateRequestMessage(string endpoint, HttpMethod method, Dictionary<string, string>? queryParams = null,
+        object? data = null)
+    {
+        HttpRequestMessage message = new HttpRequestMessage();
+        string uri = endpoint;
+        if (queryParams != null)
+        {
+            uri = QueryHelpers.AddQueryString(endpoint, queryParams);
+        }
+        message.RequestUri = new Uri(uri, UriKind.Relative);
+        message.Method = method;
+        if (data != null)
+        {
+            message.Content = new StringContent(JsonConvert.SerializeObject(data), Encoding.UTF8, MediaTypeNames.Application.Json);
+        }
+
+        return message;
+    }
+
+    protected async Task<HttpResponseWrapper> SendRequest(string endpoint, HttpMethod method, Dictionary<string, string>? queryParams = null, object? data = null)
+    {
+        HttpRequestMessage message = CreateRequestMessage(endpoint, method, queryParams, data);
+        return await SendRequest(message);
+    }
+    
+    protected async Task<HttpResponseWrapper<T>> SendRequest<T>(string endpoint, HttpMethod method, Dictionary<string, string>? queryParams = null, object? data = null)
+    {
+        HttpRequestMessage message = CreateRequestMessage(endpoint, method, queryParams, data);
+        return await SendRequest<T>(message);
+    }
+}

Common/AccessLayers/IAccessLayerAuthenticationProvider.cs

@@ -0,0 +1,6 @@
+namespace Common.AccessLayers;
+
+public interface IAccessLayerAuthenticationProvider
+{
+    Task AddAuthentication(HttpRequestMessage request);
+}

Common/Authentication/JwtBearer/JWTAuthenticationExtension.cs

@@ -0,0 +1,32 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.IdentityModel.Tokens;
+
+namespace Common.Authentication.JwtBearer;
+
+public static class JWTAuthenticationExtension
+{
+    public static void AddJwtBearerAuth(this IServiceCollection services, IConfiguration configuration)
+    {
+        var jwtAuthOptions = configuration.GetRequiredSection(JwtBearerAuthenticationOptions.ConfigrationSection)
+            .Get<JwtBearerAuthenticationOptions>();
+        services.AddAuthentication(opt =>
+        {
+            opt.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
+        }).AddJwtBearer(opt =>
+        {
+            opt.Authority = jwtAuthOptions.Authority;
+            opt.Audience = jwtAuthOptions.Audience;
+            opt.TokenValidationParameters = new TokenValidationParameters
+            {
+                NameClaimType = ClaimTypes.Name,
+                ValidateAudience = !string.IsNullOrEmpty(jwtAuthOptions.Audience),
+                ValidateIssuer = true,
+                ValidateIssuerSigningKey = true,
+                ValidateLifetime = true
+            };
+        });
+    }
+}

Common/Authentication/JwtBearer/JwtBearerAuthenticationOptions.cs

@@ -0,0 +1,8 @@
+namespace Common.Authentication.JwtBearer;
+
+public class JwtBearerAuthenticationOptions
+{
+    public const string ConfigrationSection = "JwtBearerAuthOptions";
+    public string Authority { get; set; } = null!;
+    public string? Audience { get; set; }
+}

Common/Authentication/OIDC/AuthenticationExtension.cs

@@ -0,0 +1,45 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Authentication.Cookies;
+using Microsoft.AspNetCore.Authentication.OpenIdConnect;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.IdentityModel.Protocols.OpenIdConnect;
+using Microsoft.IdentityModel.Tokens;
+using WebNovelPortal.Authentication;
+
+namespace Common.Authentication.OIDC;
+
+public static class AuthenticationExtension
+{
+    public static void AddOIDCAuth(this IServiceCollection services, IConfiguration configuration)
+    {
+        var oidcConfig = configuration.GetRequiredSection(OpenIdConnectAuthenticationOptions.ConfigurationSection)
+            .Get<OpenIdConnectAuthenticationOptions>();
+        services.AddAuthentication(opt =>
+            {
+                opt.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+                opt.DefaultChallengeScheme = OpenIdConnectDefaults.AuthenticationScheme;
+            })
+            .AddCookie()
+            .AddOpenIdConnect(opt =>
+            {
+                opt.Authority = oidcConfig.Authority;
+                opt.ClientId = oidcConfig.ClientId;
+                opt.ClientSecret = oidcConfig.ClientSecret;
+
+                opt.ResponseType = OpenIdConnectResponseType.Code;
+                opt.GetClaimsFromUserInfoEndpoint = false;
+                opt.SaveTokens = true;
+                opt.UseTokenLifetime = true;
+                foreach (var scope in oidcConfig.Scopes.Split(" "))
+                {
+                    opt.Scope.Add(scope);
+                }
+
+                opt.TokenValidationParameters = new TokenValidationParameters
+                {
+                    NameClaimType = ClaimTypes.Name
+                };
+            });
+    }
+}

Common/Authentication/OIDC/OpenIdConnectAuthenticationOptions.cs

@@ -0,0 +1,10 @@
+namespace WebNovelPortal.Authentication;
+
+public class OpenIdConnectAuthenticationOptions
+{
+    public const string ConfigurationSection = "OIDCAuthOptions";
+    public string Authority { get; set; }
+    public string ClientId { get; set; }
+    public string ClientSecret { get; set; }
+    public string Scopes { get; set; }
+}

Common/Common.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <Folder Include="BlazorComponents" />
+      <Folder Include="Interfaces" />
+      <Folder Include="Utility" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.7" />
+      <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.7" />
+      <PackageReference Include="Microsoft.AspNetCore.WebUtilities" Version="2.2.0" />
+      <PackageReference Include="Microsoft.EntityFrameworkCore.Abstractions" Version="6.0.7" />
+      <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Reference Include="Microsoft.Extensions.Configuration.Abstractions">
+        <HintPath>..\..\..\..\..\usr\share\dotnet\shared\Microsoft.AspNetCore.App\6.0.0\Microsoft.Extensions.Configuration.Abstractions.dll</HintPath>
+      </Reference>
+      <Reference Include="Microsoft.Extensions.DependencyInjection.Abstractions">
+        <HintPath>..\..\..\..\..\usr\share\dotnet\shared\Microsoft.AspNetCore.App\6.0.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll</HintPath>
+      </Reference>
+    </ItemGroup>
+
+</Project>

Common/Models/DBDomain/Author.cs

@@ -0,0 +1,32 @@
+using System.ComponentModel.DataAnnotations;
+using Newtonsoft.Json;
+
+namespace Common.Models.DBDomain
+{
+    public class Author : BaseEntity
+    {
+        [Key]
+        public string Url { get; set; }
+        public string Name { get; set; }
+        [JsonIgnore]
+        public List<Novel> Novels { get; set; }
+
+        protected bool Equals(Author other)
+        {
+            return Url == other.Url;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((Author) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Url.GetHashCode();
+        }
+    }
+}

Common/Models/DBDomain/BaseEntity.cs

@@ -0,0 +1,8 @@
+namespace Common.Models.DBDomain
+{
+    public abstract class BaseEntity
+    {
+        public DateTime DateCreated { get; set; }
+        public DateTime DateModified { get; set; }
+    }
+}

Common/Models/DBDomain/Chapter.cs

@@ -0,0 +1,37 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Common.Models.DBDomain
+{
+    public class Chapter : BaseEntity
+    {
+        public int ChapterNumber { get; set; }
+        public string Name { get; set; }
+        public string? Content { get; set; }
+        public string? RawContent { get; set; }
+        [Key]
+        public string Url { get; set; }
+        public DateTime? DatePosted { get; set; }
+        public DateTime? DateUpdated { get; set; }
+        public DateTime? LastContentFetch { get; set; }
+        [Required]
+        public Novel Novel { get; set; }
+
+        protected bool Equals(Chapter other)
+        {
+            return Url == other.Url;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((Chapter) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Url.GetHashCode();
+        }
+    }
+}

Common/Models/DBDomain/Novel.cs

@@ -0,0 +1,39 @@
+using System.ComponentModel.DataAnnotations;
+using Common.Models.Enums;
+using Microsoft.EntityFrameworkCore;
+
+namespace Common.Models.DBDomain
+{
+    [Index(nameof(Guid))]
+    public class Novel : BaseEntity
+    {
+        [Key]
+        public string Url { get; set; }
+        public Guid Guid { get; set; }
+        public string Title { get; set; }
+        public Author? Author { get; set; }
+        public List<Tag> Tags { get; set; }
+        public List<Chapter> Chapters { get; set; }
+        public NovelStatus Status { get; set; }
+        public DateTime LastUpdated { get; set; }
+        public DateTime DatePosted { get; set; }
+
+        protected bool Equals(Novel other)
+        {
+            return Url == other.Url;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((Novel) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Url.GetHashCode();
+        }
+    }
+}

Common/Models/DBDomain/Tag.cs

@@ -0,0 +1,46 @@
+using System.ComponentModel.DataAnnotations;
+using Newtonsoft.Json;
+
+namespace Common.Models.DBDomain
+{
+    public class Tag : BaseEntity
+    {
+        [Key]
+        public string TagValue { get; set; }
+        [JsonIgnore]
+        public List<Novel> Novels { get; set; }
+
+        protected bool Equals(Tag other)
+        {
+            return TagValue == other.TagValue;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((Tag) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return TagValue.GetHashCode();
+        }
+
+        public static Tag GetSiteTag(string siteUrl)
+        {
+            return new Tag {TagValue = $"site:{siteUrl.TrimEnd('/')}"};
+        }
+
+        public static Tag GetOriginalWorkTag()
+        {
+            return new Tag {TagValue = "original_work"};
+        }
+
+        public static Tag GetNsfwTag()
+        {
+            return new Tag {TagValue = "NSFW"};
+        }
+    }
+}

Common/Models/DBDomain/User.cs

@@ -0,0 +1,32 @@
+using System.ComponentModel.DataAnnotations;
+using Microsoft.EntityFrameworkCore;
+
+namespace Common.Models.DBDomain
+{
+    [Index(nameof(Email))]
+    public class User : BaseEntity
+    {
+        [Key]
+        public int Id { get; set; }
+        public string Email { get; set; }
+        public List<UserNovel> WatchedNovels { get; set; }
+
+        protected bool Equals(User other)
+        {
+            return Id == other.Id;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((User) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Id;
+        }
+    }
+}

Common/Models/DBDomain/UserNovel.cs

@@ -0,0 +1,32 @@
+using Newtonsoft.Json;
+
+namespace Common.Models.DBDomain
+{
+    public class UserNovel
+    {
+        public int UserId { get; set; }
+        public string NovelUrl { get; set; }
+        public Novel Novel { get; set; }
+        [JsonIgnore]
+        public User User { get; set; }
+        public int LastChapterRead { get; set; }
+
+        protected bool Equals(UserNovel other)
+        {
+            return UserId == other.UserId && NovelUrl == other.NovelUrl;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((UserNovel) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(UserId, NovelUrl);
+        }
+    }
+}

Common/Models/DTO/Requests/ScrapeNovelRequest.cs

@@ -0,0 +1,6 @@
+namespace Common.Models.DTO.Requests;
+
+public class ScrapeNovelRequest
+{
+    public string NovelUrl { get; set; }
+}

Common/Models/DTO/Requests/ScrapeNovelsRequest.cs

@@ -0,0 +1,9 @@
+using Newtonsoft.Json;
+
+namespace Common.Models.DTO.Requests;
+
+public class ScrapeNovelsRequest
+{
+    [JsonProperty("novelUrls")]
+    public List<string> NovelUrls { get; set; }
+}

Common/Models/DTO/Responses/ScrapeNovelsResponse.cs

@@ -0,0 +1,9 @@
+using Common.Models.DBDomain;
+
+namespace Common.Models.DTO.Responses;
+
+public class ScrapeNovelsResponse
+{
+    public List<Novel> SuccessfulNovels { get; set; }
+    public Dictionary<string, Exception> Failures { get; set; }
+}

Common/Models/Enums/NovelStatus.cs

@@ -0,0 +1,9 @@
+namespace Common.Models.Enums;
+
+public enum NovelStatus
+{
+    Unknown,
+    InProgress,
+    Completed,
+    Hiatus
+}

Common/Models/HttpResponseWrapper.cs

@@ -0,0 +1,11 @@
+namespace Common.Models;
+
+public class HttpResponseWrapper<T> : HttpResponseWrapper
+{
+    public T? ResponseObject { get; set; }
+}
+
+public class HttpResponseWrapper
+{
+    public HttpResponseMessage HttpResponseMessage { get; set; }
+}