diff --git a/.gitignore b/.gitignore index e67f738..6455a38 100644 --- a/.gitignore +++ b/.gitignore @@ -134,4 +134,9 @@ $RECYCLE.BIN/ _NCrunch* # Local user appsettings -appsettings.Local.json \ No newline at end of file +appsettings.Local.json + +# Fusion Builds +schema.graphql +*.fsp +gateway.fgp \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ada212f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,36 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `FictionArchive.sln` ties together the gateway and all subgraph services. +- `FictionArchive.API`: Fusion gateway host; GraphQL endpoint at `/graphql`, health at `/healthz`, gateway configuration in `gateway.fgp`, and helper script `build_gateway.py`. +- `FictionArchive.Service.*`: GraphQL subgraphs (`AuthenticationService`, `FileService`, `NovelService`, `SchedulerService`, `TranslationService`, `UserService`) plus shared helpers in `FictionArchive.Service.Shared`. +- `FictionArchive.Common`: shared enums and hosting extensions used across services. +- Environment/config files live beside each service (`appsettings*.json`, `Properties/launchSettings.json`); build outputs under `bin/` and `obj/` should stay untracked. + +## Build, Test, and Development Commands +- `dotnet restore` then `dotnet build FictionArchive.sln` (Debug by default) to validate all projects compile. +- Run the gateway: `dotnet run --project FictionArchive.API` (serves HTTPS; ensure certificates are trusted locally). +- Run a subgraph locally: `dotnet run --project FictionArchive.Service.NovelService` (or any other service) to debug a single domain. +- Rebuild the Fusion gateway config after subgraph changes: `python FictionArchive.API/build_gateway.py` (requires Python 3 and the `fusion` CLI on PATH; uses `gateway_skip.txt` to omit services). +- If tests are added, prefer `dotnet test FictionArchive.sln` to cover the whole solution. + +## Coding Style & Naming Conventions +- Target .NET 8/C# 12; use 4-space indentation and file-scoped namespaces where practical. +- PascalCase for classes, records, interfaces, and public members; camelCase for locals/parameters; suffix async methods with `Async`. +- Favor dependency injection and extension methods for service wiring (see `Program.cs` files and `FictionArchive.Service.Shared/Extensions`). +- Keep GraphQL schema files and other generated artifacts out of commits unless intentionally versioned. + +## Testing Guidelines +- No dedicated test projects exist yet; when adding tests, create `*.Tests` projects aligned to each service (e.g., `FictionArchive.Service.NovelService.Tests`) and name test files `*Tests.cs`. +- Prefer xUnit with fluent assertions; aim for coverage on controllers/resolvers, integration events, and critical extension methods. +- Use in-memory fakes or test containers for external dependencies to keep tests deterministic. + +## Commit & Pull Request Guidelines +- Follow the observed pattern: `[FA-123] Short, imperative summary` (reference the tracker ID and keep scope focused). +- Keep commits small and self-contained; include relevant config/schema updates produced by the gateway build script when behavior changes. +- PRs should describe the problem, the solution, and any follow-up work; link to issues, attach GraphQL schema diffs or sample queries when applicable, and note any manual steps (migrations, secrets). + +## Security & Configuration Tips +- Do not commit secrets; use user secrets or environment variables for API keys and connection strings referenced in `appsettings*.json`. +- Verify HTTPS is enabled locally; adjust `launchSettings.json` only when necessary and document non-default ports. +- Regenerate `gateway.fgp` after changing subgraph schemas to avoid stale compositions. diff --git a/FictionArchive.API/FictionArchive.API.csproj b/FictionArchive.API/FictionArchive.API.csproj index 4246f74..752aa54 100644 --- a/FictionArchive.API/FictionArchive.API.csproj +++ b/FictionArchive.API/FictionArchive.API.csproj @@ -21,6 +21,11 @@ + + + + + diff --git a/FictionArchive.API/build_gateway.bat b/FictionArchive.API/build_gateway.bat deleted file mode 100644 index 598d0e6..0000000 --- a/FictionArchive.API/build_gateway.bat +++ /dev/null @@ -1,99 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -set ROOT=%~dp0 - -for %%A in ("%ROOT%..") do set SERVICES_DIR=%%~fA\ - -REM ---------------------------------------- -REM List of project names to skip -REM (space-separated, match folder names exactly) -REM ---------------------------------------- -set SKIP_PROJECTS=FictionArchive.Service.Shared FictionArchive.Service.AuthenticationService - -echo ---------------------------------------- -echo Finding GraphQL services... -echo ---------------------------------------- - -set SERVICE_LIST= - -for /d %%F in ("%SERVICES_DIR%FictionArchive.Service.*") do ( - set "PROJECT_NAME=%%~nxF" - set "SKIP=0" - - REM Check if this project name is in the skip list - for %%X in (%SKIP_PROJECTS%) do ( - if /I "!PROJECT_NAME!"=="%%X" ( - set "SKIP=1" - ) - ) - - if !SKIP!==0 ( - echo Found service: !PROJECT_NAME! - set SERVICE_LIST=!SERVICE_LIST! %%F - ) else ( - echo Skipping service: !PROJECT_NAME! - ) -) - -echo: -echo ---------------------------------------- -echo Exporting schemas and packing subgraphs... -echo ---------------------------------------- - -for %%S in (%SERVICE_LIST%) do ( - echo Processing service folder: %%S - pushd "%%S" - - echo Running schema export... - dotnet run -- schema export --output schema.graphql - if errorlevel 1 ( - echo ERROR during schema export in %%S - popd - exit /b 1 - ) - - echo Running fusion subgraph pack... - fusion subgraph pack - if errorlevel 1 ( - echo ERROR during subgraph pack in %%S - popd - exit /b 1 - ) - - popd - echo Completed: %%S - echo. -) - -echo ---------------------------------------- -echo Running fusion compose... -echo ---------------------------------------- - -pushd "%ROOT%" - -if exist gateway.fgp del gateway.fgp - -for %%S in (%SERVICE_LIST%) do ( - REM Extract the full folder name WITH dots preserved - set "SERVICE_NAME=%%~nxS" - - echo Composing subgraph: !SERVICE_NAME! - - fusion compose -p gateway.fgp -s "..\!SERVICE_NAME!" - if errorlevel 1 ( - echo ERROR during fusion compose - popd - exit /b 1 - ) -) - -popd - - -echo ---------------------------------------- -echo Fusion build complete! -echo ---------------------------------------- - -endlocal -exit /b 0 diff --git a/FictionArchive.API/build_gateway.py b/FictionArchive.API/build_gateway.py new file mode 100644 index 0000000..cb6f68e --- /dev/null +++ b/FictionArchive.API/build_gateway.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import os +from pathlib import Path + +# ---------------------------------------- +# Helpers +# ---------------------------------------- + +def run(cmd, cwd=None): + """Run a command and exit on failure.""" + print(f"> {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd) + if result.returncode != 0: + print(f"ERROR: command failed in {cwd or os.getcwd()}") + sys.exit(result.returncode) + + +def load_skip_list(skip_file: Path): + if not skip_file.exists(): + print(f"WARNING: skip-projects.txt not found at {skip_file}") + return set() + + lines = skip_file.read_text().splitlines() + skip = {line.strip() for line in lines + if line.strip() and not line.strip().startswith("#")} + print("Skip list:", ", ".join(skip) if skip else "(none)") + return skip + + +# ---------------------------------------- +# Setup paths +# ---------------------------------------- + +script_dir = Path(__file__).parent.resolve() +services_dir = (script_dir / "..").resolve() +api_dir = services_dir / "FictionArchive.API" + +print(f"Script dir: {script_dir}") +print(f"Services dir: {services_dir}") + +skip_file = script_dir / "gateway_skip.txt" +skip_list = load_skip_list(skip_file) + +# ---------------------------------------- +# Find services +# ---------------------------------------- + +print("\n----------------------------------------") +print(" Finding GraphQL services...") +print("----------------------------------------") + +service_dirs = [ + d for d in services_dir.glob("FictionArchive.Service.*") + if d.is_dir() +] + +selected_services = [] + +for d in service_dirs: + name = d.name + if name in skip_list: + print(f"Skipping: {name}") + else: + print(f"Found: {name}") + selected_services.append(d) + +if not selected_services: + print("No services to process. Exiting.") + sys.exit(0) + +# ---------------------------------------- +# Export + pack +# ---------------------------------------- + +print("\n----------------------------------------") +print(" Exporting schemas & packing subgraphs...") +print("----------------------------------------") + +for svc in selected_services: + name = svc.name + print(f"\nProcessing {name}") + + # Build once + run(["dotnet", "build", "-c", "Release"], cwd=svc) + + # Export schema + run([ + "dotnet", "run", + "--no-build", + "--no-launch-profile", + "--", + "schema", "export", + "--output", "schema.graphql" + ], cwd=svc) + + # Pack subgraph + run(["fusion", "subgraph", "pack"], cwd=svc) + +# ---------------------------------------- +# Compose gateway +# ---------------------------------------- + +print("\n----------------------------------------") +print(" Running fusion compose...") +print("----------------------------------------") + +if not api_dir.exists(): + print(f"ERROR: FictionArchive.API not found at {api_dir}") + sys.exit(1) + +gateway_file = api_dir / "gateway.fgp" +if gateway_file.exists(): + gateway_file.unlink() + +for svc in selected_services: + name = svc.name + print(f"Composing: {name}") + + run([ + "fusion", "compose", + "-p", "gateway.fgp", + "-s", f"..{os.sep}{name}" + ], cwd=api_dir) + +print("\n----------------------------------------") +print(" Fusion build complete!") +print("----------------------------------------") diff --git a/FictionArchive.API/build_gateway.sh b/FictionArchive.API/build_gateway.sh deleted file mode 100644 index f6ed69a..0000000 --- a/FictionArchive.API/build_gateway.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -############################################### -# Resolve important directories -############################################### - -# Directory where this script lives -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Services live one directory above the script's directory -SERVICES_DIR="$(cd "$ROOT/.." && pwd)" - -############################################### -# Skip list (folder names, match exactly) -############################################### -SKIP_PROJECTS=( - "FictionArchive.Service.Shared" - "FictionArchive.Service.Legacy" -) - -echo "----------------------------------------" -echo " Finding GraphQL services..." -echo "----------------------------------------" - -SERVICE_LIST=() - -# Convert skip projects into a single searchable string -SKIP_STRING=" ${SKIP_PROJECTS[*]} " - -# Find service directories -shopt -s nullglob -for FOLDER in "$SERVICES_DIR"/FictionArchive.Service.*; do - [ -d "$FOLDER" ] || continue - - PROJECT_NAME="$(basename "$FOLDER")" - - # Skip entries that match the skip list - if [[ "$SKIP_STRING" == *" $PROJECT_NAME "* ]]; then - echo "Skipping service: $PROJECT_NAME" - continue - fi - - echo "Found service: $PROJECT_NAME" - SERVICE_LIST+=("$FOLDER") -done -shopt -u nullglob - -echo -echo "----------------------------------------" -echo " Exporting schemas and packing subgraphs..." -echo "----------------------------------------" - -for SERVICE in "${SERVICE_LIST[@]}"; do - PROJECT_NAME="$(basename "$SERVICE")" - - echo "Processing service: $PROJECT_NAME" - pushd "$SERVICE" >/dev/null - - echo "Building service..." - dotnet build -c Release >/dev/null - - # Automatically detect built DLL in bin/Release// - DLL_PATH="$(find "bin/Release" -maxdepth 3 -name '*.dll' | head -n 1)" - if [[ -z "$DLL_PATH" ]]; then - echo "ERROR: Could not locate DLL for $PROJECT_NAME" - popd >/dev/null - exit 1 - fi - - echo "Running schema export..." - dotnet exec "$DLL_PATH" schema export --output schema.graphql - - echo "Running subgraph pack..." - fusion subgraph pack - - popd >/dev/null - echo "Completed: $PROJECT_NAME" - echo -done - -echo "----------------------------------------" -echo " Running fusion compose..." -echo "----------------------------------------" - -pushd "$ROOT" >/dev/null - -# Remove old composition file -rm -f gateway.fgp - -for SERVICE in "${SERVICE_LIST[@]}"; do - SERVICE_NAME="$(basename "$SERVICE")" - - echo "Composing subgraph: $SERVICE_NAME" - - # Note: Fusion compose must reference parent dir (services live above ROOT) - fusion compose -p gateway.fgp -s "../$SERVICE_NAME" -done - -popd >/dev/null - -echo "----------------------------------------" -echo " Fusion build complete!" -echo "----------------------------------------" diff --git a/FictionArchive.API/gateway.fgp b/FictionArchive.API/gateway.fgp deleted file mode 100644 index 4eb467a..0000000 Binary files a/FictionArchive.API/gateway.fgp and /dev/null differ diff --git a/FictionArchive.API/gateway_skip.txt b/FictionArchive.API/gateway_skip.txt new file mode 100644 index 0000000..779d6de --- /dev/null +++ b/FictionArchive.API/gateway_skip.txt @@ -0,0 +1,4 @@ +# List of service folders to skip +FictionArchive.Service.Shared +FictionArchive.Service.AuthenticationService +FictionArchive.Service.FileService diff --git a/FictionArchive.Common/Enums/RequestStatus.cs b/FictionArchive.Common/Enums/RequestStatus.cs new file mode 100644 index 0000000..9b0fedd --- /dev/null +++ b/FictionArchive.Common/Enums/RequestStatus.cs @@ -0,0 +1,8 @@ +namespace FictionArchive.Common.Enums; + +public enum RequestStatus +{ + Failed = -1, + Pending = 0, + Success = 1 +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Controllers/S3ProxyController.cs b/FictionArchive.Service.FileService/Controllers/S3ProxyController.cs new file mode 100644 index 0000000..384095b --- /dev/null +++ b/FictionArchive.Service.FileService/Controllers/S3ProxyController.cs @@ -0,0 +1,49 @@ +using System.Web; +using Amazon.S3; +using Amazon.S3.Model; +using FictionArchive.Service.FileService.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +namespace FictionArchive.Service.FileService.Controllers +{ + [Route("api/{*path}")] + [ApiController] + public class S3ProxyController : ControllerBase + { + private readonly AmazonS3Client _amazonS3Client; + private readonly S3Configuration _s3Configuration; + + public S3ProxyController(AmazonS3Client amazonS3Client, IOptions s3Configuration) + { + _amazonS3Client = amazonS3Client; + _s3Configuration = s3Configuration.Value; + } + + [HttpGet] + public async Task Get(string path) + { + var decodedPath = HttpUtility.UrlDecode(path); + + try + { + var s3Response = await _amazonS3Client.GetObjectAsync(new GetObjectRequest() + { + BucketName = _s3Configuration.Bucket, + Key = decodedPath + }); + + return new FileStreamResult(s3Response.ResponseStream, s3Response.Headers.ContentType); + } + catch (AmazonS3Exception e) + { + if (e.Message == "Key not found") + { + return NotFound(); + } + throw; + } + } + } +} diff --git a/FictionArchive.Service.FileService/Dockerfile b/FictionArchive.Service.FileService/Dockerfile new file mode 100644 index 0000000..f972438 --- /dev/null +++ b/FictionArchive.Service.FileService/Dockerfile @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base +USER $APP_UID +WORKDIR /app +EXPOSE 8080 +EXPOSE 8081 + +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["FictionArchive.Service.ImageService/FictionArchive.Service.ImageService.csproj", "FictionArchive.Service.ImageService/"] +RUN dotnet restore "FictionArchive.Service.ImageService/FictionArchive.Service.ImageService.csproj" +COPY . . +WORKDIR "/src/FictionArchive.Service.ImageService" +RUN dotnet build "./FictionArchive.Service.ImageService.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./FictionArchive.Service.ImageService.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "FictionArchive.Service.ImageService.dll"] diff --git a/FictionArchive.Service.FileService/FictionArchive.Service.FileService.csproj b/FictionArchive.Service.FileService/FictionArchive.Service.FileService.csproj new file mode 100644 index 0000000..7e69402 --- /dev/null +++ b/FictionArchive.Service.FileService/FictionArchive.Service.FileService.csproj @@ -0,0 +1,30 @@ + + + + net8.0 + enable + enable + Linux + + + + + .dockerignore + + + + + + + + + + + + + + + + + + diff --git a/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs b/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs new file mode 100644 index 0000000..bf03a56 --- /dev/null +++ b/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs @@ -0,0 +1,10 @@ +using FictionArchive.Service.Shared.Services.EventBus; + +namespace FictionArchive.Service.FileService.Models.IntegrationEvents; + +public class FileUploadRequestCreatedEvent : IIntegrationEvent +{ + public Guid RequestId { get; set; } + public string FilePath { get; set; } + public byte[] FileData { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs b/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs new file mode 100644 index 0000000..b5936de --- /dev/null +++ b/FictionArchive.Service.FileService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs @@ -0,0 +1,22 @@ +using FictionArchive.Common.Enums; +using FictionArchive.Service.Shared.Services.EventBus; + +namespace FictionArchive.Service.FileService.Models.IntegrationEvents; + +public class FileUploadRequestStatusUpdateEvent : IIntegrationEvent +{ + public Guid RequestId { get; set; } + public RequestStatus Status { get; set; } + + #region Success + + public string? FileAccessUrl { get; set; } + + #endregion + + #region Failure + + public string? ErrorMessage { get; set; } + + #endregion +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Models/ProxyConfiguration.cs b/FictionArchive.Service.FileService/Models/ProxyConfiguration.cs new file mode 100644 index 0000000..c1ce02c --- /dev/null +++ b/FictionArchive.Service.FileService/Models/ProxyConfiguration.cs @@ -0,0 +1,6 @@ +namespace FictionArchive.Service.FileService.Models; + +public class ProxyConfiguration +{ + public string BaseUrl { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Models/S3Configuration.cs b/FictionArchive.Service.FileService/Models/S3Configuration.cs new file mode 100644 index 0000000..b5c601e --- /dev/null +++ b/FictionArchive.Service.FileService/Models/S3Configuration.cs @@ -0,0 +1,9 @@ +namespace FictionArchive.Service.FileService.Models; + +public class S3Configuration +{ + public string Url { get; set; } + public string Bucket { get; set; } + public string AccessKey { get; set; } + public string SecretKey { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Program.cs b/FictionArchive.Service.FileService/Program.cs new file mode 100644 index 0000000..3decc95 --- /dev/null +++ b/FictionArchive.Service.FileService/Program.cs @@ -0,0 +1,69 @@ +using Amazon.Runtime; +using Amazon.S3; +using FictionArchive.Common.Extensions; +using FictionArchive.Service.FileService.Models; +using FictionArchive.Service.FileService.Models.IntegrationEvents; +using FictionArchive.Service.FileService.Services.EventHandlers; +using FictionArchive.Service.Shared.Extensions; +using FictionArchive.Service.Shared.Services.EventBus.Implementations; +using Microsoft.Extensions.Options; + +namespace FictionArchive.Service.FileService; + +public class Program +{ + public static void Main(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + builder.AddLocalAppsettings(); + + builder.Services.AddControllers(); + // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle + builder.Services.AddEndpointsApiExplorer(); + builder.Services.AddSwaggerGen(); + + builder.Services.AddHealthChecks(); + + #region Event Bus + + builder.Services.AddRabbitMQ(opt => + { + builder.Configuration.GetSection("RabbitMQ").Bind(opt); + }) + .Subscribe(); + + #endregion + + builder.Services.Configure(builder.Configuration.GetSection("ProxyConfiguration")); + + // Add S3 Client + builder.Services.Configure(builder.Configuration.GetSection("S3")); + builder.Services.AddSingleton(provider => + { + var config = provider.GetRequiredService>().Value; + var s3Config = new AmazonS3Config + { + ServiceURL = config.Url, // Garage endpoint + ForcePathStyle = true, // REQUIRED for Garage + AuthenticationRegion = "garage" + }; + return new AmazonS3Client( + new BasicAWSCredentials(config.AccessKey, config.SecretKey), + s3Config); + }); + + var app = builder.Build(); + + if (app.Environment.IsDevelopment()) + { + app.UseSwagger(); + app.UseSwaggerUI(); + } + + app.MapHealthChecks("/healthz"); + + app.MapControllers(); + + app.Run(); + } +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/Properties/launchSettings.json b/FictionArchive.Service.FileService/Properties/launchSettings.json new file mode 100644 index 0000000..60a6c18 --- /dev/null +++ b/FictionArchive.Service.FileService/Properties/launchSettings.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:5546", + "sslPort": 44373 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5057", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7247;http://localhost:5057", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/FictionArchive.Service.FileService/Services/EventHandlers/FileUploadRequestCreatedEventHandler.cs b/FictionArchive.Service.FileService/Services/EventHandlers/FileUploadRequestCreatedEventHandler.cs new file mode 100644 index 0000000..0383030 --- /dev/null +++ b/FictionArchive.Service.FileService/Services/EventHandlers/FileUploadRequestCreatedEventHandler.cs @@ -0,0 +1,58 @@ +using Amazon.S3; +using Amazon.S3.Model; +using FictionArchive.Common.Enums; +using FictionArchive.Service.FileService.Models; +using FictionArchive.Service.FileService.Models.IntegrationEvents; +using FictionArchive.Service.Shared.Services.EventBus; +using Microsoft.Extensions.Options; + +namespace FictionArchive.Service.FileService.Services.EventHandlers; + +public class FileUploadRequestCreatedEventHandler : IIntegrationEventHandler +{ + private readonly ILogger _logger; + private readonly AmazonS3Client _amazonS3Client; + private readonly IEventBus _eventBus; + private readonly S3Configuration _s3Configuration; + private readonly ProxyConfiguration _proxyConfiguration; + + public FileUploadRequestCreatedEventHandler(ILogger logger, AmazonS3Client amazonS3Client, IEventBus eventBus, IOptions s3Configuration, IOptions proxyConfiguration) + { + _logger = logger; + _amazonS3Client = amazonS3Client; + _eventBus = eventBus; + _proxyConfiguration = proxyConfiguration.Value; + _s3Configuration = s3Configuration.Value; + } + + public async Task Handle(FileUploadRequestCreatedEvent @event) + { + var putObjectRequest = new PutObjectRequest(); + putObjectRequest.BucketName = _s3Configuration.Bucket; + putObjectRequest.Key = @event.FilePath; + putObjectRequest.UseChunkEncoding = false; // Needed to avoid an error with Garage + + using MemoryStream memoryStream = new MemoryStream(@event.FileData); + putObjectRequest.InputStream = memoryStream; + + var s3Response = await _amazonS3Client.PutObjectAsync(putObjectRequest); + if (s3Response.HttpStatusCode != System.Net.HttpStatusCode.OK) + { + _logger.LogError("An error occurred while uploading file to S3. Response code: {responsecode}", s3Response.HttpStatusCode); + await _eventBus.Publish(new FileUploadRequestStatusUpdateEvent() + { + RequestId = @event.RequestId, + Status = RequestStatus.Failed, + ErrorMessage = "An error occurred while uploading file to S3." + }); + return; + } + + await _eventBus.Publish(new FileUploadRequestStatusUpdateEvent() + { + Status = RequestStatus.Success, + RequestId = @event.RequestId, + FileAccessUrl = _proxyConfiguration.BaseUrl + "/" + @event.FilePath + }); + } +} \ No newline at end of file diff --git a/FictionArchive.Service.FileService/appsettings.Development.json b/FictionArchive.Service.FileService/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/FictionArchive.Service.FileService/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/FictionArchive.Service.FileService/appsettings.json b/FictionArchive.Service.FileService/appsettings.json new file mode 100644 index 0000000..c18d35f --- /dev/null +++ b/FictionArchive.Service.FileService/appsettings.json @@ -0,0 +1,22 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "ProxyConfiguration": { + "BaseUrl": "https://localhost:7247/api" + }, + "RabbitMQ": { + "ConnectionString": "amqp://localhost", + "ClientIdentifier": "FileService" + }, + "S3": { + "Url": "https://s3.orfl.xyz", + "Bucket": "fictionarchive", + "AccessKey": "REPLACE_ME", + "SecretKey": "REPLACE_ME" + }, + "AllowedHosts": "*" +} diff --git a/FictionArchive.Service.NovelService.Tests/FictionArchive.Service.NovelService.Tests.csproj b/FictionArchive.Service.NovelService.Tests/FictionArchive.Service.NovelService.Tests.csproj new file mode 100644 index 0000000..24e3242 --- /dev/null +++ b/FictionArchive.Service.NovelService.Tests/FictionArchive.Service.NovelService.Tests.csproj @@ -0,0 +1,30 @@ + + + + net8.0 + enable + enable + false + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/FictionArchive.Service.NovelService.Tests/NovelUpdateServiceTests.cs b/FictionArchive.Service.NovelService.Tests/NovelUpdateServiceTests.cs new file mode 100644 index 0000000..b82835b --- /dev/null +++ b/FictionArchive.Service.NovelService.Tests/NovelUpdateServiceTests.cs @@ -0,0 +1,165 @@ +using FictionArchive.Common.Enums; +using FictionArchive.Service.FileService.IntegrationEvents; +using FictionArchive.Service.NovelService.Models.Configuration; +using FictionArchive.Service.NovelService.Models.Enums; +using FictionArchive.Service.NovelService.Models.Images; +using FictionArchive.Service.NovelService.Models.Localization; +using FictionArchive.Service.NovelService.Models.Novels; +using FictionArchive.Service.NovelService.Models.SourceAdapters; +using FictionArchive.Service.NovelService.Services; +using FictionArchive.Service.NovelService.Services.SourceAdapters; +using FictionArchive.Service.Shared.Services.EventBus; +using FluentAssertions; +using HtmlAgilityPack; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace FictionArchive.Service.NovelService.Tests; + +public class NovelUpdateServiceTests +{ + private static NovelServiceDbContext CreateDbContext() + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase($"NovelUpdateServiceTests-{Guid.NewGuid()}") + .Options; + + return new NovelServiceDbContext(options, NullLogger.Instance); + } + + private static NovelCreateResult CreateNovelWithSingleChapter(NovelServiceDbContext dbContext, Source source) + { + var chapter = new Chapter + { + Order = 1, + Revision = 1, + Url = "http://demo/chapter-1", + Name = LocalizationKey.CreateFromText("Chapter 1", Language.En), + Body = new LocalizationKey { Texts = new List() }, + Images = new List() + }; + + var novel = new Novel + { + Url = "http://demo/novel", + ExternalId = "demo-1", + Author = new Person { Name = LocalizationKey.CreateFromText("Author", Language.En) }, + RawLanguage = Language.En, + RawStatus = NovelStatus.InProgress, + Source = source, + Name = LocalizationKey.CreateFromText("Demo Novel", Language.En), + Description = LocalizationKey.CreateFromText("Description", Language.En), + Chapters = new List { chapter }, + Tags = new List() + }; + + dbContext.Novels.Add(novel); + dbContext.SaveChanges(); + + return new NovelCreateResult(novel, chapter); + } + + private static NovelUpdateService CreateService( + NovelServiceDbContext dbContext, + ISourceAdapter adapter, + IEventBus eventBus, + string pendingImageUrl = "https://pending/placeholder.jpg") + { + var options = Options.Create(new NovelUpdateServiceConfiguration + { + PendingImageUrl = pendingImageUrl + }); + + return new NovelUpdateService(dbContext, NullLogger.Instance, new[] { adapter }, eventBus, options); + } + + [Fact] + public async Task PullChapterContents_rewrites_images_and_publishes_requests() + { + using var dbContext = CreateDbContext(); + var source = new Source { Name = "Demo", Key = "demo", Url = "http://demo" }; + var (novel, chapter) = CreateNovelWithSingleChapter(dbContext, source); + + var rawHtml = "

Hello

\"first\"\"second\""; + var image1 = new ImageData { Url = "http://img/x1.jpg", Data = new byte[] { 1, 2, 3 } }; + var image2 = new ImageData { Url = "http://img/x2.jpg", Data = new byte[] { 4, 5, 6 } }; + + var adapter = Substitute.For(); + adapter.SourceDescriptor.Returns(new SourceDescriptor { Key = "demo", Name = "Demo", Url = "http://demo" }); + adapter.GetRawChapter(chapter.Url).Returns(Task.FromResult(new ChapterFetchResult + { + Text = rawHtml, + ImageData = new List { image1, image2 } + })); + + var publishedEvents = new List(); + var eventBus = Substitute.For(); + eventBus.Publish(Arg.Do(publishedEvents.Add)).Returns(Task.CompletedTask); + eventBus.Publish(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + + var pendingImageUrl = "https://pending/placeholder.jpg"; + var service = CreateService(dbContext, adapter, eventBus, pendingImageUrl); + + var updatedChapter = await service.PullChapterContents(novel.Id, chapter.Order); + + updatedChapter.Images.Should().HaveCount(2); + updatedChapter.Images.Select(i => i.OriginalPath).Should().BeEquivalentTo(new[] { image1.Url, image2.Url }); + updatedChapter.Images.All(i => i.Id != Guid.Empty).Should().BeTrue(); + + var storedHtml = updatedChapter.Body.Texts.Single().Text; + var doc = new HtmlDocument(); + doc.LoadHtml(storedHtml); + var imgNodes = doc.DocumentNode.SelectNodes("//img"); + imgNodes.Should().NotBeNull(); + imgNodes!.Count.Should().Be(2); + imgNodes.Should().OnlyContain(node => node.GetAttributeValue("src", string.Empty) == pendingImageUrl); + imgNodes.Select(node => node.GetAttributeValue("alt", string.Empty)) + .Should() + .BeEquivalentTo(updatedChapter.Images.Select(img => img.Id.ToString())); + + publishedEvents.Should().HaveCount(2); + publishedEvents.Select(e => e.RequestId).Should().BeEquivalentTo(updatedChapter.Images.Select(i => i.Id)); + publishedEvents.Select(e => e.FileData).Should().BeEquivalentTo(new[] { image1.Data, image2.Data }); + publishedEvents.Should().OnlyContain(e => e.FilePath.StartsWith($"{novel.Id}/Images/Chapter-{updatedChapter.Id}/")); + } + + [Fact] + public async Task PullChapterContents_adds_alt_when_missing() + { + using var dbContext = CreateDbContext(); + var source = new Source { Name = "Demo", Key = "demo", Url = "http://demo" }; + var (novel, chapter) = CreateNovelWithSingleChapter(dbContext, source); + + var rawHtml = "

Hi

"; + var image = new ImageData { Url = "http://img/x1.jpg", Data = new byte[] { 7, 8, 9 } }; + + var adapter = Substitute.For(); + adapter.SourceDescriptor.Returns(new SourceDescriptor { Key = "demo", Name = "Demo", Url = "http://demo" }); + adapter.GetRawChapter(chapter.Url).Returns(Task.FromResult(new ChapterFetchResult + { + Text = rawHtml, + ImageData = new List { image } + })); + + var eventBus = Substitute.For(); + eventBus.Publish(Arg.Any()).Returns(Task.CompletedTask); + eventBus.Publish(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + + var service = CreateService(dbContext, adapter, eventBus); + + var updatedChapter = await service.PullChapterContents(novel.Id, chapter.Order); + + var storedHtml = updatedChapter.Body.Texts.Single().Text; + var doc = new HtmlDocument(); + doc.LoadHtml(storedHtml); + var imgNode = doc.DocumentNode.SelectSingleNode("//img"); + imgNode.Should().NotBeNull(); + imgNode!.GetAttributeValue("alt", string.Empty).Should().Be(updatedChapter.Images.Single().Id.ToString()); + imgNode.GetAttributeValue("src", string.Empty).Should().Be("https://pending/placeholder.jpg"); + } + + private record NovelCreateResult(Novel Novel, Chapter Chapter); +} diff --git a/FictionArchive.Service.NovelService/FictionArchive.Service.NovelService.csproj b/FictionArchive.Service.NovelService/FictionArchive.Service.NovelService.csproj index ac56df2..1d90435 100644 --- a/FictionArchive.Service.NovelService/FictionArchive.Service.NovelService.csproj +++ b/FictionArchive.Service.NovelService/FictionArchive.Service.NovelService.csproj @@ -9,6 +9,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.Designer.cs b/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.Designer.cs new file mode 100644 index 0000000..584b138 --- /dev/null +++ b/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.Designer.cs @@ -0,0 +1,540 @@ +// +using System; +using FictionArchive.Service.NovelService.Services; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FictionArchive.Service.NovelService.Migrations +{ + [DbContext(typeof(NovelServiceDbContext))] + [Migration("20251123203953_AddImages")] + partial class AddImages + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChapterId") + .HasColumnType("bigint"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("NewPath") + .HasColumnType("text"); + + b.Property("OriginalPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ChapterId"); + + b.ToTable("Images"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("LocalizationKeys"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("EngineId") + .HasColumnType("bigint"); + + b.Property("KeyRequestedForTranslationId") + .HasColumnType("uuid"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("TranslateTo") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("EngineId"); + + b.HasIndex("KeyRequestedForTranslationId"); + + b.ToTable("LocalizationRequests"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Language") + .HasColumnType("integer"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("LocalizationKeyId") + .HasColumnType("uuid"); + + b.Property("Text") + .IsRequired() + .HasColumnType("text"); + + b.Property("TranslationEngineId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("LocalizationKeyId"); + + b.HasIndex("TranslationEngineId"); + + b.ToTable("LocalizationText"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BodyId") + .HasColumnType("uuid"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("NameId") + .HasColumnType("uuid"); + + b.Property("NovelId") + .HasColumnType("bigint"); + + b.Property("Order") + .HasColumnType("bigint"); + + b.Property("Revision") + .HasColumnType("bigint"); + + b.Property("Url") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("BodyId"); + + b.HasIndex("NameId"); + + b.HasIndex("NovelId"); + + b.ToTable("Chapter"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AuthorId") + .HasColumnType("bigint"); + + b.Property("CoverImageId") + .HasColumnType("uuid"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DescriptionId") + .HasColumnType("uuid"); + + b.Property("ExternalId") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("NameId") + .HasColumnType("uuid"); + + b.Property("RawLanguage") + .HasColumnType("integer"); + + b.Property("RawStatus") + .HasColumnType("integer"); + + b.Property("SourceId") + .HasColumnType("bigint"); + + b.Property("StatusOverride") + .HasColumnType("integer"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AuthorId"); + + b.HasIndex("CoverImageId"); + + b.HasIndex("DescriptionId"); + + b.HasIndex("NameId"); + + b.HasIndex("SourceId"); + + b.ToTable("Novels"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DisplayNameId") + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceId") + .HasColumnType("bigint"); + + b.Property("TagType") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("DisplayNameId"); + + b.HasIndex("SourceId"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ExternalUrl") + .HasColumnType("text"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("NameId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("NameId"); + + b.ToTable("Person"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Source", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Sources"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("TranslationEngines"); + }); + + modelBuilder.Entity("NovelNovelTag", b => + { + b.Property("NovelsId") + .HasColumnType("bigint"); + + b.Property("TagsId") + .HasColumnType("bigint"); + + b.HasKey("NovelsId", "TagsId"); + + b.HasIndex("TagsId"); + + b.ToTable("NovelNovelTag"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Chapter", "Chapter") + .WithMany("Images") + .HasForeignKey("ChapterId"); + + b.Navigation("Chapter"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "Engine") + .WithMany() + .HasForeignKey("EngineId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "KeyRequestedForTranslation") + .WithMany() + .HasForeignKey("KeyRequestedForTranslationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Engine"); + + b.Navigation("KeyRequestedForTranslation"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", null) + .WithMany("Texts") + .HasForeignKey("LocalizationKeyId"); + + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "TranslationEngine") + .WithMany() + .HasForeignKey("TranslationEngineId"); + + b.Navigation("TranslationEngine"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Body") + .WithMany() + .HasForeignKey("BodyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name") + .WithMany() + .HasForeignKey("NameId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", null) + .WithMany("Chapters") + .HasForeignKey("NovelId"); + + b.Navigation("Body"); + + b.Navigation("Name"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Person", "Author") + .WithMany() + .HasForeignKey("AuthorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Images.Image", "CoverImage") + .WithMany() + .HasForeignKey("CoverImageId"); + + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Description") + .WithMany() + .HasForeignKey("DescriptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name") + .WithMany() + .HasForeignKey("NameId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Author"); + + b.Navigation("CoverImage"); + + b.Navigation("Description"); + + b.Navigation("Name"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "DisplayName") + .WithMany() + .HasForeignKey("DisplayNameId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("DisplayName"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name") + .WithMany() + .HasForeignKey("NameId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Name"); + }); + + modelBuilder.Entity("NovelNovelTag", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", null) + .WithMany() + .HasForeignKey("NovelsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.NovelTag", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b => + { + b.Navigation("Texts"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b => + { + b.Navigation("Images"); + }); + + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b => + { + b.Navigation("Chapters"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.cs b/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.cs new file mode 100644 index 0000000..8c9f8a6 --- /dev/null +++ b/FictionArchive.Service.NovelService/Migrations/20251123203953_AddImages.cs @@ -0,0 +1,79 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace FictionArchive.Service.NovelService.Migrations +{ + /// + public partial class AddImages : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "CoverImageId", + table: "Novels", + type: "uuid", + nullable: true); + + migrationBuilder.CreateTable( + name: "Images", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + OriginalPath = table.Column(type: "text", nullable: false), + NewPath = table.Column(type: "text", nullable: true), + ChapterId = table.Column(type: "bigint", nullable: true), + CreatedTime = table.Column(type: "timestamp with time zone", nullable: false), + LastUpdatedTime = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Images", x => x.Id); + table.ForeignKey( + name: "FK_Images_Chapter_ChapterId", + column: x => x.ChapterId, + principalTable: "Chapter", + principalColumn: "Id"); + }); + + migrationBuilder.CreateIndex( + name: "IX_Novels_CoverImageId", + table: "Novels", + column: "CoverImageId"); + + migrationBuilder.CreateIndex( + name: "IX_Images_ChapterId", + table: "Images", + column: "ChapterId"); + + migrationBuilder.AddForeignKey( + name: "FK_Novels_Images_CoverImageId", + table: "Novels", + column: "CoverImageId", + principalTable: "Images", + principalColumn: "Id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Novels_Images_CoverImageId", + table: "Novels"); + + migrationBuilder.DropTable( + name: "Images"); + + migrationBuilder.DropIndex( + name: "IX_Novels_CoverImageId", + table: "Novels"); + + migrationBuilder.DropColumn( + name: "CoverImageId", + table: "Novels"); + } + } +} diff --git a/FictionArchive.Service.NovelService/Migrations/NovelServiceDbContextModelSnapshot.cs b/FictionArchive.Service.NovelService/Migrations/NovelServiceDbContextModelSnapshot.cs index ed31991..470f4a9 100644 --- a/FictionArchive.Service.NovelService/Migrations/NovelServiceDbContextModelSnapshot.cs +++ b/FictionArchive.Service.NovelService/Migrations/NovelServiceDbContextModelSnapshot.cs @@ -23,6 +23,35 @@ namespace FictionArchive.Service.NovelService.Migrations NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChapterId") + .HasColumnType("bigint"); + + b.Property("CreatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("LastUpdatedTime") + .HasColumnType("timestamp with time zone"); + + b.Property("NewPath") + .HasColumnType("text"); + + b.Property("OriginalPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ChapterId"); + + b.ToTable("Images"); + }); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b => { b.Property("Id") @@ -158,6 +187,9 @@ namespace FictionArchive.Service.NovelService.Migrations b.Property("AuthorId") .HasColumnType("bigint"); + b.Property("CoverImageId") + .HasColumnType("uuid"); + b.Property("CreatedTime") .HasColumnType("timestamp with time zone"); @@ -194,6 +226,8 @@ namespace FictionArchive.Service.NovelService.Migrations b.HasIndex("AuthorId"); + b.HasIndex("CoverImageId"); + b.HasIndex("DescriptionId"); b.HasIndex("NameId"); @@ -335,6 +369,15 @@ namespace FictionArchive.Service.NovelService.Migrations b.ToTable("NovelNovelTag"); }); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b => + { + b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Chapter", "Chapter") + .WithMany("Images") + .HasForeignKey("ChapterId"); + + b.Navigation("Chapter"); + }); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b => { b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "Engine") @@ -398,6 +441,10 @@ namespace FictionArchive.Service.NovelService.Migrations .OnDelete(DeleteBehavior.Cascade) .IsRequired(); + b.HasOne("FictionArchive.Service.NovelService.Models.Images.Image", "CoverImage") + .WithMany() + .HasForeignKey("CoverImageId"); + b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Description") .WithMany() .HasForeignKey("DescriptionId") @@ -418,6 +465,8 @@ namespace FictionArchive.Service.NovelService.Migrations b.Navigation("Author"); + b.Navigation("CoverImage"); + b.Navigation("Description"); b.Navigation("Name"); @@ -473,6 +522,11 @@ namespace FictionArchive.Service.NovelService.Migrations b.Navigation("Texts"); }); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b => + { + b.Navigation("Images"); + }); + modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b => { b.Navigation("Chapters"); diff --git a/FictionArchive.Service.NovelService/Models/Configuration/NovelUpdateServiceConfiguration.cs b/FictionArchive.Service.NovelService/Models/Configuration/NovelUpdateServiceConfiguration.cs new file mode 100644 index 0000000..e3d5e96 --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/Configuration/NovelUpdateServiceConfiguration.cs @@ -0,0 +1,6 @@ +namespace FictionArchive.Service.NovelService.Models.Configuration; + +public class NovelUpdateServiceConfiguration +{ + public string PendingImageUrl { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/Images/Image.cs b/FictionArchive.Service.NovelService/Models/Images/Image.cs new file mode 100644 index 0000000..c7141ee --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/Images/Image.cs @@ -0,0 +1,13 @@ +using FictionArchive.Service.NovelService.Models.Novels; +using FictionArchive.Service.Shared.Models; + +namespace FictionArchive.Service.NovelService.Models.Images; + +public class Image : BaseEntity +{ + public string OriginalPath { get; set; } + public string? NewPath { get; set; } + + // Chapter link. Even if an image appears in another chapter, we should rehost it separately. + public Chapter? Chapter { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs b/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs new file mode 100644 index 0000000..44e3f46 --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestCreatedEvent.cs @@ -0,0 +1,10 @@ +using FictionArchive.Service.Shared.Services.EventBus; + +namespace FictionArchive.Service.FileService.IntegrationEvents; + +public class FileUploadRequestCreatedEvent : IIntegrationEvent +{ + public Guid RequestId { get; set; } + public string FilePath { get; set; } + public byte[] FileData { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs b/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs new file mode 100644 index 0000000..4f61613 --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/IntegrationEvents/FileUploadRequestStatusUpdateEvent.cs @@ -0,0 +1,22 @@ +using FictionArchive.Common.Enums; +using FictionArchive.Service.Shared.Services.EventBus; + +namespace FictionArchive.Service.NovelService.Models.IntegrationEvents; + +public class FileUploadRequestStatusUpdateEvent : IIntegrationEvent +{ + public Guid RequestId { get; set; } + public RequestStatus Status { get; set; } + + #region Success + + public string? FileAccessUrl { get; set; } + + #endregion + + #region Failure + + public string? ErrorMessage { get; set; } + + #endregion +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/Novels/Chapter.cs b/FictionArchive.Service.NovelService/Models/Novels/Chapter.cs index e59705d..05c8011 100644 --- a/FictionArchive.Service.NovelService/Models/Novels/Chapter.cs +++ b/FictionArchive.Service.NovelService/Models/Novels/Chapter.cs @@ -1,3 +1,4 @@ +using FictionArchive.Service.NovelService.Models.Images; using FictionArchive.Service.NovelService.Models.Localization; using FictionArchive.Service.Shared.Models; @@ -11,4 +12,7 @@ public class Chapter : BaseEntity public LocalizationKey Name { get; set; } public LocalizationKey Body { get; set; } + + // Images appearing in this chapter. + public List Images { get; set; } } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/Novels/Novel.cs b/FictionArchive.Service.NovelService/Models/Novels/Novel.cs index 81d5ca7..582cd96 100644 --- a/FictionArchive.Service.NovelService/Models/Novels/Novel.cs +++ b/FictionArchive.Service.NovelService/Models/Novels/Novel.cs @@ -1,4 +1,5 @@ using FictionArchive.Common.Enums; +using FictionArchive.Service.NovelService.Models.Images; using FictionArchive.Service.NovelService.Models.Localization; using FictionArchive.Service.Shared.Models; using NovelStatus = FictionArchive.Service.NovelService.Models.Enums.NovelStatus; @@ -22,4 +23,5 @@ public class Novel : BaseEntity public List Chapters { get; set; } public List Tags { get; set; } + public Image? CoverImage { get; set; } } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterFetchResult.cs b/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterFetchResult.cs new file mode 100644 index 0000000..ca6805d --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterFetchResult.cs @@ -0,0 +1,7 @@ +namespace FictionArchive.Service.NovelService.Models.SourceAdapters; + +public class ChapterFetchResult +{ + public string Text { get; set; } + public List ImageData { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterMetadata.cs b/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterMetadata.cs index 353c9bd..d6be4df 100644 --- a/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterMetadata.cs +++ b/FictionArchive.Service.NovelService/Models/SourceAdapters/ChapterMetadata.cs @@ -6,5 +6,5 @@ public class ChapterMetadata public uint Order { get; set; } public string? Url { get; set; } public string Name { get; set; } - + public List ImageUrls { get; set; } } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/SourceAdapters/ImageData.cs b/FictionArchive.Service.NovelService/Models/SourceAdapters/ImageData.cs new file mode 100644 index 0000000..52838cb --- /dev/null +++ b/FictionArchive.Service.NovelService/Models/SourceAdapters/ImageData.cs @@ -0,0 +1,7 @@ +namespace FictionArchive.Service.NovelService.Models.SourceAdapters; + +public class ImageData +{ + public string Url { get; set; } + public byte[] Data { get; set; } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Models/SourceAdapters/NovelMetadata.cs b/FictionArchive.Service.NovelService/Models/SourceAdapters/NovelMetadata.cs index af1467a..716dcbe 100644 --- a/FictionArchive.Service.NovelService/Models/SourceAdapters/NovelMetadata.cs +++ b/FictionArchive.Service.NovelService/Models/SourceAdapters/NovelMetadata.cs @@ -11,6 +11,7 @@ public class NovelMetadata public string AuthorUrl { get; set; } public string Url { get; set; } public string ExternalId { get; set; } + public ImageData? CoverImage { get; set; } public Language RawLanguage { get; set; } public NovelStatus RawStatus { get; set; } diff --git a/FictionArchive.Service.NovelService/Program.cs b/FictionArchive.Service.NovelService/Program.cs index 233118f..68fe593 100644 --- a/FictionArchive.Service.NovelService/Program.cs +++ b/FictionArchive.Service.NovelService/Program.cs @@ -1,4 +1,6 @@ +using FictionArchive.Common.Extensions; using FictionArchive.Service.NovelService.GraphQL; +using FictionArchive.Service.NovelService.Models.Configuration; using FictionArchive.Service.NovelService.Models.IntegrationEvents; using FictionArchive.Service.NovelService.Services; using FictionArchive.Service.NovelService.Services.EventHandlers; @@ -16,6 +18,7 @@ public class Program public static void Main(string[] args) { var builder = WebApplication.CreateBuilder(args); + builder.AddLocalAppsettings(); builder.Services.AddMemoryCache(); @@ -27,7 +30,8 @@ public class Program }) .Subscribe() .Subscribe() - .Subscribe(); + .Subscribe() + .Subscribe(); #endregion @@ -56,6 +60,7 @@ public class Program }) .AddHttpMessageHandler(); + builder.Services.Configure(builder.Configuration.GetSection("UpdateService")); builder.Services.AddTransient(); #endregion diff --git a/FictionArchive.Service.NovelService/Services/EventHandlers/FileUploadRequestStatusUpdateEventHandler.cs b/FictionArchive.Service.NovelService/Services/EventHandlers/FileUploadRequestStatusUpdateEventHandler.cs new file mode 100644 index 0000000..f6b3dc3 --- /dev/null +++ b/FictionArchive.Service.NovelService/Services/EventHandlers/FileUploadRequestStatusUpdateEventHandler.cs @@ -0,0 +1,39 @@ +using FictionArchive.Common.Enums; +using FictionArchive.Service.NovelService.Models.IntegrationEvents; +using FictionArchive.Service.Shared.Services.EventBus; + +namespace FictionArchive.Service.NovelService.Services.EventHandlers; + +public class FileUploadRequestStatusUpdateEventHandler : IIntegrationEventHandler +{ + private readonly ILogger _logger; + private readonly NovelServiceDbContext _context; + private readonly NovelUpdateService _novelUpdateService; + + public FileUploadRequestStatusUpdateEventHandler(ILogger logger, NovelServiceDbContext context, NovelUpdateService novelUpdateService) + { + _logger = logger; + _context = context; + _novelUpdateService = novelUpdateService; + } + + public async Task Handle(FileUploadRequestStatusUpdateEvent @event) + { + var image = await _context.Images.FindAsync(@event.RequestId); + if (image == null) + { + // Not a request we care about. + return; + } + if (@event.Status == RequestStatus.Failed) + { + _logger.LogError("Image upload failed for image with id {imageId}", image.Id); + return; + } + else if (@event.Status == RequestStatus.Success) + { + _logger.LogInformation("Image upload succeeded for image with id {imageId}", image.Id); + await _novelUpdateService.UpdateImage(image.Id, @event.FileAccessUrl); + } + } +} \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Services/NovelServiceDbContext.cs b/FictionArchive.Service.NovelService/Services/NovelServiceDbContext.cs index a6d0204..6bcb7c7 100644 --- a/FictionArchive.Service.NovelService/Services/NovelServiceDbContext.cs +++ b/FictionArchive.Service.NovelService/Services/NovelServiceDbContext.cs @@ -1,3 +1,4 @@ +using FictionArchive.Service.NovelService.Models.Images; using FictionArchive.Service.NovelService.Models.Localization; using FictionArchive.Service.NovelService.Models.Novels; using FictionArchive.Service.Shared.Services.Database; @@ -14,4 +15,5 @@ public class NovelServiceDbContext(DbContextOptions options, ILogger Tags { get; set; } public DbSet LocalizationKeys { get; set; } public DbSet LocalizationRequests { get; set; } + public DbSet Images { get; set; } } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Services/NovelUpdateService.cs b/FictionArchive.Service.NovelService/Services/NovelUpdateService.cs index cf3669e..de3438b 100644 --- a/FictionArchive.Service.NovelService/Services/NovelUpdateService.cs +++ b/FictionArchive.Service.NovelService/Services/NovelUpdateService.cs @@ -1,9 +1,15 @@ +using FictionArchive.Service.FileService.IntegrationEvents; +using FictionArchive.Service.NovelService.Models.Configuration; using FictionArchive.Service.NovelService.Models.Enums; +using FictionArchive.Service.NovelService.Models.Images; using FictionArchive.Service.NovelService.Models.Localization; using FictionArchive.Service.NovelService.Models.Novels; using FictionArchive.Service.NovelService.Models.SourceAdapters; using FictionArchive.Service.NovelService.Services.SourceAdapters; +using FictionArchive.Service.Shared.Services.EventBus; +using HtmlAgilityPack; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; namespace FictionArchive.Service.NovelService.Services; @@ -12,12 +18,16 @@ public class NovelUpdateService private readonly NovelServiceDbContext _dbContext; private readonly ILogger _logger; private readonly IEnumerable _sourceAdapters; + private readonly IEventBus _eventBus; + private readonly NovelUpdateServiceConfiguration _novelUpdateServiceConfiguration; - public NovelUpdateService(NovelServiceDbContext dbContext, ILogger logger, IEnumerable sourceAdapters) + public NovelUpdateService(NovelServiceDbContext dbContext, ILogger logger, IEnumerable sourceAdapters, IEventBus eventBus, IOptions novelUpdateServiceConfiguration) { _dbContext = dbContext; _logger = logger; _sourceAdapters = sourceAdapters; + _eventBus = eventBus; + _novelUpdateServiceConfiguration = novelUpdateServiceConfiguration.Value; } public async Task ImportNovel(string novelUrl) @@ -59,6 +69,10 @@ public class NovelUpdateService RawLanguage = metadata.RawLanguage, Url = metadata.Url, ExternalId = metadata.ExternalId, + CoverImage = metadata.CoverImage != null ? new Image() + { + OriginalPath = metadata.CoverImage.Url, + } : null, Chapters = metadata.Chapters.Select(chapter => { return new Chapter() @@ -85,7 +99,18 @@ public class NovelUpdateService } }); await _dbContext.SaveChangesAsync(); - + + // Signal request for cover image if present + if (addedNovel.Entity.CoverImage != null) + { + await _eventBus.Publish(new FileUploadRequestCreatedEvent() + { + RequestId = addedNovel.Entity.CoverImage.Id, + FileData = metadata.CoverImage.Data, + FilePath = $"Novels/{addedNovel.Entity.Id}/Images/cover.jpg" + }); + } + return addedNovel.Entity; } @@ -95,17 +120,85 @@ public class NovelUpdateService .Include(novel => novel.Chapters) .ThenInclude(chapter => chapter.Body) .ThenInclude(body => body.Texts) - .Include(novel => novel.Source) + .Include(novel => novel.Source).Include(novel => novel.Chapters).ThenInclude(chapter => chapter.Images) .FirstOrDefaultAsync(); var chapter = novel.Chapters.Where(chapter => chapter.Order == chapterNumber).FirstOrDefault(); var adapter = _sourceAdapters.FirstOrDefault(adapter => adapter.SourceDescriptor.Key == novel.Source.Key); var rawChapter = await adapter.GetRawChapter(chapter.Url); - chapter.Body.Texts.Add(new LocalizationText() + var localizationText = new LocalizationText() { - Text = rawChapter, + Text = rawChapter.Text, Language = novel.RawLanguage - }); + }; + chapter.Body.Texts.Add(localizationText); + chapter.Images = rawChapter.ImageData.Select(img => new Image() + { + OriginalPath = img.Url + }).ToList(); await _dbContext.SaveChangesAsync(); + + // Images are saved and have ids, update the chapter body to replace image tags + var chapterDoc = new HtmlDocument(); + chapterDoc.LoadHtml(rawChapter.Text); + foreach (var image in chapter.Images) + { + var match = chapterDoc.DocumentNode.SelectSingleNode(@$"//img[@src='{image.OriginalPath}']"); + if (match != null) + { + match.Attributes["src"].Value = _novelUpdateServiceConfiguration.PendingImageUrl; + if (match.Attributes.Contains("alt")) + { + match.Attributes["alt"].Value = image.Id.ToString(); + } + else + { + match.Attributes.Add("alt", image.Id.ToString()); + } + } + } + localizationText.Text = chapterDoc.DocumentNode.OuterHtml; + await _dbContext.SaveChangesAsync(); + + // Body was updated, raise image request + int imgCount = 0; + foreach (var image in chapter.Images) + { + var data = rawChapter.ImageData.FirstOrDefault(img => img.Url == image.OriginalPath); + await _eventBus.Publish(new FileUploadRequestCreatedEvent() + { + FileData = data.Data, + FilePath = $"{novel.Id}/Images/Chapter-{chapter.Id}/{imgCount++}.jpg", + RequestId = image.Id + }); + } + return chapter; } -} \ No newline at end of file + + public async Task UpdateImage(Guid imageId, string newUrl) + { + var image = await _dbContext.Images + .Include(img => img.Chapter) + .ThenInclude(chapter => chapter.Body) + .ThenInclude(body => body.Texts) + .FirstOrDefaultAsync(image => image.Id == imageId); + image.NewPath = newUrl; + + // If this is an image from a chapter, let's update the chapter body(s) + if (image.Chapter != null) + { + foreach (var bodyText in image.Chapter.Body.Texts) + { + var chapterDoc = new HtmlDocument(); + chapterDoc.LoadHtml(bodyText.Text); + var match = chapterDoc.DocumentNode.SelectSingleNode(@$"//img[@alt='{image.Id}']"); + if (match != null) + { + match.Attributes["src"].Value = newUrl; + } + } + } + + await _dbContext.SaveChangesAsync(); + } +} diff --git a/FictionArchive.Service.NovelService/Services/SourceAdapters/ISourceAdapter.cs b/FictionArchive.Service.NovelService/Services/SourceAdapters/ISourceAdapter.cs index cd4ff0b..d4870f4 100644 --- a/FictionArchive.Service.NovelService/Services/SourceAdapters/ISourceAdapter.cs +++ b/FictionArchive.Service.NovelService/Services/SourceAdapters/ISourceAdapter.cs @@ -8,5 +8,5 @@ public interface ISourceAdapter public SourceDescriptor SourceDescriptor { get; } public Task CanProcessNovel(string url); public Task GetMetadata(string novelUrl); - public Task GetRawChapter(string chapterUrl); + public Task GetRawChapter(string chapterUrl); } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAdapter.cs b/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAdapter.cs index b99e785..ae22269 100644 --- a/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAdapter.cs +++ b/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAdapter.cs @@ -81,6 +81,21 @@ public class NovelpiaAdapter : ISourceAdapter novel.AuthorName = authorMatch.Groups[2].Value; novel.AuthorUrl = authorMatch.Groups[2].Value; + // Cover image URL + var coverMatch = Regex.Match(novelData, @"href=""(//images\.novelpia\.com/imagebox/cover/.+?\.file)"""); + string coverImageUrl = coverMatch.Groups[1].Value; + if (string.IsNullOrEmpty(coverImageUrl)) + { + coverMatch = Regex.Match(novelData, @"src=""(//images\.novelpia\.com/imagebox/cover/.+?\.file)"""); + coverImageUrl = coverMatch.Groups[1].Value; + } + + novel.CoverImage = new ImageData() + { + Url = coverImageUrl, + Data = await GetImageData(coverImageUrl), + }; + // Some badge info var badgeSet = Regex.Match(novelData, @"(?s)(.*?)<\/p>"); var badgeMatches = Regex.Matches(badgeSet.Groups[1].Value, @"]*>(.*?)<\/span>"); @@ -160,7 +175,7 @@ public class NovelpiaAdapter : ISourceAdapter return novel; } - public async Task GetRawChapter(string chapterUrl) + public async Task GetRawChapter(string chapterUrl) { var chapterId = uint.Parse(Regex.Match(chapterUrl, ChapterIdRegex).Groups[1].Value); var endpoint = ChapterDownloadEndpoint + chapterId; @@ -171,6 +186,11 @@ public class NovelpiaAdapter : ISourceAdapter { throw new Exception(); } + + var fetchResult = new ChapterFetchResult() + { + ImageData = new List() + }; StringBuilder builder = new StringBuilder(); using var doc = JsonDocument.Parse(responseContent); @@ -182,10 +202,20 @@ public class NovelpiaAdapter : ISourceAdapter foreach (JsonElement item in sArray.EnumerateArray()) { string text = item.GetProperty("text").GetString(); + var imageMatch = Regex.Match(text, @""); if (text.Contains("cover-wrapper")) { continue; } + if (imageMatch.Success) + { + var url = imageMatch.Groups[1].Value; + fetchResult.ImageData.Add(new ImageData() + { + Url = url, + Data = await GetImageData(url) + }); + } if (text.Contains("opacity: 0")) { continue; @@ -193,8 +223,24 @@ public class NovelpiaAdapter : ISourceAdapter builder.Append(WebUtility.HtmlDecode(text)); } + fetchResult.Text = builder.ToString(); - return builder.ToString(); - + return fetchResult; + } + + private async Task GetImageData(string url) + { + if (!url.StartsWith("http")) + { + url = "https:" + url; + } + + var image = await _httpClient.GetAsync(url); + if (!image.IsSuccessStatusCode) + { + _logger.LogError("Attempting to fetch image with url {imgUrl} returned status code {code}.", url, image.StatusCode); + throw new Exception(); + } + return await image.Content.ReadAsByteArrayAsync(); } } \ No newline at end of file diff --git a/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAuthMessageHandler.cs b/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAuthMessageHandler.cs index 532985c..466d20b 100644 --- a/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAuthMessageHandler.cs +++ b/FictionArchive.Service.NovelService/Services/SourceAdapters/Novelpia/NovelpiaAuthMessageHandler.cs @@ -52,10 +52,15 @@ public class NovelpiaAuthMessageHandler : DelegatingHandler var response = await _httpClient.SendAsync(loginMessage); using (var streamReader = new StreamReader(response.Content.ReadAsStream())) { - if (streamReader.ReadToEnd().Contains(LoginSuccessMessage)) + var message = await streamReader.ReadToEndAsync(); + if (message.Contains(LoginSuccessMessage)) { _cache.Set(CacheKey, loginKey); } + else + { + throw new Exception("An error occured while retrieving the login key. Message: " + message); + } } } diff --git a/FictionArchive.Service.NovelService/appsettings.json b/FictionArchive.Service.NovelService/appsettings.json index bde9f89..3bf5160 100644 --- a/FictionArchive.Service.NovelService/appsettings.json +++ b/FictionArchive.Service.NovelService/appsettings.json @@ -9,6 +9,9 @@ "Username": "REPLACE_ME", "Password": "REPLACE_ME" }, + "UpdateService": { + "PendingImageUrl": "https://localhost:7247/api/pendingupload.png" + }, "ConnectionStrings": { "DefaultConnection": "Host=localhost;Database=FictionArchive_NovelService;Username=postgres;password=postgres" }, diff --git a/FictionArchive.sln b/FictionArchive.sln index cee3dc2..81af8d3 100644 --- a/FictionArchive.sln +++ b/FictionArchive.sln @@ -16,6 +16,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FictionArchive.Service.User EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FictionArchive.Service.AuthenticationService", "FictionArchive.Service.AuthenticationService\FictionArchive.Service.AuthenticationService.csproj", "{70C4AE82-B01E-421D-B590-C0F47E63CD0C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FictionArchive.Service.FileService", "FictionArchive.Service.FileService\FictionArchive.Service.FileService.csproj", "{EC64A336-F8A0-4BED-9CA3-1B05AD00631D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FictionArchive.Service.NovelService.Tests", "FictionArchive.Service.NovelService.Tests\FictionArchive.Service.NovelService.Tests.csproj", "{166E645E-9DFB-44E8-8CC8-FA249A11679F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -54,5 +58,13 @@ Global {70C4AE82-B01E-421D-B590-C0F47E63CD0C}.Debug|Any CPU.Build.0 = Debug|Any CPU {70C4AE82-B01E-421D-B590-C0F47E63CD0C}.Release|Any CPU.ActiveCfg = Release|Any CPU {70C4AE82-B01E-421D-B590-C0F47E63CD0C}.Release|Any CPU.Build.0 = Release|Any CPU + {EC64A336-F8A0-4BED-9CA3-1B05AD00631D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC64A336-F8A0-4BED-9CA3-1B05AD00631D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC64A336-F8A0-4BED-9CA3-1B05AD00631D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC64A336-F8A0-4BED-9CA3-1B05AD00631D}.Release|Any CPU.Build.0 = Release|Any CPU + {166E645E-9DFB-44E8-8CC8-FA249A11679F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {166E645E-9DFB-44E8-8CC8-FA249A11679F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {166E645E-9DFB-44E8-8CC8-FA249A11679F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {166E645E-9DFB-44E8-8CC8-FA249A11679F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal