[FA-misc] Saga seems to work, fixed a UserNovelDataService bug

This commit is contained in:
gamer147
2026-01-28 12:11:06 -05:00
parent 579e05b853
commit ec967770d3
34 changed files with 1341 additions and 97 deletions

View File

@@ -1,7 +1,6 @@
using Amazon.S3;
using Amazon.S3.Model;
using FictionArchive.Common.Enums;
using FictionArchive.Service.FileService.Contracts;
using FictionArchive.Service.FileService.Models;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
@@ -53,6 +52,7 @@ public class FileUploadRequestCreatedConsumer : IConsumer<IFileUploadRequestCrea
await _publishEndpoint.Publish<IFileUploadRequestStatusUpdate>(
new FileUploadRequestStatusUpdate(
ImportId: message.ImportId,
RequestId: message.RequestId,
Status: RequestStatus.Failed,
FileAccessUrl: null,
@@ -66,6 +66,7 @@ public class FileUploadRequestCreatedConsumer : IConsumer<IFileUploadRequestCrea
await _publishEndpoint.Publish<IFileUploadRequestStatusUpdate>(
new FileUploadRequestStatusUpdate(
ImportId: message.ImportId,
RequestId: message.RequestId,
Status: RequestStatus.Success,
FileAccessUrl: fileAccessUrl,

View File

@@ -1,10 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.FileService.Contracts;
public record FileUploadRequestStatusUpdate(
Guid RequestId,
RequestStatus Status,
string? FileAccessUrl,
string? ErrorMessage) : IFileUploadRequestStatusUpdate;

View File

@@ -9,8 +9,10 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="MassTransit" Version="8.5.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="9.0.11" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="NodaTime.Testing" Version="3.3.0" />
<PackageReference Include="NSubstitute" Version="5.1.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">

View File

@@ -14,6 +14,7 @@ using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NodaTime;
using NSubstitute;
using Xunit;
@@ -80,7 +81,10 @@ public class NovelUpdateServiceTests
PendingImageUrl = pendingImageUrl
});
return new NovelUpdateService(dbContext, NullLogger<NovelUpdateService>.Instance, new[] { adapter }, publishEndpoint, options);
var clock = Substitute.For<IClock>();
clock.GetCurrentInstant().Returns(Instant.FromUnixTimeSeconds(0));
return new NovelUpdateService(dbContext, NullLogger<NovelUpdateService>.Instance, new[] { adapter }, publishEndpoint, options, clock);
}
[Fact]
@@ -110,8 +114,10 @@ public class NovelUpdateServiceTests
var pendingImageUrl = "https://pending/placeholder.jpg";
var service = CreateService(dbContext, adapter, publishEndpoint, pendingImageUrl);
var updatedChapter = await service.PullChapterContents(novel.Id, volume.Id, chapter.Order);
var importId = Guid.NewGuid();
var (updatedChapter, imageCount) = await service.PullChapterContents(importId, novel.Id, volume.Id, chapter.Order);
imageCount.Should().Be(2);
updatedChapter.Images.Should().HaveCount(2);
updatedChapter.Images.Select(i => i.OriginalPath).Should().BeEquivalentTo(new[] { image1.Url, image2.Url });
updatedChapter.Images.All(i => i.Id != Guid.Empty).Should().BeTrue();
@@ -128,9 +134,10 @@ public class NovelUpdateServiceTests
.BeEquivalentTo(updatedChapter.Images.Select(img => img.Id.ToString()));
publishedEvents.Should().HaveCount(2);
publishedEvents.Should().OnlyContain(e => e.ImportId == importId);
publishedEvents.Select(e => e.RequestId).Should().BeEquivalentTo(updatedChapter.Images.Select(i => i.Id));
publishedEvents.Select(e => e.FileData).Should().BeEquivalentTo(new[] { image1.Data, image2.Data });
publishedEvents.Should().OnlyContain(e => e.FilePath.StartsWith($"{novel.Id}/Images/Chapter-{updatedChapter.Id}/"));
publishedEvents.Should().OnlyContain(e => e.FilePath.StartsWith($"Novels/{novel.Id}/Images/Chapter-{updatedChapter.Id}/"));
}
[Fact]
@@ -155,8 +162,10 @@ public class NovelUpdateServiceTests
var service = CreateService(dbContext, adapter, publishEndpoint);
var updatedChapter = await service.PullChapterContents(novel.Id, volume.Id, chapter.Order);
var importId = Guid.NewGuid();
var (updatedChapter, imageCount) = await service.PullChapterContents(importId, novel.Id, volume.Id, chapter.Order);
imageCount.Should().Be(1);
var storedHtml = updatedChapter.Body.Texts.Single().Text;
var doc = new HtmlDocument();
doc.LoadHtml(storedHtml);

View File

@@ -0,0 +1,95 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.NovelService.Sagas;
using FictionArchive.Service.Shared.Contracts.Events;
using FluentAssertions;
using MassTransit;
using MassTransit.Testing;
using Microsoft.Extensions.DependencyInjection;
using NodaTime;
using NodaTime.Testing;
using Xunit;
namespace FictionArchive.Service.NovelService.Tests.Sagas;
public class NovelImportSagaTests
{
private readonly FakeClock _clock = new(Instant.FromUtc(2026, 1, 27, 12, 0, 0));
[Fact]
public async Task Should_transition_to_importing_on_import_requested()
{
await using var provider = CreateTestProvider();
var harness = provider.GetRequiredService<ITestHarness>();
await harness.Start();
var importId = Guid.NewGuid();
await harness.Bus.Publish<INovelImportRequested>(new NovelImportRequested(importId, "https://example.com/novel"));
var sagaHarness = harness.GetSagaStateMachineHarness<NovelImportSaga, NovelImportSagaState>();
(await sagaHarness.Exists(importId, x => x.Importing)).HasValue.Should().BeTrue();
}
[Fact]
public async Task Should_transition_to_completed_when_no_chapters()
{
await using var provider = CreateTestProvider();
var harness = provider.GetRequiredService<ITestHarness>();
await harness.Start();
var importId = Guid.NewGuid();
await harness.Bus.Publish<INovelImportRequested>(new NovelImportRequested(importId, "https://example.com/novel"));
await harness.Bus.Publish<INovelMetadataImported>(new NovelMetadataImported(importId, 1, 0));
var sagaHarness = harness.GetSagaStateMachineHarness<NovelImportSaga, NovelImportSagaState>();
(await sagaHarness.Exists(importId, x => x.Completed)).HasValue.Should().BeTrue();
(await harness.Published.Any<INovelImportCompleted>(x =>
x.Context.Message.ImportId == importId && x.Context.Message.Success)).Should().BeTrue();
}
[Fact]
public async Task Should_transition_to_processing_when_chapters_pending()
{
await using var provider = CreateTestProvider();
var harness = provider.GetRequiredService<ITestHarness>();
await harness.Start();
var importId = Guid.NewGuid();
await harness.Bus.Publish<INovelImportRequested>(new NovelImportRequested(importId, "https://example.com/novel"));
await harness.Bus.Publish<INovelMetadataImported>(new NovelMetadataImported(importId, 1, 2));
var sagaHarness = harness.GetSagaStateMachineHarness<NovelImportSaga, NovelImportSagaState>();
(await sagaHarness.Exists(importId, x => x.Processing)).HasValue.Should().BeTrue();
}
[Fact]
public async Task Should_complete_when_all_chapters_pulled_and_images_uploaded()
{
await using var provider = CreateTestProvider();
var harness = provider.GetRequiredService<ITestHarness>();
await harness.Start();
var importId = Guid.NewGuid();
await harness.Bus.Publish<INovelImportRequested>(new NovelImportRequested(importId, "https://example.com/novel"));
await harness.Bus.Publish<INovelMetadataImported>(new NovelMetadataImported(importId, 1, 2));
await harness.Bus.Publish<IChapterPullCompleted>(new ChapterPullCompleted(importId, 1, 1));
await harness.Bus.Publish<IChapterPullCompleted>(new ChapterPullCompleted(importId, 2, 0));
await harness.Bus.Publish<IFileUploadRequestStatusUpdate>(new FileUploadRequestStatusUpdate(
importId, Guid.NewGuid(), RequestStatus.Success, "https://cdn.example.com/image.jpg", null));
var sagaHarness = harness.GetSagaStateMachineHarness<NovelImportSaga, NovelImportSagaState>();
(await sagaHarness.Exists(importId, x => x.Completed)).HasValue.Should().BeTrue();
}
private ServiceProvider CreateTestProvider()
{
return new ServiceCollection()
.AddSingleton<IClock>(_clock)
.AddMassTransitTestHarness(cfg =>
{
cfg.AddSagaStateMachine<NovelImportSaga, NovelImportSagaState>()
.InMemoryRepository();
})
.BuildServiceProvider(true);
}
}

View File

@@ -21,6 +21,17 @@ public class ChapterPullRequestedConsumer : IConsumer<IChapterPullRequested>
public async Task Consume(ConsumeContext<IChapterPullRequested> context)
{
var message = context.Message;
await _novelUpdateService.PullChapterContents(message.NovelId, message.VolumeId, message.ChapterOrder);
var (chapter, imageCount) = await _novelUpdateService.PullChapterContents(
message.ImportId,
message.NovelId,
message.VolumeId,
message.ChapterOrder);
await context.Publish<IChapterPullCompleted>(new ChapterPullCompleted(
message.ImportId,
chapter.Id,
imageCount
));
}
}

View File

@@ -0,0 +1,43 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelImportCompletedConsumer : IConsumer<INovelImportCompleted>
{
private readonly ILogger<NovelImportCompletedConsumer> _logger;
private readonly NovelServiceDbContext _dbContext;
public NovelImportCompletedConsumer(
ILogger<NovelImportCompletedConsumer> logger,
NovelServiceDbContext dbContext)
{
_logger = logger;
_dbContext = dbContext;
}
public async Task Consume(ConsumeContext<INovelImportCompleted> context)
{
var message = context.Message;
_logger.LogInformation(
"Novel import {ImportId} completed. Success: {Success}, NovelId: {NovelId}, Error: {Error}",
message.ImportId,
message.Success,
message.NovelId,
message.ErrorMessage);
// Remove from ActiveImports to allow future imports
var activeImport = await _dbContext.ActiveImports
.FirstOrDefaultAsync(a => a.ImportId == message.ImportId);
if (activeImport != null)
{
_dbContext.ActiveImports.Remove(activeImport);
await _dbContext.SaveChangesAsync();
}
}
}

View File

@@ -0,0 +1,29 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelImportRequestedConsumer : IConsumer<INovelImportRequested>
{
private readonly ILogger<NovelImportRequestedConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public NovelImportRequestedConsumer(
ILogger<NovelImportRequestedConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<INovelImportRequested> context)
{
var message = context.Message;
_logger.LogInformation("Starting novel import for {NovelUrl} with ImportId {ImportId}",
message.NovelUrl, message.ImportId);
await _novelUpdateService.ImportNovel(message.ImportId, message.NovelUrl);
}
}

View File

@@ -1,26 +0,0 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelUpdateRequestedConsumer : IConsumer<INovelUpdateRequested>
{
private readonly ILogger<NovelUpdateRequestedConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public NovelUpdateRequestedConsumer(
ILogger<NovelUpdateRequestedConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<INovelUpdateRequested> context)
{
var message = context.Message;
await _novelUpdateService.ImportNovel(message.NovelUrl);
}
}

View File

@@ -1,8 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record ChapterPullRequested(
uint NovelId,
uint VolumeId,
uint ChapterOrder) : IChapterPullRequested;

View File

@@ -1,8 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record FileUploadRequestCreated(
Guid RequestId,
string FilePath,
byte[] FileData) : IFileUploadRequestCreated;

View File

@@ -1,6 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record NovelUpdateRequested(
string NovelUrl) : INovelUpdateRequested;

View File

@@ -10,11 +10,12 @@
<ItemGroup>
<PackageReference Include="HotChocolate.AspNetCore.CommandLine" Version="15.1.11" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
<PackageReference Include="MassTransit.EntityFrameworkCore" Version="8.5.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.11">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
</ItemGroup>
<ItemGroup>

View File

@@ -5,6 +5,7 @@ using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Models.SourceAdapters;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.Shared.Contracts.Events;
using HotChocolate.Authorization;
using HotChocolate.Types;
using Microsoft.EntityFrameworkCore;
@@ -13,20 +14,22 @@ namespace FictionArchive.Service.NovelService.GraphQL;
public class Mutation
{
[Error<InvalidOperationException>]
[Authorize]
public async Task<NovelUpdateRequested> ImportNovel(string novelUrl, NovelUpdateService service)
public async Task<NovelImportRequested> ImportNovel(string novelUrl, NovelUpdateService service)
{
return await service.QueueNovelImport(novelUrl);
}
[Authorize]
public async Task<ChapterPullRequested> FetchChapterContents(
Guid importId,
uint novelId,
uint volumeId,
uint chapterOrder,
NovelUpdateService service)
{
return await service.QueueChapterPull(novelId, volumeId, chapterOrder);
return await service.QueueChapterPull(importId, novelId, volumeId, chapterOrder);
}
[Error<KeyNotFoundException>]

View File

@@ -0,0 +1,673 @@
// <auto-generated />
using System;
using FictionArchive.Service.NovelService.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FictionArchive.Service.NovelService.Migrations
{
[DbContext(typeof(NovelServiceDbContext))]
[Migration("20260127161500_AddNovelImportSaga")]
partial class AddNovelImportSaga
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.11")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.ActiveImport", b =>
{
b.Property<Guid>("ImportId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("ImportId");
b.HasIndex("NovelUrl")
.IsUnique();
b.ToTable("ActiveImports");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<long?>("ChapterId")
.HasColumnType("bigint");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("NewPath")
.HasColumnType("text");
b.Property<string>("OriginalPath")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("ChapterId");
b.ToTable("Images");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("LocalizationKeys");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<long>("EngineId")
.HasColumnType("bigint");
b.Property<Guid>("KeyRequestedForTranslationId")
.HasColumnType("uuid");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<int>("TranslateTo")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("EngineId");
b.HasIndex("KeyRequestedForTranslationId");
b.ToTable("LocalizationRequests");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<int>("Language")
.HasColumnType("integer");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid?>("LocalizationKeyId")
.HasColumnType("uuid");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.Property<long?>("TranslationEngineId")
.HasColumnType("bigint");
b.HasKey("Id");
b.HasIndex("LocalizationKeyId");
b.HasIndex("TranslationEngineId");
b.ToTable("LocalizationText");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Guid>("BodyId")
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<long>("Order")
.HasColumnType("bigint");
b.Property<long>("Revision")
.HasColumnType("bigint");
b.Property<string>("Url")
.HasColumnType("text");
b.Property<long>("VolumeId")
.HasColumnType("bigint");
b.HasKey("Id");
b.HasIndex("BodyId");
b.HasIndex("NameId");
b.HasIndex("VolumeId", "Order")
.IsUnique();
b.ToTable("Chapter");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("AuthorId")
.HasColumnType("bigint");
b.Property<Guid?>("CoverImageId")
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("DescriptionId")
.HasColumnType("uuid");
b.Property<string>("ExternalId")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<int>("RawLanguage")
.HasColumnType("integer");
b.Property<int>("RawStatus")
.HasColumnType("integer");
b.Property<long>("SourceId")
.HasColumnType("bigint");
b.Property<int?>("StatusOverride")
.HasColumnType("integer");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AuthorId");
b.HasIndex("CoverImageId");
b.HasIndex("DescriptionId");
b.HasIndex("NameId");
b.HasIndex("SourceId");
b.HasIndex("ExternalId", "SourceId")
.IsUnique();
b.ToTable("Novels");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("DisplayNameId")
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<long?>("SourceId")
.HasColumnType("bigint");
b.Property<int>("TagType")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("DisplayNameId");
b.HasIndex("SourceId");
b.ToTable("Tags");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("ExternalUrl")
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("NameId");
b.ToTable("Person");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Source", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Sources");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("TranslationEngines");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<long>("NovelId")
.HasColumnType("bigint");
b.Property<int>("Order")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("NameId");
b.HasIndex("NovelId", "Order")
.IsUnique();
b.ToTable("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Sagas.NovelImportSagaState", b =>
{
b.Property<Guid>("CorrelationId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone");
b.Property<int>("CompletedChapters")
.HasColumnType("integer");
b.Property<int>("CompletedImages")
.HasColumnType("integer");
b.Property<string>("CurrentState")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ErrorMessage")
.HasColumnType("text");
b.Property<int>("ExpectedChapters")
.HasColumnType("integer");
b.Property<int>("ExpectedImages")
.HasColumnType("integer");
b.Property<long?>("NovelId")
.HasColumnType("bigint");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("CorrelationId");
b.HasIndex("CurrentState");
b.HasIndex("NovelUrl");
b.ToTable("NovelImportSagaStates");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.Property<long>("NovelsId")
.HasColumnType("bigint");
b.Property<long>("TagsId")
.HasColumnType("bigint");
b.HasKey("NovelsId", "TagsId");
b.HasIndex("TagsId");
b.ToTable("NovelNovelTag");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Chapter", "Chapter")
.WithMany("Images")
.HasForeignKey("ChapterId");
b.Navigation("Chapter");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "Engine")
.WithMany()
.HasForeignKey("EngineId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "KeyRequestedForTranslation")
.WithMany()
.HasForeignKey("KeyRequestedForTranslationId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Engine");
b.Navigation("KeyRequestedForTranslation");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", null)
.WithMany("Texts")
.HasForeignKey("LocalizationKeyId");
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "TranslationEngine")
.WithMany()
.HasForeignKey("TranslationEngineId");
b.Navigation("TranslationEngine");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Body")
.WithMany()
.HasForeignKey("BodyId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Volume", "Volume")
.WithMany("Chapters")
.HasForeignKey("VolumeId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Body");
b.Navigation("Name");
b.Navigation("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Person", "Author")
.WithMany()
.HasForeignKey("AuthorId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Images.Image", "CoverImage")
.WithMany()
.HasForeignKey("CoverImageId");
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Description")
.WithMany()
.HasForeignKey("DescriptionId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source")
.WithMany()
.HasForeignKey("SourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Author");
b.Navigation("CoverImage");
b.Navigation("Description");
b.Navigation("Name");
b.Navigation("Source");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "DisplayName")
.WithMany()
.HasForeignKey("DisplayNameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source")
.WithMany()
.HasForeignKey("SourceId");
b.Navigation("DisplayName");
b.Navigation("Source");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Name");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", "Novel")
.WithMany("Volumes")
.HasForeignKey("NovelId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Name");
b.Navigation("Novel");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", null)
.WithMany()
.HasForeignKey("NovelsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.NovelTag", null)
.WithMany()
.HasForeignKey("TagsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b =>
{
b.Navigation("Texts");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.Navigation("Images");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.Navigation("Volumes");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.Navigation("Chapters");
});
#pragma warning restore 612, 618
}
}
}

View File

@@ -0,0 +1,76 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace FictionArchive.Service.NovelService.Migrations
{
/// <inheritdoc />
public partial class AddNovelImportSaga : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "ActiveImports",
columns: table => new
{
ImportId = table.Column<Guid>(type: "uuid", nullable: false),
NovelUrl = table.Column<string>(type: "text", nullable: false),
StartedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_ActiveImports", x => x.ImportId);
});
migrationBuilder.CreateTable(
name: "NovelImportSagaStates",
columns: table => new
{
CorrelationId = table.Column<Guid>(type: "uuid", nullable: false),
CurrentState = table.Column<string>(type: "text", nullable: false),
NovelUrl = table.Column<string>(type: "text", nullable: false),
NovelId = table.Column<long>(type: "bigint", nullable: true),
ExpectedChapters = table.Column<int>(type: "integer", nullable: false),
CompletedChapters = table.Column<int>(type: "integer", nullable: false),
ExpectedImages = table.Column<int>(type: "integer", nullable: false),
CompletedImages = table.Column<int>(type: "integer", nullable: false),
StartedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
CompletedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
ErrorMessage = table.Column<string>(type: "text", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_NovelImportSagaStates", x => x.CorrelationId);
});
migrationBuilder.CreateIndex(
name: "IX_ActiveImports_NovelUrl",
table: "ActiveImports",
column: "NovelUrl",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_NovelImportSagaStates_CurrentState",
table: "NovelImportSagaStates",
column: "CurrentState");
migrationBuilder.CreateIndex(
name: "IX_NovelImportSagaStates_NovelUrl",
table: "NovelImportSagaStates",
column: "NovelUrl");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "ActiveImports");
migrationBuilder.DropTable(
name: "NovelImportSagaStates");
}
}
}

View File

@@ -23,6 +23,27 @@ namespace FictionArchive.Service.NovelService.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.ActiveImport", b =>
{
b.Property<Guid>("ImportId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("ImportId");
b.HasIndex("NovelUrl")
.IsUnique();
b.ToTable("ActiveImports");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.Property<Guid>("Id")
@@ -391,6 +412,53 @@ namespace FictionArchive.Service.NovelService.Migrations
b.ToTable("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Sagas.NovelImportSagaState", b =>
{
b.Property<Guid>("CorrelationId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone");
b.Property<int>("CompletedChapters")
.HasColumnType("integer");
b.Property<int>("CompletedImages")
.HasColumnType("integer");
b.Property<string>("CurrentState")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ErrorMessage")
.HasColumnType("text");
b.Property<int>("ExpectedChapters")
.HasColumnType("integer");
b.Property<int>("ExpectedImages")
.HasColumnType("integer");
b.Property<long?>("NovelId")
.HasColumnType("bigint");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("CorrelationId");
b.HasIndex("CurrentState");
b.HasIndex("NovelUrl");
b.ToTable("NovelImportSagaStates");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.Property<long>("NovelsId")

View File

@@ -0,0 +1,10 @@
using NodaTime;
namespace FictionArchive.Service.NovelService.Models;
public class ActiveImport
{
public Guid ImportId { get; set; }
public required string NovelUrl { get; set; }
public Instant StartedAt { get; set; }
}

View File

@@ -2,13 +2,16 @@ using FictionArchive.Common.Extensions;
using FictionArchive.Service.NovelService.Consumers;
using FictionArchive.Service.NovelService.GraphQL;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Sagas;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.NovelService.Services.SourceAdapters.Novelpia;
using FictionArchive.Service.Shared;
using FictionArchive.Service.Shared.Extensions;
using FictionArchive.Service.Shared.Services.GraphQL;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace FictionArchive.Service.NovelService;
@@ -33,8 +36,17 @@ public class Program
{
x.AddConsumer<TranslationRequestCompletedConsumer>();
x.AddConsumer<FileUploadRequestStatusUpdateConsumer>();
x.AddConsumer<NovelUpdateRequestedConsumer>();
x.AddConsumer<ChapterPullRequestedConsumer>();
x.AddConsumer<NovelImportRequestedConsumer>();
x.AddConsumer<NovelImportCompletedConsumer>();
x.AddSagaStateMachine<NovelImportSaga, NovelImportSagaState>()
.EntityFrameworkRepository(r =>
{
r.ConcurrencyMode = ConcurrencyMode.Optimistic;
r.ExistingDbContext<NovelServiceDbContext>();
r.UsePostgres();
});
});
}
@@ -72,9 +84,12 @@ public class Program
builder.Services.Configure<NovelUpdateServiceConfiguration>(builder.Configuration.GetSection("UpdateService"));
builder.Services.AddTransient<NovelUpdateService>();
#endregion
// Register IClock for saga and service use
builder.Services.AddSingleton<IClock>(SystemClock.Instance);
builder.Services.AddHealthChecks();
// Authentication & Authorization

View File

@@ -0,0 +1,135 @@
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using NodaTime;
namespace FictionArchive.Service.NovelService.Sagas;
public class NovelImportSaga : MassTransitStateMachine<NovelImportSagaState>
{
public State Importing { get; private set; } = null!;
public State Processing { get; private set; } = null!;
public State Completed { get; private set; } = null!;
public State Failed { get; private set; } = null!;
public Event<INovelImportRequested> NovelImportRequested { get; private set; } = null!;
public Event<INovelMetadataImported> NovelMetadataImported { get; private set; } = null!;
public Event<IChapterPullCompleted> ChapterPullCompleted { get; private set; } = null!;
public Event<IFileUploadRequestStatusUpdate> FileUploadStatusUpdate { get; private set; } = null!;
public Event<Fault<IChapterPullRequested>> ChapterPullFaulted { get; private set; } = null!;
public Event<Fault<IFileUploadRequestCreated>> FileUploadFaulted { get; private set; } = null!;
private readonly IClock _clock;
public NovelImportSaga(IClock clock)
{
_clock = clock;
InstanceState(x => x.CurrentState);
Event(() => NovelImportRequested, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => NovelMetadataImported, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => ChapterPullCompleted, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => FileUploadStatusUpdate, x =>
{
x.CorrelateById(ctx => ctx.Message.ImportId ?? Guid.Empty);
x.OnMissingInstance(m => m.Discard());
});
Event(() => ChapterPullFaulted, x => x.CorrelateById(ctx => ctx.Message.Message.ImportId));
Event(() => FileUploadFaulted, x =>
{
x.CorrelateById(ctx => ctx.Message.Message.ImportId ?? Guid.Empty);
x.OnMissingInstance(m => m.Discard());
});
Initially(
When(NovelImportRequested)
.Then(ctx =>
{
ctx.Saga.NovelUrl = ctx.Message.NovelUrl;
ctx.Saga.StartedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Importing)
);
During(Importing,
When(NovelMetadataImported)
.Then(ctx =>
{
ctx.Saga.NovelId = ctx.Message.NovelId;
ctx.Saga.ExpectedChapters = ctx.Message.ChaptersPendingPull;
})
.IfElse(
ctx => ctx.Saga.ExpectedChapters == 0,
thenBinder => thenBinder
.Then(ctx => ctx.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
true,
null))),
elseBinder => elseBinder.TransitionTo(Processing)
)
);
During(Processing,
When(ChapterPullCompleted)
.Then(ctx =>
{
ctx.Saga.CompletedChapters++;
ctx.Saga.ExpectedImages += ctx.Message.ImagesQueued;
})
.If(ctx => IsComplete(ctx.Saga), ctx => ctx
.Then(c => c.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(c => c.Init<INovelImportCompleted>(new NovelImportCompleted(
c.Saga.CorrelationId,
c.Saga.NovelId,
true,
null)))),
When(FileUploadStatusUpdate)
.Then(ctx => ctx.Saga.CompletedImages++)
.If(ctx => IsComplete(ctx.Saga), ctx => ctx
.Then(c => c.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(c => c.Init<INovelImportCompleted>(new NovelImportCompleted(
c.Saga.CorrelationId,
c.Saga.NovelId,
true,
null)))),
When(ChapterPullFaulted)
.Then(ctx =>
{
ctx.Saga.ErrorMessage = ctx.Message.Exceptions.FirstOrDefault()?.Message;
ctx.Saga.CompletedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Failed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
false,
ctx.Saga.ErrorMessage))),
When(FileUploadFaulted)
.Then(ctx =>
{
ctx.Saga.ErrorMessage = ctx.Message.Exceptions.FirstOrDefault()?.Message;
ctx.Saga.CompletedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Failed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
false,
ctx.Saga.ErrorMessage)))
);
SetCompletedWhenFinalized();
}
private static bool IsComplete(NovelImportSagaState saga) =>
saga.CompletedChapters >= saga.ExpectedChapters &&
saga.CompletedImages >= saga.ExpectedImages;
}

View File

@@ -0,0 +1,29 @@
using MassTransit;
using NodaTime;
namespace FictionArchive.Service.NovelService.Sagas;
public class NovelImportSagaState : SagaStateMachineInstance
{
public Guid CorrelationId { get; set; }
public string CurrentState { get; set; } = null!;
// Identity
public string NovelUrl { get; set; } = null!;
public uint? NovelId { get; set; }
// Chapter tracking
public int ExpectedChapters { get; set; }
public int CompletedChapters { get; set; }
// Image tracking
public int ExpectedImages { get; set; }
public int CompletedImages { get; set; }
// Timestamps
public Instant StartedAt { get; set; }
public Instant? CompletedAt { get; set; }
// Error info
public string? ErrorMessage { get; set; }
}

View File

@@ -1,6 +1,8 @@
using FictionArchive.Service.NovelService.Models;
using FictionArchive.Service.NovelService.Models.Images;
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Sagas;
using FictionArchive.Service.Shared.Services.Database;
using Microsoft.EntityFrameworkCore;
@@ -18,6 +20,8 @@ public class NovelServiceDbContext(DbContextOptions options, ILogger<NovelServic
public DbSet<LocalizationKey> LocalizationKeys { get; set; }
public DbSet<LocalizationRequest> LocalizationRequests { get; set; }
public DbSet<Image> Images { get; set; }
public DbSet<ActiveImport> ActiveImports { get; set; }
public DbSet<NovelImportSagaState> NovelImportSagaStates { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
@@ -36,5 +40,18 @@ public class NovelServiceDbContext(DbContextOptions options, ILogger<NovelServic
modelBuilder.Entity<Chapter>()
.HasIndex("VolumeId", "Order")
.IsUnique();
modelBuilder.Entity<ActiveImport>(entity =>
{
entity.HasKey(e => e.ImportId);
entity.HasIndex(e => e.NovelUrl).IsUnique();
});
modelBuilder.Entity<NovelImportSagaState>(entity =>
{
entity.HasKey(e => e.CorrelationId);
entity.HasIndex(e => e.NovelUrl);
entity.HasIndex(e => e.CurrentState);
});
}
}

View File

@@ -1,5 +1,6 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.NovelService.Contracts;
using FictionArchive.Service.NovelService.Models;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Models.Enums;
using FictionArchive.Service.NovelService.Models.Images;
@@ -12,6 +13,7 @@ using HtmlAgilityPack;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using NodaTime;
namespace FictionArchive.Service.NovelService.Services;
@@ -22,14 +24,16 @@ public class NovelUpdateService
private readonly IEnumerable<ISourceAdapter> _sourceAdapters;
private readonly IPublishEndpoint _publishEndpoint;
private readonly NovelUpdateServiceConfiguration _novelUpdateServiceConfiguration;
private readonly IClock _clock;
public NovelUpdateService(NovelServiceDbContext dbContext, ILogger<NovelUpdateService> logger, IEnumerable<ISourceAdapter> sourceAdapters, IPublishEndpoint publishEndpoint, IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration)
public NovelUpdateService(NovelServiceDbContext dbContext, ILogger<NovelUpdateService> logger, IEnumerable<ISourceAdapter> sourceAdapters, IPublishEndpoint publishEndpoint, IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration, IClock clock)
{
_dbContext = dbContext;
_logger = logger;
_sourceAdapters = sourceAdapters;
_publishEndpoint = publishEndpoint;
_novelUpdateServiceConfiguration = novelUpdateServiceConfiguration.Value;
_clock = clock;
}
#region Helper Methods
@@ -299,7 +303,7 @@ public class NovelUpdateService
#endregion
public async Task<Novel> ImportNovel(string novelUrl)
public async Task<Novel> ImportNovel(Guid importId, string novelUrl)
{
// Step 1: Get metadata from source adapter
NovelMetadata? metadata = null;
@@ -417,10 +421,24 @@ public class NovelUpdateService
}
}
// Count chapters that need pulling
var chaptersNeedingPull = novel.Volumes
.SelectMany(v => v.Chapters)
.Where(c => c.Body?.Texts == null || !c.Body.Texts.Any())
.ToList();
// Publish metadata imported event for saga
await _publishEndpoint.Publish<INovelMetadataImported>(new NovelMetadataImported(
importId,
novel.Id,
chaptersNeedingPull.Count
));
// Publish cover image event if needed
if (shouldPublishCoverEvent && novel.CoverImage != null && metadata.CoverImage != null)
{
await _publishEndpoint.Publish<IFileUploadRequestCreated>(new FileUploadRequestCreated(
importId,
novel.CoverImage.Id,
$"Novels/{novel.Id}/Images/cover.jpg",
metadata.CoverImage.Data));
@@ -429,13 +447,14 @@ public class NovelUpdateService
// Publish chapter pull events for chapters without body content
foreach (var volume in novel.Volumes)
{
var chaptersNeedingPull = volume.Chapters
var volumeChaptersNeedingPull = volume.Chapters
.Where(c => c.Body?.Texts == null || !c.Body.Texts.Any())
.ToList();
foreach (var chapter in chaptersNeedingPull)
foreach (var chapter in volumeChaptersNeedingPull)
{
await _publishEndpoint.Publish<IChapterPullRequested>(new ChapterPullRequested(
importId,
novel.Id,
volume.Id,
chapter.Order));
@@ -445,7 +464,7 @@ public class NovelUpdateService
return novel;
}
public async Task<Chapter> PullChapterContents(uint novelId, uint volumeId, uint chapterOrder)
public async Task<(Chapter chapter, int imageCount)> PullChapterContents(Guid importId, uint novelId, uint volumeId, uint chapterOrder)
{
var novel = await _dbContext.Novels.Where(novel => novel.Id == novelId)
.Include(novel => novel.Volumes)
@@ -512,12 +531,13 @@ public class NovelUpdateService
{
var data = rawChapter.ImageData.FirstOrDefault(img => img.Url == image.OriginalPath);
await _publishEndpoint.Publish<IFileUploadRequestCreated>(new FileUploadRequestCreated(
importId,
image.Id,
$"Novels/{novel.Id}/Images/Chapter-{chapter.Id}/{imgCount++}.jpg",
data.Data));
}
return chapter;
return (chapter, chapter.Images.Count);
}
public async Task UpdateImage(Guid imageId, string newUrl)
@@ -548,16 +568,34 @@ public class NovelUpdateService
await _dbContext.SaveChangesAsync();
}
public async Task<NovelUpdateRequested> QueueNovelImport(string novelUrl)
public async Task<NovelImportRequested> QueueNovelImport(string novelUrl)
{
var importNovelRequestEvent = new NovelUpdateRequested(novelUrl);
await _publishEndpoint.Publish<INovelUpdateRequested>(importNovelRequestEvent);
var importId = Guid.NewGuid();
var activeImport = new ActiveImport
{
ImportId = importId,
NovelUrl = novelUrl,
StartedAt = _clock.GetCurrentInstant()
};
try
{
await _dbContext.ActiveImports.AddAsync(activeImport);
await _dbContext.SaveChangesAsync();
}
catch (DbUpdateException)
{
throw new InvalidOperationException($"An import is already in progress for {novelUrl}");
}
var importNovelRequestEvent = new NovelImportRequested(importId, novelUrl);
await _publishEndpoint.Publish<INovelImportRequested>(importNovelRequestEvent);
return importNovelRequestEvent;
}
public async Task<ChapterPullRequested> QueueChapterPull(uint novelId, uint volumeId, uint chapterOrder)
public async Task<ChapterPullRequested> QueueChapterPull(Guid importId, uint novelId, uint volumeId, uint chapterOrder)
{
var chapterPullEvent = new ChapterPullRequested(novelId, volumeId, chapterOrder);
var chapterPullEvent = new ChapterPullRequested(importId, novelId, volumeId, chapterOrder);
await _publishEndpoint.Publish<IChapterPullRequested>(chapterPullEvent);
return chapterPullEvent;
}

View File

@@ -0,0 +1,10 @@
namespace FictionArchive.Service.Shared.Contracts.Events;
public interface IChapterPullCompleted
{
Guid ImportId { get; }
uint ChapterId { get; }
int ImagesQueued { get; }
}
public record ChapterPullCompleted(Guid ImportId, uint ChapterId, int ImagesQueued) : IChapterPullCompleted;

View File

@@ -2,7 +2,10 @@ namespace FictionArchive.Service.Shared.Contracts.Events;
public interface IChapterPullRequested
{
Guid ImportId { get; }
uint NovelId { get; }
uint VolumeId { get; }
uint ChapterOrder { get; }
}
public record ChapterPullRequested(Guid ImportId, uint NovelId, uint VolumeId, uint ChapterOrder) : IChapterPullRequested;

View File

@@ -2,7 +2,10 @@ namespace FictionArchive.Service.Shared.Contracts.Events;
public interface IFileUploadRequestCreated
{
Guid? ImportId { get; }
Guid RequestId { get; }
string FilePath { get; }
byte[] FileData { get; }
}
public record FileUploadRequestCreated(Guid? ImportId, Guid RequestId, string FilePath, byte[] FileData) : IFileUploadRequestCreated;

View File

@@ -4,8 +4,11 @@ namespace FictionArchive.Service.Shared.Contracts.Events;
public interface IFileUploadRequestStatusUpdate
{
Guid? ImportId { get; }
Guid RequestId { get; }
RequestStatus Status { get; }
string? FileAccessUrl { get; }
string? ErrorMessage { get; }
}
public record FileUploadRequestStatusUpdate(Guid? ImportId, Guid RequestId, RequestStatus Status, string? FileAccessUrl, string? ErrorMessage) : IFileUploadRequestStatusUpdate;

View File

@@ -0,0 +1,11 @@
namespace FictionArchive.Service.Shared.Contracts.Events;
public interface INovelImportCompleted
{
Guid ImportId { get; }
uint? NovelId { get; }
bool Success { get; }
string? ErrorMessage { get; }
}
public record NovelImportCompleted(Guid ImportId, uint? NovelId, bool Success, string? ErrorMessage) : INovelImportCompleted;

View File

@@ -0,0 +1,9 @@
namespace FictionArchive.Service.Shared.Contracts.Events;
public interface INovelImportRequested
{
Guid ImportId { get; }
string NovelUrl { get; }
}
public record NovelImportRequested(Guid ImportId, string NovelUrl) : INovelImportRequested;

View File

@@ -0,0 +1,10 @@
namespace FictionArchive.Service.Shared.Contracts.Events;
public interface INovelMetadataImported
{
Guid ImportId { get; }
uint NovelId { get; }
int ChaptersPendingPull { get; }
}
public record NovelMetadataImported(Guid ImportId, uint NovelId, int ChaptersPendingPull) : INovelMetadataImported;

View File

@@ -1,6 +0,0 @@
namespace FictionArchive.Service.Shared.Contracts.Events;
public interface INovelUpdateRequested
{
string NovelUrl { get; }
}

View File

@@ -35,7 +35,7 @@ public class ChapterCreatedConsumer : IConsumer<IChapterCreated>
var volumeExists = await _dbContext.Volumes.AnyAsync(v => v.Id == message.VolumeId);
if (!volumeExists)
{
var volume = new Volume { Id = message.VolumeId };
var volume = new Volume { Id = message.VolumeId, NovelId = message.NovelId };
_dbContext.Volumes.Add(volume);
}
@@ -47,7 +47,7 @@ public class ChapterCreatedConsumer : IConsumer<IChapterCreated>
return;
}
var chapter = new Chapter { Id = message.ChapterId };
var chapter = new Chapter { Id = message.ChapterId, VolumeId = message.VolumeId };
_dbContext.Chapters.Add(chapter);
await _dbContext.SaveChangesAsync();

View File

@@ -82,6 +82,7 @@ export type ChapterDtoFilterInput = {
export type ChapterPullRequested = {
chapterOrder: Scalars['UnsignedInt']['output'];
importId: Scalars['UUID']['output'];
novelId: Scalars['UnsignedInt']['output'];
volumeId: Scalars['UnsignedInt']['output'];
};
@@ -163,6 +164,7 @@ export type Error = {
export type FetchChapterContentsInput = {
chapterOrder: Scalars['UnsignedInt']['input'];
importId: Scalars['UUID']['input'];
novelId: Scalars['UnsignedInt']['input'];
volumeId: Scalars['UnsignedInt']['input'];
};
@@ -203,7 +205,7 @@ export type ImportNovelInput = {
};
export type ImportNovelPayload = {
novelUpdateRequested: Maybe<NovelUpdateRequested>;
novelImportRequested: Maybe<NovelImportRequested>;
};
export type InstantFilterInput = {
@@ -461,6 +463,11 @@ export type NovelDtoSortInput = {
url?: InputMaybe<SortEnumType>;
};
export type NovelImportRequested = {
importId: Scalars['UUID']['output'];
novelUrl: Scalars['String']['output'];
};
export const NovelStatus = {
Abandoned: 'ABANDONED',
Completed: 'COMPLETED',
@@ -499,10 +506,6 @@ export type NovelTagDtoFilterInput = {
tagType?: InputMaybe<TagTypeOperationFilterInput>;
};
export type NovelUpdateRequested = {
novelUrl: Scalars['String']['output'];
};
/** A connection to a list of items. */
export type NovelsConnection = {
/** A list of edges. */
@@ -1010,7 +1013,7 @@ export type ImportNovelMutationVariables = Exact<{
}>;
export type ImportNovelMutation = { importNovel: { novelUpdateRequested: { novelUrl: string } | null } };
export type ImportNovelMutation = { importNovel: { novelImportRequested: { importId: any, novelUrl: string } | null } };
export type InviteUserMutationVariables = Exact<{
input: InviteUserInput;
@@ -1114,7 +1117,7 @@ export const AddToReadingListDocument = {"kind":"Document","definitions":[{"kind
export const CreateReadingListDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateReadingList"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateReadingListInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createReadingList"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"readingListPayload"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}},{"kind":"Field","name":{"kind":"Name","value":"readingList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"itemCount"}},{"kind":"Field","name":{"kind":"Name","value":"createdTime"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Error"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<CreateReadingListMutation, CreateReadingListMutationVariables>;
export const DeleteNovelDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteNovel"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"DeleteNovelInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteNovel"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"boolean"}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Error"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<DeleteNovelMutation, DeleteNovelMutationVariables>;
export const DeleteReadingListDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteReadingList"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"DeleteReadingListInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteReadingList"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Error"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<DeleteReadingListMutation, DeleteReadingListMutationVariables>;
export const ImportNovelDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ImportNovel"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ImportNovelInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"importNovel"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"novelUpdateRequested"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"novelUrl"}}]}}]}}]}}]} as unknown as DocumentNode<ImportNovelMutation, ImportNovelMutationVariables>;
export const ImportNovelDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ImportNovel"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ImportNovelInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"importNovel"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"novelImportRequested"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"importId"}},{"kind":"Field","name":{"kind":"Name","value":"novelUrl"}}]}}]}}]}}]} as unknown as DocumentNode<ImportNovelMutation, ImportNovelMutationVariables>;
export const InviteUserDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"InviteUser"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"InviteUserInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"inviteUser"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userDto"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"username"}},{"kind":"Field","name":{"kind":"Name","value":"email"}}]}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"InvalidOperationError"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<InviteUserMutation, InviteUserMutationVariables>;
export const RemoveBookmarkDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"RemoveBookmark"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"RemoveBookmarkInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"removeBookmark"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"bookmarkPayload"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}}]}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Error"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<RemoveBookmarkMutation, RemoveBookmarkMutationVariables>;
export const RemoveFromReadingListDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"RemoveFromReadingList"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"RemoveFromReadingListInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"removeFromReadingList"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"readingListPayload"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}},{"kind":"Field","name":{"kind":"Name","value":"readingList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"itemCount"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"errors"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Error"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]}}]}}]} as unknown as DocumentNode<RemoveFromReadingListMutation, RemoveFromReadingListMutationVariables>;

View File

@@ -1,6 +1,7 @@
mutation ImportNovel($input: ImportNovelInput!) {
importNovel(input: $input) {
novelUpdateRequested {
novelImportRequested {
importId
novelUrl
}
}