[FA-misc] Saga seems to work, fixed a UserNovelDataService bug

This commit is contained in:
gamer147
2026-01-28 12:11:06 -05:00
parent 579e05b853
commit ec967770d3
34 changed files with 1341 additions and 97 deletions

View File

@@ -21,6 +21,17 @@ public class ChapterPullRequestedConsumer : IConsumer<IChapterPullRequested>
public async Task Consume(ConsumeContext<IChapterPullRequested> context)
{
var message = context.Message;
await _novelUpdateService.PullChapterContents(message.NovelId, message.VolumeId, message.ChapterOrder);
var (chapter, imageCount) = await _novelUpdateService.PullChapterContents(
message.ImportId,
message.NovelId,
message.VolumeId,
message.ChapterOrder);
await context.Publish<IChapterPullCompleted>(new ChapterPullCompleted(
message.ImportId,
chapter.Id,
imageCount
));
}
}

View File

@@ -0,0 +1,43 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelImportCompletedConsumer : IConsumer<INovelImportCompleted>
{
private readonly ILogger<NovelImportCompletedConsumer> _logger;
private readonly NovelServiceDbContext _dbContext;
public NovelImportCompletedConsumer(
ILogger<NovelImportCompletedConsumer> logger,
NovelServiceDbContext dbContext)
{
_logger = logger;
_dbContext = dbContext;
}
public async Task Consume(ConsumeContext<INovelImportCompleted> context)
{
var message = context.Message;
_logger.LogInformation(
"Novel import {ImportId} completed. Success: {Success}, NovelId: {NovelId}, Error: {Error}",
message.ImportId,
message.Success,
message.NovelId,
message.ErrorMessage);
// Remove from ActiveImports to allow future imports
var activeImport = await _dbContext.ActiveImports
.FirstOrDefaultAsync(a => a.ImportId == message.ImportId);
if (activeImport != null)
{
_dbContext.ActiveImports.Remove(activeImport);
await _dbContext.SaveChangesAsync();
}
}
}

View File

@@ -0,0 +1,29 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelImportRequestedConsumer : IConsumer<INovelImportRequested>
{
private readonly ILogger<NovelImportRequestedConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public NovelImportRequestedConsumer(
ILogger<NovelImportRequestedConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<INovelImportRequested> context)
{
var message = context.Message;
_logger.LogInformation("Starting novel import for {NovelUrl} with ImportId {ImportId}",
message.NovelUrl, message.ImportId);
await _novelUpdateService.ImportNovel(message.ImportId, message.NovelUrl);
}
}

View File

@@ -1,26 +0,0 @@
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using Microsoft.Extensions.Logging;
namespace FictionArchive.Service.NovelService.Consumers;
public class NovelUpdateRequestedConsumer : IConsumer<INovelUpdateRequested>
{
private readonly ILogger<NovelUpdateRequestedConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public NovelUpdateRequestedConsumer(
ILogger<NovelUpdateRequestedConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<INovelUpdateRequested> context)
{
var message = context.Message;
await _novelUpdateService.ImportNovel(message.NovelUrl);
}
}

View File

@@ -1,8 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record ChapterPullRequested(
uint NovelId,
uint VolumeId,
uint ChapterOrder) : IChapterPullRequested;

View File

@@ -1,8 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record FileUploadRequestCreated(
Guid RequestId,
string FilePath,
byte[] FileData) : IFileUploadRequestCreated;

View File

@@ -1,6 +0,0 @@
using FictionArchive.Service.Shared.Contracts.Events;
namespace FictionArchive.Service.NovelService.Contracts;
public record NovelUpdateRequested(
string NovelUrl) : INovelUpdateRequested;

View File

@@ -10,11 +10,12 @@
<ItemGroup>
<PackageReference Include="HotChocolate.AspNetCore.CommandLine" Version="15.1.11" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
<PackageReference Include="MassTransit.EntityFrameworkCore" Version="8.5.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.11">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
</ItemGroup>
<ItemGroup>

View File

@@ -5,6 +5,7 @@ using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Models.SourceAdapters;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.Shared.Contracts.Events;
using HotChocolate.Authorization;
using HotChocolate.Types;
using Microsoft.EntityFrameworkCore;
@@ -13,20 +14,22 @@ namespace FictionArchive.Service.NovelService.GraphQL;
public class Mutation
{
[Error<InvalidOperationException>]
[Authorize]
public async Task<NovelUpdateRequested> ImportNovel(string novelUrl, NovelUpdateService service)
public async Task<NovelImportRequested> ImportNovel(string novelUrl, NovelUpdateService service)
{
return await service.QueueNovelImport(novelUrl);
}
[Authorize]
public async Task<ChapterPullRequested> FetchChapterContents(
Guid importId,
uint novelId,
uint volumeId,
uint chapterOrder,
NovelUpdateService service)
{
return await service.QueueChapterPull(novelId, volumeId, chapterOrder);
return await service.QueueChapterPull(importId, novelId, volumeId, chapterOrder);
}
[Error<KeyNotFoundException>]

View File

@@ -0,0 +1,673 @@
// <auto-generated />
using System;
using FictionArchive.Service.NovelService.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FictionArchive.Service.NovelService.Migrations
{
[DbContext(typeof(NovelServiceDbContext))]
[Migration("20260127161500_AddNovelImportSaga")]
partial class AddNovelImportSaga
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.11")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.ActiveImport", b =>
{
b.Property<Guid>("ImportId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("ImportId");
b.HasIndex("NovelUrl")
.IsUnique();
b.ToTable("ActiveImports");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<long?>("ChapterId")
.HasColumnType("bigint");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("NewPath")
.HasColumnType("text");
b.Property<string>("OriginalPath")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("ChapterId");
b.ToTable("Images");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("LocalizationKeys");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<long>("EngineId")
.HasColumnType("bigint");
b.Property<Guid>("KeyRequestedForTranslationId")
.HasColumnType("uuid");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<int>("TranslateTo")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("EngineId");
b.HasIndex("KeyRequestedForTranslationId");
b.ToTable("LocalizationRequests");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<int>("Language")
.HasColumnType("integer");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid?>("LocalizationKeyId")
.HasColumnType("uuid");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("text");
b.Property<long?>("TranslationEngineId")
.HasColumnType("bigint");
b.HasKey("Id");
b.HasIndex("LocalizationKeyId");
b.HasIndex("TranslationEngineId");
b.ToTable("LocalizationText");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Guid>("BodyId")
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<long>("Order")
.HasColumnType("bigint");
b.Property<long>("Revision")
.HasColumnType("bigint");
b.Property<string>("Url")
.HasColumnType("text");
b.Property<long>("VolumeId")
.HasColumnType("bigint");
b.HasKey("Id");
b.HasIndex("BodyId");
b.HasIndex("NameId");
b.HasIndex("VolumeId", "Order")
.IsUnique();
b.ToTable("Chapter");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<long>("AuthorId")
.HasColumnType("bigint");
b.Property<Guid?>("CoverImageId")
.HasColumnType("uuid");
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("DescriptionId")
.HasColumnType("uuid");
b.Property<string>("ExternalId")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<int>("RawLanguage")
.HasColumnType("integer");
b.Property<int>("RawStatus")
.HasColumnType("integer");
b.Property<long>("SourceId")
.HasColumnType("bigint");
b.Property<int?>("StatusOverride")
.HasColumnType("integer");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AuthorId");
b.HasIndex("CoverImageId");
b.HasIndex("DescriptionId");
b.HasIndex("NameId");
b.HasIndex("SourceId");
b.HasIndex("ExternalId", "SourceId")
.IsUnique();
b.ToTable("Novels");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("DisplayNameId")
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<long?>("SourceId")
.HasColumnType("bigint");
b.Property<int>("TagType")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("DisplayNameId");
b.HasIndex("SourceId");
b.ToTable("Tags");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("ExternalUrl")
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("NameId");
b.ToTable("Person");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Source", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Sources");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("TranslationEngines");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<Instant>("CreatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Instant>("LastUpdatedTime")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("NameId")
.HasColumnType("uuid");
b.Property<long>("NovelId")
.HasColumnType("bigint");
b.Property<int>("Order")
.HasColumnType("integer");
b.HasKey("Id");
b.HasIndex("NameId");
b.HasIndex("NovelId", "Order")
.IsUnique();
b.ToTable("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Sagas.NovelImportSagaState", b =>
{
b.Property<Guid>("CorrelationId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone");
b.Property<int>("CompletedChapters")
.HasColumnType("integer");
b.Property<int>("CompletedImages")
.HasColumnType("integer");
b.Property<string>("CurrentState")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ErrorMessage")
.HasColumnType("text");
b.Property<int>("ExpectedChapters")
.HasColumnType("integer");
b.Property<int>("ExpectedImages")
.HasColumnType("integer");
b.Property<long?>("NovelId")
.HasColumnType("bigint");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("CorrelationId");
b.HasIndex("CurrentState");
b.HasIndex("NovelUrl");
b.ToTable("NovelImportSagaStates");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.Property<long>("NovelsId")
.HasColumnType("bigint");
b.Property<long>("TagsId")
.HasColumnType("bigint");
b.HasKey("NovelsId", "TagsId");
b.HasIndex("TagsId");
b.ToTable("NovelNovelTag");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Chapter", "Chapter")
.WithMany("Images")
.HasForeignKey("ChapterId");
b.Navigation("Chapter");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationRequest", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "Engine")
.WithMany()
.HasForeignKey("EngineId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "KeyRequestedForTranslation")
.WithMany()
.HasForeignKey("KeyRequestedForTranslationId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Engine");
b.Navigation("KeyRequestedForTranslation");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationText", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", null)
.WithMany("Texts")
.HasForeignKey("LocalizationKeyId");
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.TranslationEngine", "TranslationEngine")
.WithMany()
.HasForeignKey("TranslationEngineId");
b.Navigation("TranslationEngine");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Body")
.WithMany()
.HasForeignKey("BodyId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Volume", "Volume")
.WithMany("Chapters")
.HasForeignKey("VolumeId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Body");
b.Navigation("Name");
b.Navigation("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Person", "Author")
.WithMany()
.HasForeignKey("AuthorId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Images.Image", "CoverImage")
.WithMany()
.HasForeignKey("CoverImageId");
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Description")
.WithMany()
.HasForeignKey("DescriptionId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source")
.WithMany()
.HasForeignKey("SourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Author");
b.Navigation("CoverImage");
b.Navigation("Description");
b.Navigation("Name");
b.Navigation("Source");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.NovelTag", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "DisplayName")
.WithMany()
.HasForeignKey("DisplayNameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Source", "Source")
.WithMany()
.HasForeignKey("SourceId");
b.Navigation("DisplayName");
b.Navigation("Source");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Person", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Name");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", "Name")
.WithMany()
.HasForeignKey("NameId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", "Novel")
.WithMany("Volumes")
.HasForeignKey("NovelId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Name");
b.Navigation("Novel");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.Novel", null)
.WithMany()
.HasForeignKey("NovelsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FictionArchive.Service.NovelService.Models.Novels.NovelTag", null)
.WithMany()
.HasForeignKey("TagsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Localization.LocalizationKey", b =>
{
b.Navigation("Texts");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Chapter", b =>
{
b.Navigation("Images");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Novel", b =>
{
b.Navigation("Volumes");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Novels.Volume", b =>
{
b.Navigation("Chapters");
});
#pragma warning restore 612, 618
}
}
}

View File

@@ -0,0 +1,76 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace FictionArchive.Service.NovelService.Migrations
{
/// <inheritdoc />
public partial class AddNovelImportSaga : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "ActiveImports",
columns: table => new
{
ImportId = table.Column<Guid>(type: "uuid", nullable: false),
NovelUrl = table.Column<string>(type: "text", nullable: false),
StartedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_ActiveImports", x => x.ImportId);
});
migrationBuilder.CreateTable(
name: "NovelImportSagaStates",
columns: table => new
{
CorrelationId = table.Column<Guid>(type: "uuid", nullable: false),
CurrentState = table.Column<string>(type: "text", nullable: false),
NovelUrl = table.Column<string>(type: "text", nullable: false),
NovelId = table.Column<long>(type: "bigint", nullable: true),
ExpectedChapters = table.Column<int>(type: "integer", nullable: false),
CompletedChapters = table.Column<int>(type: "integer", nullable: false),
ExpectedImages = table.Column<int>(type: "integer", nullable: false),
CompletedImages = table.Column<int>(type: "integer", nullable: false),
StartedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
CompletedAt = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
ErrorMessage = table.Column<string>(type: "text", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_NovelImportSagaStates", x => x.CorrelationId);
});
migrationBuilder.CreateIndex(
name: "IX_ActiveImports_NovelUrl",
table: "ActiveImports",
column: "NovelUrl",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_NovelImportSagaStates_CurrentState",
table: "NovelImportSagaStates",
column: "CurrentState");
migrationBuilder.CreateIndex(
name: "IX_NovelImportSagaStates_NovelUrl",
table: "NovelImportSagaStates",
column: "NovelUrl");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "ActiveImports");
migrationBuilder.DropTable(
name: "NovelImportSagaStates");
}
}
}

View File

@@ -23,6 +23,27 @@ namespace FictionArchive.Service.NovelService.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.ActiveImport", b =>
{
b.Property<Guid>("ImportId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("ImportId");
b.HasIndex("NovelUrl")
.IsUnique();
b.ToTable("ActiveImports");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Models.Images.Image", b =>
{
b.Property<Guid>("Id")
@@ -391,6 +412,53 @@ namespace FictionArchive.Service.NovelService.Migrations
b.ToTable("Volume");
});
modelBuilder.Entity("FictionArchive.Service.NovelService.Sagas.NovelImportSagaState", b =>
{
b.Property<Guid>("CorrelationId")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone");
b.Property<int>("CompletedChapters")
.HasColumnType("integer");
b.Property<int>("CompletedImages")
.HasColumnType("integer");
b.Property<string>("CurrentState")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ErrorMessage")
.HasColumnType("text");
b.Property<int>("ExpectedChapters")
.HasColumnType("integer");
b.Property<int>("ExpectedImages")
.HasColumnType("integer");
b.Property<long?>("NovelId")
.HasColumnType("bigint");
b.Property<string>("NovelUrl")
.IsRequired()
.HasColumnType("text");
b.Property<Instant>("StartedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("CorrelationId");
b.HasIndex("CurrentState");
b.HasIndex("NovelUrl");
b.ToTable("NovelImportSagaStates");
});
modelBuilder.Entity("NovelNovelTag", b =>
{
b.Property<long>("NovelsId")

View File

@@ -0,0 +1,10 @@
using NodaTime;
namespace FictionArchive.Service.NovelService.Models;
public class ActiveImport
{
public Guid ImportId { get; set; }
public required string NovelUrl { get; set; }
public Instant StartedAt { get; set; }
}

View File

@@ -2,13 +2,16 @@ using FictionArchive.Common.Extensions;
using FictionArchive.Service.NovelService.Consumers;
using FictionArchive.Service.NovelService.GraphQL;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Sagas;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.NovelService.Services.SourceAdapters.Novelpia;
using FictionArchive.Service.Shared;
using FictionArchive.Service.Shared.Extensions;
using FictionArchive.Service.Shared.Services.GraphQL;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace FictionArchive.Service.NovelService;
@@ -33,8 +36,17 @@ public class Program
{
x.AddConsumer<TranslationRequestCompletedConsumer>();
x.AddConsumer<FileUploadRequestStatusUpdateConsumer>();
x.AddConsumer<NovelUpdateRequestedConsumer>();
x.AddConsumer<ChapterPullRequestedConsumer>();
x.AddConsumer<NovelImportRequestedConsumer>();
x.AddConsumer<NovelImportCompletedConsumer>();
x.AddSagaStateMachine<NovelImportSaga, NovelImportSagaState>()
.EntityFrameworkRepository(r =>
{
r.ConcurrencyMode = ConcurrencyMode.Optimistic;
r.ExistingDbContext<NovelServiceDbContext>();
r.UsePostgres();
});
});
}
@@ -72,9 +84,12 @@ public class Program
builder.Services.Configure<NovelUpdateServiceConfiguration>(builder.Configuration.GetSection("UpdateService"));
builder.Services.AddTransient<NovelUpdateService>();
#endregion
// Register IClock for saga and service use
builder.Services.AddSingleton<IClock>(SystemClock.Instance);
builder.Services.AddHealthChecks();
// Authentication & Authorization

View File

@@ -0,0 +1,135 @@
using FictionArchive.Service.Shared.Contracts.Events;
using MassTransit;
using NodaTime;
namespace FictionArchive.Service.NovelService.Sagas;
public class NovelImportSaga : MassTransitStateMachine<NovelImportSagaState>
{
public State Importing { get; private set; } = null!;
public State Processing { get; private set; } = null!;
public State Completed { get; private set; } = null!;
public State Failed { get; private set; } = null!;
public Event<INovelImportRequested> NovelImportRequested { get; private set; } = null!;
public Event<INovelMetadataImported> NovelMetadataImported { get; private set; } = null!;
public Event<IChapterPullCompleted> ChapterPullCompleted { get; private set; } = null!;
public Event<IFileUploadRequestStatusUpdate> FileUploadStatusUpdate { get; private set; } = null!;
public Event<Fault<IChapterPullRequested>> ChapterPullFaulted { get; private set; } = null!;
public Event<Fault<IFileUploadRequestCreated>> FileUploadFaulted { get; private set; } = null!;
private readonly IClock _clock;
public NovelImportSaga(IClock clock)
{
_clock = clock;
InstanceState(x => x.CurrentState);
Event(() => NovelImportRequested, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => NovelMetadataImported, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => ChapterPullCompleted, x => x.CorrelateById(ctx => ctx.Message.ImportId));
Event(() => FileUploadStatusUpdate, x =>
{
x.CorrelateById(ctx => ctx.Message.ImportId ?? Guid.Empty);
x.OnMissingInstance(m => m.Discard());
});
Event(() => ChapterPullFaulted, x => x.CorrelateById(ctx => ctx.Message.Message.ImportId));
Event(() => FileUploadFaulted, x =>
{
x.CorrelateById(ctx => ctx.Message.Message.ImportId ?? Guid.Empty);
x.OnMissingInstance(m => m.Discard());
});
Initially(
When(NovelImportRequested)
.Then(ctx =>
{
ctx.Saga.NovelUrl = ctx.Message.NovelUrl;
ctx.Saga.StartedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Importing)
);
During(Importing,
When(NovelMetadataImported)
.Then(ctx =>
{
ctx.Saga.NovelId = ctx.Message.NovelId;
ctx.Saga.ExpectedChapters = ctx.Message.ChaptersPendingPull;
})
.IfElse(
ctx => ctx.Saga.ExpectedChapters == 0,
thenBinder => thenBinder
.Then(ctx => ctx.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
true,
null))),
elseBinder => elseBinder.TransitionTo(Processing)
)
);
During(Processing,
When(ChapterPullCompleted)
.Then(ctx =>
{
ctx.Saga.CompletedChapters++;
ctx.Saga.ExpectedImages += ctx.Message.ImagesQueued;
})
.If(ctx => IsComplete(ctx.Saga), ctx => ctx
.Then(c => c.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(c => c.Init<INovelImportCompleted>(new NovelImportCompleted(
c.Saga.CorrelationId,
c.Saga.NovelId,
true,
null)))),
When(FileUploadStatusUpdate)
.Then(ctx => ctx.Saga.CompletedImages++)
.If(ctx => IsComplete(ctx.Saga), ctx => ctx
.Then(c => c.Saga.CompletedAt = _clock.GetCurrentInstant())
.TransitionTo(Completed)
.PublishAsync(c => c.Init<INovelImportCompleted>(new NovelImportCompleted(
c.Saga.CorrelationId,
c.Saga.NovelId,
true,
null)))),
When(ChapterPullFaulted)
.Then(ctx =>
{
ctx.Saga.ErrorMessage = ctx.Message.Exceptions.FirstOrDefault()?.Message;
ctx.Saga.CompletedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Failed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
false,
ctx.Saga.ErrorMessage))),
When(FileUploadFaulted)
.Then(ctx =>
{
ctx.Saga.ErrorMessage = ctx.Message.Exceptions.FirstOrDefault()?.Message;
ctx.Saga.CompletedAt = _clock.GetCurrentInstant();
})
.TransitionTo(Failed)
.PublishAsync(ctx => ctx.Init<INovelImportCompleted>(new NovelImportCompleted(
ctx.Saga.CorrelationId,
ctx.Saga.NovelId,
false,
ctx.Saga.ErrorMessage)))
);
SetCompletedWhenFinalized();
}
private static bool IsComplete(NovelImportSagaState saga) =>
saga.CompletedChapters >= saga.ExpectedChapters &&
saga.CompletedImages >= saga.ExpectedImages;
}

View File

@@ -0,0 +1,29 @@
using MassTransit;
using NodaTime;
namespace FictionArchive.Service.NovelService.Sagas;
public class NovelImportSagaState : SagaStateMachineInstance
{
public Guid CorrelationId { get; set; }
public string CurrentState { get; set; } = null!;
// Identity
public string NovelUrl { get; set; } = null!;
public uint? NovelId { get; set; }
// Chapter tracking
public int ExpectedChapters { get; set; }
public int CompletedChapters { get; set; }
// Image tracking
public int ExpectedImages { get; set; }
public int CompletedImages { get; set; }
// Timestamps
public Instant StartedAt { get; set; }
public Instant? CompletedAt { get; set; }
// Error info
public string? ErrorMessage { get; set; }
}

View File

@@ -1,6 +1,8 @@
using FictionArchive.Service.NovelService.Models;
using FictionArchive.Service.NovelService.Models.Images;
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Sagas;
using FictionArchive.Service.Shared.Services.Database;
using Microsoft.EntityFrameworkCore;
@@ -18,6 +20,8 @@ public class NovelServiceDbContext(DbContextOptions options, ILogger<NovelServic
public DbSet<LocalizationKey> LocalizationKeys { get; set; }
public DbSet<LocalizationRequest> LocalizationRequests { get; set; }
public DbSet<Image> Images { get; set; }
public DbSet<ActiveImport> ActiveImports { get; set; }
public DbSet<NovelImportSagaState> NovelImportSagaStates { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
@@ -36,5 +40,18 @@ public class NovelServiceDbContext(DbContextOptions options, ILogger<NovelServic
modelBuilder.Entity<Chapter>()
.HasIndex("VolumeId", "Order")
.IsUnique();
modelBuilder.Entity<ActiveImport>(entity =>
{
entity.HasKey(e => e.ImportId);
entity.HasIndex(e => e.NovelUrl).IsUnique();
});
modelBuilder.Entity<NovelImportSagaState>(entity =>
{
entity.HasKey(e => e.CorrelationId);
entity.HasIndex(e => e.NovelUrl);
entity.HasIndex(e => e.CurrentState);
});
}
}

View File

@@ -1,5 +1,6 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.NovelService.Contracts;
using FictionArchive.Service.NovelService.Models;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Models.Enums;
using FictionArchive.Service.NovelService.Models.Images;
@@ -12,6 +13,7 @@ using HtmlAgilityPack;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using NodaTime;
namespace FictionArchive.Service.NovelService.Services;
@@ -22,14 +24,16 @@ public class NovelUpdateService
private readonly IEnumerable<ISourceAdapter> _sourceAdapters;
private readonly IPublishEndpoint _publishEndpoint;
private readonly NovelUpdateServiceConfiguration _novelUpdateServiceConfiguration;
private readonly IClock _clock;
public NovelUpdateService(NovelServiceDbContext dbContext, ILogger<NovelUpdateService> logger, IEnumerable<ISourceAdapter> sourceAdapters, IPublishEndpoint publishEndpoint, IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration)
public NovelUpdateService(NovelServiceDbContext dbContext, ILogger<NovelUpdateService> logger, IEnumerable<ISourceAdapter> sourceAdapters, IPublishEndpoint publishEndpoint, IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration, IClock clock)
{
_dbContext = dbContext;
_logger = logger;
_sourceAdapters = sourceAdapters;
_publishEndpoint = publishEndpoint;
_novelUpdateServiceConfiguration = novelUpdateServiceConfiguration.Value;
_clock = clock;
}
#region Helper Methods
@@ -299,7 +303,7 @@ public class NovelUpdateService
#endregion
public async Task<Novel> ImportNovel(string novelUrl)
public async Task<Novel> ImportNovel(Guid importId, string novelUrl)
{
// Step 1: Get metadata from source adapter
NovelMetadata? metadata = null;
@@ -417,10 +421,24 @@ public class NovelUpdateService
}
}
// Count chapters that need pulling
var chaptersNeedingPull = novel.Volumes
.SelectMany(v => v.Chapters)
.Where(c => c.Body?.Texts == null || !c.Body.Texts.Any())
.ToList();
// Publish metadata imported event for saga
await _publishEndpoint.Publish<INovelMetadataImported>(new NovelMetadataImported(
importId,
novel.Id,
chaptersNeedingPull.Count
));
// Publish cover image event if needed
if (shouldPublishCoverEvent && novel.CoverImage != null && metadata.CoverImage != null)
{
await _publishEndpoint.Publish<IFileUploadRequestCreated>(new FileUploadRequestCreated(
importId,
novel.CoverImage.Id,
$"Novels/{novel.Id}/Images/cover.jpg",
metadata.CoverImage.Data));
@@ -429,13 +447,14 @@ public class NovelUpdateService
// Publish chapter pull events for chapters without body content
foreach (var volume in novel.Volumes)
{
var chaptersNeedingPull = volume.Chapters
var volumeChaptersNeedingPull = volume.Chapters
.Where(c => c.Body?.Texts == null || !c.Body.Texts.Any())
.ToList();
foreach (var chapter in chaptersNeedingPull)
foreach (var chapter in volumeChaptersNeedingPull)
{
await _publishEndpoint.Publish<IChapterPullRequested>(new ChapterPullRequested(
importId,
novel.Id,
volume.Id,
chapter.Order));
@@ -445,7 +464,7 @@ public class NovelUpdateService
return novel;
}
public async Task<Chapter> PullChapterContents(uint novelId, uint volumeId, uint chapterOrder)
public async Task<(Chapter chapter, int imageCount)> PullChapterContents(Guid importId, uint novelId, uint volumeId, uint chapterOrder)
{
var novel = await _dbContext.Novels.Where(novel => novel.Id == novelId)
.Include(novel => novel.Volumes)
@@ -512,12 +531,13 @@ public class NovelUpdateService
{
var data = rawChapter.ImageData.FirstOrDefault(img => img.Url == image.OriginalPath);
await _publishEndpoint.Publish<IFileUploadRequestCreated>(new FileUploadRequestCreated(
importId,
image.Id,
$"Novels/{novel.Id}/Images/Chapter-{chapter.Id}/{imgCount++}.jpg",
data.Data));
}
return chapter;
return (chapter, chapter.Images.Count);
}
public async Task UpdateImage(Guid imageId, string newUrl)
@@ -548,16 +568,34 @@ public class NovelUpdateService
await _dbContext.SaveChangesAsync();
}
public async Task<NovelUpdateRequested> QueueNovelImport(string novelUrl)
public async Task<NovelImportRequested> QueueNovelImport(string novelUrl)
{
var importNovelRequestEvent = new NovelUpdateRequested(novelUrl);
await _publishEndpoint.Publish<INovelUpdateRequested>(importNovelRequestEvent);
var importId = Guid.NewGuid();
var activeImport = new ActiveImport
{
ImportId = importId,
NovelUrl = novelUrl,
StartedAt = _clock.GetCurrentInstant()
};
try
{
await _dbContext.ActiveImports.AddAsync(activeImport);
await _dbContext.SaveChangesAsync();
}
catch (DbUpdateException)
{
throw new InvalidOperationException($"An import is already in progress for {novelUrl}");
}
var importNovelRequestEvent = new NovelImportRequested(importId, novelUrl);
await _publishEndpoint.Publish<INovelImportRequested>(importNovelRequestEvent);
return importNovelRequestEvent;
}
public async Task<ChapterPullRequested> QueueChapterPull(uint novelId, uint volumeId, uint chapterOrder)
public async Task<ChapterPullRequested> QueueChapterPull(Guid importId, uint novelId, uint volumeId, uint chapterOrder)
{
var chapterPullEvent = new ChapterPullRequested(novelId, volumeId, chapterOrder);
var chapterPullEvent = new ChapterPullRequested(importId, novelId, volumeId, chapterOrder);
await _publishEndpoint.Publish<IChapterPullRequested>(chapterPullEvent);
return chapterPullEvent;
}