[FA-misc] Mass transit overhaul, needs testing and review

This commit is contained in:
gamer147
2026-01-21 23:16:31 -05:00
parent 055ef33666
commit f88f340d0a
97 changed files with 1150 additions and 858 deletions

View File

@@ -1,27 +1,20 @@
using FictionArchive.Service.NovelService.Models.Enums;
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Models.SourceAdapters;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.Shared.Services.EventBus;
using FictionArchive.Service.Shared.MassTransit.Contracts.Commands;
using HotChocolate.Authorization;
using HotChocolate.Types;
using Microsoft.EntityFrameworkCore;
namespace FictionArchive.Service.NovelService.GraphQL;
public class Mutation
{
[Authorize]
public async Task<NovelUpdateRequestedEvent> ImportNovel(string novelUrl, NovelUpdateService service)
public async Task<ImportNovelCommand> ImportNovel(string novelUrl, NovelUpdateService service)
{
return await service.QueueNovelImport(novelUrl);
}
[Authorize]
public async Task<ChapterPullRequestedEvent> FetchChapterContents(
public async Task<PullChapterContentCommand> FetchChapterContents(
uint novelId,
uint volumeId,
uint chapterOrder,

View File

@@ -1,13 +0,0 @@
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class ChapterCreatedEvent : IIntegrationEvent
{
public required uint ChapterId { get; init; }
public required uint NovelId { get; init; }
public required uint VolumeId { get; init; }
public required int VolumeOrder { get; init; }
public required uint ChapterOrder { get; init; }
public required string ChapterTitle { get; init; }
}

View File

@@ -1,10 +0,0 @@
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class ChapterPullRequestedEvent : IIntegrationEvent
{
public uint NovelId { get; set; }
public uint VolumeId { get; set; }
public uint ChapterOrder { get; set; }
}

View File

@@ -1,10 +0,0 @@
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.FileService.IntegrationEvents;
public class FileUploadRequestCreatedEvent : IIntegrationEvent
{
public Guid RequestId { get; set; }
public string FilePath { get; set; }
public byte[] FileData { get; set; }
}

View File

@@ -1,22 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class FileUploadRequestStatusUpdateEvent : IIntegrationEvent
{
public Guid RequestId { get; set; }
public RequestStatus Status { get; set; }
#region Success
public string? FileAccessUrl { get; set; }
#endregion
#region Failure
public string? ErrorMessage { get; set; }
#endregion
}

View File

@@ -1,13 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class NovelCreatedEvent : IIntegrationEvent
{
public required uint NovelId { get; init; }
public required string Title { get; init; }
public required Language OriginalLanguage { get; init; }
public required string Source { get; init; }
public required string AuthorName { get; init; }
}

View File

@@ -1,8 +0,0 @@
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class NovelUpdateRequestedEvent : IIntegrationEvent
{
public string NovelUrl { get; set; }
}

View File

@@ -1,17 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class TranslationRequestCompletedEvent : IIntegrationEvent
{
/// <summary>
/// Maps this event back to a triggering request.
/// </summary>
public Guid? TranslationRequestId { get; set; }
/// <summary>
/// The resulting text.
/// </summary>
public string? TranslatedText { get; set; }
}

View File

@@ -1,13 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Models.IntegrationEvents;
public class TranslationRequestCreatedEvent : IIntegrationEvent
{
public Guid TranslationRequestId { get; set; }
public Language From { get; set; }
public Language To { get; set; }
public string Body { get; set; }
public string TranslationEngineKey { get; set; }
}

View File

@@ -1,14 +1,13 @@
using FictionArchive.Common.Extensions;
using FictionArchive.Service.NovelService.GraphQL;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.NovelService.Services;
using FictionArchive.Service.NovelService.Services.EventHandlers;
using FictionArchive.Service.NovelService.Services.Consumers;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.NovelService.Services.SourceAdapters.Novelpia;
using FictionArchive.Service.Shared;
using FictionArchive.Service.Shared.Extensions;
using FictionArchive.Service.Shared.Services.EventBus.Implementations;
using FictionArchive.Service.Shared.MassTransit;
using FictionArchive.Service.Shared.Services.GraphQL;
using Microsoft.EntityFrameworkCore;
@@ -25,18 +24,19 @@ public class Program
builder.Services.AddMemoryCache();
#region Event Bus
#region MassTransit
if (!isSchemaExport)
{
builder.Services.AddRabbitMQ(opt =>
{
builder.Configuration.GetSection("RabbitMQ").Bind(opt);
})
.Subscribe<TranslationRequestCompletedEvent, TranslationRequestCompletedEventHandler>()
.Subscribe<NovelUpdateRequestedEvent, NovelUpdateRequestedEventHandler>()
.Subscribe<ChapterPullRequestedEvent, ChapterPullRequestedEventHandler>()
.Subscribe<FileUploadRequestStatusUpdateEvent, FileUploadRequestStatusUpdateEventHandler>();
builder.Services.AddFictionArchiveMassTransit<NovelServiceDbContext>(
builder.Configuration,
cfg =>
{
cfg.AddConsumer<ImportNovelCommandConsumer>();
cfg.AddConsumer<PullChapterContentCommandConsumer>();
cfg.AddConsumer<TranslationCompletedEventConsumer>();
cfg.AddConsumer<FileUploadCompletedEventConsumer>();
});
}
#endregion

View File

@@ -0,0 +1,53 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.Shared.MassTransit.Contracts.Events;
using MassTransit;
namespace FictionArchive.Service.NovelService.Services.Consumers;
public class FileUploadCompletedEventConsumer : IConsumer<FileUploadCompletedEvent>
{
private readonly ILogger<FileUploadCompletedEventConsumer> _logger;
private readonly NovelServiceDbContext _dbContext;
private readonly NovelUpdateService _novelUpdateService;
public FileUploadCompletedEventConsumer(
ILogger<FileUploadCompletedEventConsumer> logger,
NovelServiceDbContext dbContext,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_dbContext = dbContext;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<FileUploadCompletedEvent> context)
{
var @event = context.Message;
var image = await _dbContext.Images.FindAsync(@event.RequestId);
if (image == null)
{
// Not a request we care about.
_logger.LogDebug(
"FileUploadCompletedEvent received for unknown image: {RequestId}",
@event.RequestId);
return;
}
if (@event.Status == RequestStatus.Failed)
{
_logger.LogError(
"Image upload failed for image with id {ImageId}: {ErrorMessage}",
image.Id, @event.ErrorMessage);
return;
}
if (@event.Status == RequestStatus.Success)
{
_logger.LogInformation(
"Image upload succeeded for image with id {ImageId}",
image.Id);
await _novelUpdateService.UpdateImage(image.Id, @event.FileAccessUrl!);
}
}
}

View File

@@ -0,0 +1,26 @@
using FictionArchive.Service.Shared.MassTransit.Contracts.Commands;
using MassTransit;
namespace FictionArchive.Service.NovelService.Services.Consumers;
public class ImportNovelCommandConsumer : IConsumer<ImportNovelCommand>
{
private readonly ILogger<ImportNovelCommandConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public ImportNovelCommandConsumer(
ILogger<ImportNovelCommandConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<ImportNovelCommand> context)
{
var command = context.Message;
_logger.LogInformation("Processing ImportNovelCommand for URL: {NovelUrl}", command.NovelUrl);
await _novelUpdateService.ImportNovel(command.NovelUrl);
}
}

View File

@@ -0,0 +1,28 @@
using FictionArchive.Service.Shared.MassTransit.Contracts.Commands;
using MassTransit;
namespace FictionArchive.Service.NovelService.Services.Consumers;
public class PullChapterContentCommandConsumer : IConsumer<PullChapterContentCommand>
{
private readonly ILogger<PullChapterContentCommandConsumer> _logger;
private readonly NovelUpdateService _novelUpdateService;
public PullChapterContentCommandConsumer(
ILogger<PullChapterContentCommandConsumer> logger,
NovelUpdateService novelUpdateService)
{
_logger = logger;
_novelUpdateService = novelUpdateService;
}
public async Task Consume(ConsumeContext<PullChapterContentCommand> context)
{
var command = context.Message;
_logger.LogInformation(
"Processing PullChapterContentCommand for Novel: {NovelId}, Volume: {VolumeId}, Chapter: {ChapterOrder}",
command.NovelId, command.VolumeId, command.ChapterOrder);
await _novelUpdateService.PullChapterContents(command.NovelId, command.VolumeId, command.ChapterOrder);
}
}

View File

@@ -0,0 +1,52 @@
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.Shared.MassTransit.Contracts.Events;
using MassTransit;
using Microsoft.EntityFrameworkCore;
namespace FictionArchive.Service.NovelService.Services.Consumers;
public class TranslationCompletedEventConsumer : IConsumer<TranslationCompletedEvent>
{
private readonly ILogger<TranslationCompletedEventConsumer> _logger;
private readonly NovelServiceDbContext _dbContext;
public TranslationCompletedEventConsumer(
ILogger<TranslationCompletedEventConsumer> logger,
NovelServiceDbContext dbContext)
{
_logger = logger;
_dbContext = dbContext;
}
public async Task Consume(ConsumeContext<TranslationCompletedEvent> context)
{
var @event = context.Message;
var localizationRequest = await _dbContext.LocalizationRequests
.Include(r => r.KeyRequestedForTranslation)
.ThenInclude(lk => lk.Texts)
.FirstOrDefaultAsync(lk => lk.Id == @event.TranslationRequestId);
if (localizationRequest == null)
{
// Not one of our requests, discard it
_logger.LogDebug(
"TranslationCompletedEvent received for unknown request: {RequestId}",
@event.TranslationRequestId);
return;
}
localizationRequest.KeyRequestedForTranslation.Texts.Add(new LocalizationText()
{
Language = localizationRequest.TranslateTo,
Text = @event.TranslatedText,
TranslationEngine = localizationRequest.Engine
});
_dbContext.LocalizationRequests.Remove(localizationRequest);
await _dbContext.SaveChangesAsync();
_logger.LogInformation(
"Completed translation for request: {RequestId}",
@event.TranslationRequestId);
}
}

View File

@@ -1,19 +0,0 @@
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Services.EventHandlers;
public class ChapterPullRequestedEventHandler : IIntegrationEventHandler<ChapterPullRequestedEvent>
{
private readonly NovelUpdateService _novelUpdateService;
public ChapterPullRequestedEventHandler(NovelUpdateService novelUpdateService)
{
_novelUpdateService = novelUpdateService;
}
public async Task Handle(ChapterPullRequestedEvent @event)
{
await _novelUpdateService.PullChapterContents(@event.NovelId, @event.VolumeId, @event.ChapterOrder);
}
}

View File

@@ -1,39 +0,0 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Services.EventHandlers;
public class FileUploadRequestStatusUpdateEventHandler : IIntegrationEventHandler<FileUploadRequestStatusUpdateEvent>
{
private readonly ILogger<FileUploadRequestStatusUpdateEventHandler> _logger;
private readonly NovelServiceDbContext _context;
private readonly NovelUpdateService _novelUpdateService;
public FileUploadRequestStatusUpdateEventHandler(ILogger<FileUploadRequestStatusUpdateEventHandler> logger, NovelServiceDbContext context, NovelUpdateService novelUpdateService)
{
_logger = logger;
_context = context;
_novelUpdateService = novelUpdateService;
}
public async Task Handle(FileUploadRequestStatusUpdateEvent @event)
{
var image = await _context.Images.FindAsync(@event.RequestId);
if (image == null)
{
// Not a request we care about.
return;
}
if (@event.Status == RequestStatus.Failed)
{
_logger.LogError("Image upload failed for image with id {imageId}", image.Id);
return;
}
else if (@event.Status == RequestStatus.Success)
{
_logger.LogInformation("Image upload succeeded for image with id {imageId}", image.Id);
await _novelUpdateService.UpdateImage(image.Id, @event.FileAccessUrl);
}
}
}

View File

@@ -1,23 +0,0 @@
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.Shared.Services.EventBus;
namespace FictionArchive.Service.NovelService.Services.EventHandlers;
public class NovelUpdateRequestedEventHandler : IIntegrationEventHandler<NovelUpdateRequestedEvent>
{
private readonly ILogger<NovelUpdateRequestedEventHandler> _logger;
private readonly IEventBus _eventBus;
private readonly NovelUpdateService _novelUpdateService;
public NovelUpdateRequestedEventHandler(ILogger<NovelUpdateRequestedEventHandler> logger, IEventBus eventBus, NovelUpdateService novelUpdateService)
{
_logger = logger;
_eventBus = eventBus;
_novelUpdateService = novelUpdateService;
}
public async Task Handle(NovelUpdateRequestedEvent @event)
{
await _novelUpdateService.ImportNovel(@event.NovelUrl);
}
}

View File

@@ -1,39 +0,0 @@
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.Shared.Services.EventBus;
using Microsoft.EntityFrameworkCore;
namespace FictionArchive.Service.NovelService.Services.EventHandlers;
public class TranslationRequestCompletedEventHandler : IIntegrationEventHandler<TranslationRequestCompletedEvent>
{
private readonly ILogger<TranslationRequestCompletedEventHandler> _logger;
private readonly NovelServiceDbContext _dbContext;
public TranslationRequestCompletedEventHandler(ILogger<TranslationRequestCompletedEventHandler> logger, NovelServiceDbContext dbContext)
{
_logger = logger;
_dbContext = dbContext;
}
public async Task Handle(TranslationRequestCompletedEvent @event)
{
var localizationRequest = await _dbContext.LocalizationRequests.Include(r => r.KeyRequestedForTranslation)
.ThenInclude(lk => lk.Texts)
.FirstOrDefaultAsync(lk => lk.Id == @event.TranslationRequestId);
if (localizationRequest == null)
{
// Not one of our requests, discard it
return;
}
localizationRequest.KeyRequestedForTranslation.Texts.Add(new LocalizationText()
{
Language = localizationRequest.TranslateTo,
Text = @event.TranslatedText,
TranslationEngine = localizationRequest.Engine
});
_dbContext.LocalizationRequests.Remove(localizationRequest);
await _dbContext.SaveChangesAsync();
}
}

View File

@@ -1,15 +1,15 @@
using FictionArchive.Common.Enums;
using FictionArchive.Service.FileService.IntegrationEvents;
using FictionArchive.Service.NovelService.Models.Configuration;
using FictionArchive.Service.NovelService.Models.Enums;
using FictionArchive.Service.NovelService.Models.Images;
using FictionArchive.Service.NovelService.Models.IntegrationEvents;
using FictionArchive.Service.NovelService.Models.Localization;
using FictionArchive.Service.NovelService.Models.Novels;
using FictionArchive.Service.NovelService.Models.SourceAdapters;
using FictionArchive.Service.NovelService.Services.SourceAdapters;
using FictionArchive.Service.Shared.Services.EventBus;
using FictionArchive.Service.Shared.MassTransit.Contracts.Commands;
using FictionArchive.Service.Shared.MassTransit.Contracts.Events;
using HtmlAgilityPack;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
@@ -20,15 +20,23 @@ public class NovelUpdateService
private readonly NovelServiceDbContext _dbContext;
private readonly ILogger<NovelUpdateService> _logger;
private readonly IEnumerable<ISourceAdapter> _sourceAdapters;
private readonly IEventBus _eventBus;
private readonly IPublishEndpoint _publishEndpoint;
private readonly ISendEndpointProvider _sendEndpointProvider;
private readonly NovelUpdateServiceConfiguration _novelUpdateServiceConfiguration;
public NovelUpdateService(NovelServiceDbContext dbContext, ILogger<NovelUpdateService> logger, IEnumerable<ISourceAdapter> sourceAdapters, IEventBus eventBus, IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration)
public NovelUpdateService(
NovelServiceDbContext dbContext,
ILogger<NovelUpdateService> logger,
IEnumerable<ISourceAdapter> sourceAdapters,
IPublishEndpoint publishEndpoint,
ISendEndpointProvider sendEndpointProvider,
IOptions<NovelUpdateServiceConfiguration> novelUpdateServiceConfiguration)
{
_dbContext = dbContext;
_logger = logger;
_sourceAdapters = sourceAdapters;
_eventBus = eventBus;
_publishEndpoint = publishEndpoint;
_sendEndpointProvider = sendEndpointProvider;
_novelUpdateServiceConfiguration = novelUpdateServiceConfiguration.Value;
}
@@ -393,7 +401,7 @@ public class NovelUpdateService
// Publish novel created event for new novels
if (existingNovel == null)
{
await _eventBus.Publish(new NovelCreatedEvent
await _publishEndpoint.Publish(new NovelCreatedEvent
{
NovelId = novel.Id,
Title = novel.Name.Texts.First(t => t.Language == novel.RawLanguage).Text,
@@ -408,7 +416,7 @@ public class NovelUpdateService
{
foreach (var chapter in volume.Chapters.Where(c => !existingChapterIds.Contains(c.Id)))
{
await _eventBus.Publish(new ChapterCreatedEvent
await _publishEndpoint.Publish(new ChapterCreatedEvent
{
ChapterId = chapter.Id,
NovelId = novel.Id,
@@ -420,10 +428,11 @@ public class NovelUpdateService
}
}
// Publish cover image event if needed
// Send cover image upload command if needed
if (shouldPublishCoverEvent && novel.CoverImage != null && metadata.CoverImage != null)
{
await _eventBus.Publish(new FileUploadRequestCreatedEvent
var uploadEndpoint = await _sendEndpointProvider.GetSendEndpoint(new Uri("queue:upload-file-command"));
await uploadEndpoint.Send(new UploadFileCommand
{
RequestId = novel.CoverImage.Id,
FileData = metadata.CoverImage.Data,
@@ -431,7 +440,8 @@ public class NovelUpdateService
});
}
// Publish chapter pull events for chapters without body content
// Send chapter pull commands for chapters without body content
var pullChapterEndpoint = await _sendEndpointProvider.GetSendEndpoint(new Uri("queue:pull-chapter-content-command"));
foreach (var volume in novel.Volumes)
{
var chaptersNeedingPull = volume.Chapters
@@ -440,7 +450,7 @@ public class NovelUpdateService
foreach (var chapter in chaptersNeedingPull)
{
await _eventBus.Publish(new ChapterPullRequestedEvent
await pullChapterEndpoint.Send(new PullChapterContentCommand
{
NovelId = novel.Id,
VolumeId = volume.Id,
@@ -513,12 +523,13 @@ public class NovelUpdateService
localizationText.Text = chapterDoc.DocumentNode.OuterHtml;
await _dbContext.SaveChangesAsync();
// Body was updated, raise image request
// Body was updated, send upload commands for images
var uploadEndpoint = await _sendEndpointProvider.GetSendEndpoint(new Uri("queue:upload-file-command"));
int imgCount = 0;
foreach (var image in chapter.Images)
{
var data = rawChapter.ImageData.FirstOrDefault(img => img.Url == image.OriginalPath);
await _eventBus.Publish(new FileUploadRequestCreatedEvent()
await uploadEndpoint.Send(new UploadFileCommand
{
FileData = data.Data,
FilePath = $"{novel.Id}/Images/Chapter-{chapter.Id}/{imgCount++}.jpg",
@@ -557,26 +568,28 @@ public class NovelUpdateService
await _dbContext.SaveChangesAsync();
}
public async Task<NovelUpdateRequestedEvent> QueueNovelImport(string novelUrl)
public async Task<ImportNovelCommand> QueueNovelImport(string novelUrl)
{
var importNovelRequestEvent = new NovelUpdateRequestedEvent()
var command = new ImportNovelCommand
{
NovelUrl = novelUrl
};
await _eventBus.Publish(importNovelRequestEvent);
return importNovelRequestEvent;
var endpoint = await _sendEndpointProvider.GetSendEndpoint(new Uri("queue:import-novel-command"));
await endpoint.Send(command);
return command;
}
public async Task<ChapterPullRequestedEvent> QueueChapterPull(uint novelId, uint volumeId, uint chapterOrder)
public async Task<PullChapterContentCommand> QueueChapterPull(uint novelId, uint volumeId, uint chapterOrder)
{
var chapterPullEvent = new ChapterPullRequestedEvent()
var command = new PullChapterContentCommand
{
NovelId = novelId,
VolumeId = volumeId,
ChapterOrder = chapterOrder
};
await _eventBus.Publish(chapterPullEvent);
return chapterPullEvent;
var endpoint = await _sendEndpointProvider.GetSendEndpoint(new Uri("queue:pull-chapter-content-command"));
await endpoint.Send(command);
return command;
}
public async Task DeleteNovel(uint novelId)

View File

@@ -16,8 +16,10 @@
"DefaultConnection": "Host=localhost;Database=FictionArchive_NovelService;Username=postgres;password=postgres"
},
"RabbitMQ": {
"ConnectionString": "amqp://localhost",
"ClientIdentifier": "NovelService"
"Host": "localhost",
"VirtualHost": "/",
"Username": "guest",
"Password": "guest"
},
"AllowedHosts": "*",
"OIDC": {