feature/FA-11_CICD #33

Merged
conco merged 27 commits from feature/FA-11_CICD into master 2025-11-26 23:39:45 +00:00
15 changed files with 1441 additions and 11 deletions
Showing only changes of commit caa36648e2 - Show all commits

View File

@@ -0,0 +1,122 @@
name: Build Gateway
on:
workflow_dispatch:
push:
branches:
- master
paths:
- 'FictionArchive.API/**'
env:
REGISTRY: ${{ gitea.server_url }}
IMAGE_NAME: ${{ gitea.repository_owner }}/fictionarchive-api
jobs:
build-gateway:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
- name: Install Fusion CLI
run: dotnet tool install -g HotChocolate.Fusion.CommandLine
- name: Create subgraphs directory
run: mkdir -p subgraphs
# Download all subgraph packages from latest successful builds
- name: Download Novel Service subgraph
uses: actions/download-artifact@v4
with:
name: novel-service-subgraph
path: subgraphs/novel
continue-on-error: true
- name: Download Translation Service subgraph
uses: actions/download-artifact@v4
with:
name: translation-service-subgraph
path: subgraphs/translation
continue-on-error: true
- name: Download Scheduler Service subgraph
uses: actions/download-artifact@v4
with:
name: scheduler-service-subgraph
path: subgraphs/scheduler
continue-on-error: true
- name: Download User Service subgraph
uses: actions/download-artifact@v4
with:
name: user-service-subgraph
path: subgraphs/user
continue-on-error: true
- name: Download File Service subgraph
uses: actions/download-artifact@v4
with:
name: file-service-subgraph
path: subgraphs/file
continue-on-error: true
- name: Configure subgraph URLs for Docker
run: |
for fsp in subgraphs/*/*.fsp; do
if [ -f "$fsp" ]; then
dir=$(dirname "$fsp")
name=$(basename "$dir")
url="http://${name}-service:8080/graphql"
echo "Setting $name URL to $url"
fusion subgraph config set http --url "$url" -c "$fsp"
fi
done
- name: Compose gateway
run: |
cd FictionArchive.API
rm -f gateway.fgp
for fsp in ../subgraphs/*/*.fsp; do
if [ -f "$fsp" ]; then
echo "Composing: $fsp"
fusion compose -p gateway.fgp -s "$fsp"
fi
done
- name: Restore dependencies
run: dotnet restore FictionArchive.API/FictionArchive.API.csproj
- name: Build gateway
run: dotnet build FictionArchive.API/FictionArchive.API.csproj -c Release --no-restore -p:SkipFusionBuild=true
- name: Run tests
run: dotnet test FictionArchive.sln -c Release --no-build --verbosity normal
continue-on-error: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Gitea Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ gitea.actor }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
file: FictionArchive.API/Dockerfile
push: true
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ gitea.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max

View File

@@ -0,0 +1,77 @@
name: Build Subgraphs
on:
push:
branches:
- master
paths:
- 'FictionArchive.Service.*/**'
- 'FictionArchive.Common/**'
- 'FictionArchive.Service.Shared/**'
jobs:
build-subgraphs:
runs-on: ubuntu-latest
strategy:
matrix:
service:
- name: novel-service
project: FictionArchive.Service.NovelService
subgraph: Novel
- name: translation-service
project: FictionArchive.Service.TranslationService
subgraph: Translation
- name: scheduler-service
project: FictionArchive.Service.SchedulerService
subgraph: Scheduler
- name: user-service
project: FictionArchive.Service.UserService
subgraph: User
- name: file-service
project: FictionArchive.Service.FileService
subgraph: File
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
- name: Install Fusion CLI
run: dotnet tool install -g HotChocolate.Fusion.CommandLine
- name: Restore dependencies
run: dotnet restore ${{ matrix.service.project }}/${{ matrix.service.project }}.csproj
- name: Build
run: dotnet build ${{ matrix.service.project }}/${{ matrix.service.project }}.csproj -c Release --no-restore
- name: Export schema
run: |
dotnet run --project ${{ matrix.service.project }}/${{ matrix.service.project }}.csproj \
--no-build -c Release --no-launch-profile \
-- schema export --output ${{ matrix.service.project }}/schema.graphql
- name: Pack subgraph
run: fusion subgraph pack -w ${{ matrix.service.project }}
- name: Upload subgraph package
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.service.name }}-subgraph
path: ${{ matrix.service.project }}/*.fsp
retention-days: 30
# Trigger gateway build after all subgraphs are built
trigger-gateway:
runs-on: ubuntu-latest
needs: build-subgraphs
steps:
- name: Trigger gateway workflow
run: |
curl -X POST \
-H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
"${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}/actions/workflows/build-gateway.yml/dispatches" \
-d '{"ref":"master"}'

View File

@@ -0,0 +1,63 @@
name: CI
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build-backend:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install Fusion CLI
run: dotnet tool install -g HotChocolate.Fusion.CommandLine
- name: Restore dependencies
run: dotnet restore FictionArchive.sln
- name: Build solution
run: dotnet build FictionArchive.sln --configuration Release --no-restore
- name: Run tests
run: dotnet test FictionArchive.sln --configuration Release --no-build --verbosity normal
build-frontend:
runs-on: ubuntu-latest
defaults:
run:
working-directory: fictionarchive-web
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: fictionarchive-web/package-lock.json
- name: Install dependencies
run: npm ci
- name: Lint
run: npm run lint
- name: Build
run: npm run build

View File

@@ -0,0 +1,98 @@
name: Release
on:
push:
tags:
- 'v*.*.*'
env:
REGISTRY: ${{ gitea.server_url }}
IMAGE_PREFIX: ${{ gitea.repository_owner }}/fictionarchive
jobs:
build-and-push:
runs-on: ubuntu-latest
strategy:
matrix:
service:
- name: api
dockerfile: FictionArchive.API/Dockerfile
- name: novel-service
dockerfile: FictionArchive.Service.NovelService/Dockerfile
- name: user-service
dockerfile: FictionArchive.Service.UserService/Dockerfile
- name: translation-service
dockerfile: FictionArchive.Service.TranslationService/Dockerfile
- name: file-service
dockerfile: FictionArchive.Service.FileService/Dockerfile
- name: scheduler-service
dockerfile: FictionArchive.Service.SchedulerService/Dockerfile
- name: authentication-service
dockerfile: FictionArchive.Service.AuthenticationService/Dockerfile
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Extract version from tag
id: version
run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
- name: Log in to Gitea Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ gitea.actor }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
file: ${{ matrix.service.dockerfile }}
push: true
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:${{ steps.version.outputs.VERSION }}
${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:latest
cache-from: type=gha
cache-to: type=gha,mode=max
build-frontend:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Extract version from tag
id: version
run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
- name: Log in to Gitea Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ gitea.actor }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Build and push frontend Docker image
uses: docker/build-push-action@v6
with:
context: ./fictionarchive-web
file: fictionarchive-web/Dockerfile
push: true
build-args: |
VITE_GRAPHQL_URI=${{ vars.VITE_GRAPHQL_URI }}
VITE_OIDC_AUTHORITY=${{ vars.VITE_OIDC_AUTHORITY }}
VITE_OIDC_CLIENT_ID=${{ vars.VITE_OIDC_CLIENT_ID }}
VITE_OIDC_REDIRECT_URI=${{ vars.VITE_OIDC_REDIRECT_URI }}
VITE_OIDC_POST_LOGOUT_REDIRECT_URI=${{ vars.VITE_OIDC_POST_LOGOUT_REDIRECT_URI }}
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-frontend:${{ steps.version.outputs.VERSION }}
${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-frontend:latest
cache-from: type=gha
cache-to: type=gha,mode=max

View File

@@ -0,0 +1,405 @@
# FictionArchive Architecture Overview
## High-Level Architecture
```
┌────────────────────────────────────────────────────────────────┐
│ React 19 Frontend │
│ (Apollo Client, TailwindCSS, OIDC Auth) │
└───────────────────────────┬────────────────────────────────────┘
│ GraphQL
┌────────────────────────────────────────────────────────────────┐
│ Hot Chocolate Fusion Gateway │
│ (FictionArchive.API) │
└──────┬────────┬────────┬────────┬────────┬─────────────────────┘
│ │ │ │ │
▼ ▼ ▼ ▼ ▼
┌──────────┐┌──────────┐┌───────────┐┌──────────┐┌──────────────┐
│ Novel ││ User ││Translation││Scheduler ││ File │
│ Service ││ Service ││ Service ││ Service ││ Service │
└────┬─────┘└────┬─────┘└─────┬─────┘└────┬─────┘└──────┬───────┘
│ │ │ │ │
└───────────┴────────────┴───────────┴─────────────┘
┌────────┴────────┐
│ RabbitMQ │
│ (Event Bus) │
└─────────────────┘
┌────────┴────────┐
│ PostgreSQL │
│ (per service) │
└─────────────────┘
```
## Technology Stack
| Layer | Technology | Version |
|-------|------------|---------|
| Runtime | .NET | 8.0 |
| GraphQL | Hot Chocolate / Fusion | 13+ |
| Database | PostgreSQL | 12+ |
| ORM | Entity Framework Core | 8.0 |
| Message Broker | RabbitMQ | 3.12+ |
| Job Scheduler | Quartz.NET | Latest |
| Object Storage | AWS S3 / Garage | - |
| Date/Time | NodaTime | Latest |
| Frontend | React | 19.2 |
| Frontend Build | Vite | 7.2 |
| GraphQL Client | Apollo Client | 4.0 |
| Auth | OIDC Client TS | 3.4 |
| Styling | TailwindCSS | 3.4 |
| UI Components | Radix UI | Latest |
## Project Structure
```
FictionArchive.sln
├── FictionArchive.Common # Shared enums and extensions
├── FictionArchive.API # GraphQL Fusion Gateway
├── FictionArchive.Service.Shared # Shared infrastructure
├── FictionArchive.Service.NovelService
├── FictionArchive.Service.UserService
├── FictionArchive.Service.TranslationService
├── FictionArchive.Service.FileService
├── FictionArchive.Service.SchedulerService
├── FictionArchive.Service.AuthenticationService
├── FictionArchive.Service.NovelService.Tests
└── fictionarchive-web # React frontend
```
## Services
### FictionArchive.API - GraphQL Fusion Gateway
- **Role**: Single entry point for all GraphQL queries
- **Port**: 5001 (HTTPS)
- **Endpoints**:
- `/graphql` - GraphQL endpoint
- `/healthz` - Health check
- **Responsibilities**:
- Compose GraphQL schemas from all subgraphs
- Route queries to appropriate services
- CORS policy management
### FictionArchive.Service.NovelService
- **Role**: Novel/fiction content management
- **Port**: 8081 (HTTPS)
- **Database**: `FictionArchive_NovelService`
- **GraphQL Operations**:
- `GetNovels` - Paginated, filterable novel listing
- `ImportNovel` - Trigger novel import
- `FetchChapterContents` - Fetch chapter content
- **Models**: Novel, Chapter, Source, NovelTag, Image, LocalizationKey
- **External Integration**: Novelpia adapter
- **Events Published**: `TranslationRequestCreatedEvent`, `FileUploadRequestCreatedEvent`
- **Events Subscribed**: `TranslationRequestCompletedEvent`, `NovelUpdateRequestedEvent`, `ChapterPullRequestedEvent`, `FileUploadRequestStatusUpdateEvent`
### FictionArchive.Service.UserService
- **Role**: User identity and profile management
- **Port**: 8081 (HTTPS)
- **Database**: `FictionArchive_UserService`
- **Models**: User (with OAuth provider linking)
- **Events Subscribed**: `AuthUserAddedEvent`
### FictionArchive.Service.TranslationService
- **Role**: Text translation orchestration
- **Port**: 8081 (HTTPS)
- **Database**: `FictionArchive_TranslationService`
- **External Integration**: DeepL API
- **Models**: TranslationRequest
- **Events Published**: `TranslationRequestCompletedEvent`
- **Events Subscribed**: `TranslationRequestCreatedEvent`
### FictionArchive.Service.FileService
- **Role**: File storage and S3 proxy
- **Port**: 8080 (HTTP)
- **Protocol**: REST only (not GraphQL)
- **Endpoints**: `GET /api/{*path}` - S3 file proxy
- **External Integration**: S3-compatible storage (AWS S3 / Garage)
- **Events Published**: `FileUploadRequestStatusUpdateEvent`
- **Events Subscribed**: `FileUploadRequestCreatedEvent`
### FictionArchive.Service.SchedulerService
- **Role**: Job scheduling and automation
- **Port**: 8081 (HTTPS)
- **Database**: `FictionArchive_SchedulerService`
- **Scheduler**: Quartz.NET with persistent job store
- **GraphQL Operations**: `ScheduleEventJob`, `GetScheduledJobs`
- **Models**: SchedulerJob, EventJobTemplate
### FictionArchive.Service.AuthenticationService
- **Role**: OAuth/OIDC webhook receiver
- **Port**: 8080 (HTTP)
- **Protocol**: REST only
- **Endpoints**: `POST /api/AuthenticationWebhook/UserRegistered`
- **Events Published**: `AuthUserAddedEvent`
- **No Database** - Stateless webhook handler
## Communication Patterns
### GraphQL Federation
- Hot Chocolate Fusion Gateway composes subgraph schemas
- Schema export automated via `build_gateway.py`
- Each service defines its own Query/Mutation types
### Event-Driven Architecture (RabbitMQ)
- Direct exchange: `fiction-archive-event-bus`
- Per-service queues based on `ClientIdentifier`
- Routing key = event class name
- Headers: `X-Created-At`, `X-Event-Id`
- NodaTime JSON serialization
### Event Flow Examples
**Novel Import:**
```
1. Frontend → importNovel mutation
2. NovelService publishes NovelUpdateRequestedEvent
3. NovelUpdateRequestedEventHandler processes
4. Fetches metadata via NovelpiaAdapter
5. Publishes FileUploadRequestCreatedEvent (for cover)
6. FileService uploads to S3
7. FileService publishes FileUploadRequestStatusUpdateEvent
8. NovelService updates image path
```
**Translation:**
```
1. NovelService publishes TranslationRequestCreatedEvent
2. TranslationService translates via DeepL
3. TranslationService publishes TranslationRequestCompletedEvent
4. NovelService updates chapter translation
```
## Data Storage
### Database Pattern
- Database per service (PostgreSQL)
- Connection string format: `Host=localhost;Database=FictionArchive_{ServiceName};...`
- Auto-migration on startup via `dbContext.UpdateDatabase()`
### Audit Trail
- `AuditInterceptor` auto-sets `CreatedTime` and `LastUpdatedTime`
- `IAuditable` interface with NodaTime `Instant` fields
- `BaseEntity<TKey>` abstract base class
### Object Storage
- S3-compatible (AWS S3 or Garage)
- Path-style URLs for Garage compatibility
- Proxied through FileService
## Frontend Architecture
### Structure
```
fictionarchive-web/
├── src/
│ ├── auth/ # OIDC authentication
│ ├── components/ # React components
│ │ └── ui/ # Radix-based primitives
│ ├── pages/ # Route pages
│ ├── layouts/ # Layout components
│ ├── graphql/ # GraphQL queries
│ ├── __generated__/ # Codegen output
│ └── lib/ # Utilities
└── codegen.ts # GraphQL Codegen config
```
### Authentication
- OIDC via `oidc-client-ts`
- Environment variables for configuration
- `useAuth` hook for state access
### State Management
- Apollo Client for GraphQL state
- React Context for auth state
## Infrastructure
### Docker
- Multi-stage builds
- Base: `mcr.microsoft.com/dotnet/aspnet:8.0`
- Non-root user for security
- Ports: 8080 (HTTP) or 8081 (HTTPS)
### Health Checks
- All services expose `/healthz`
### Configuration
- `appsettings.json` - Default settings
- `appsettings.Development.json` - Dev overrides
- `appsettings.Local.json` - Local secrets (not committed)
---
# Improvement Recommendations
## Critical
### 1. Event Bus - No Dead Letter Queue or Retry Logic
**Location**: `FictionArchive.Service.Shared/Services/EventBus/Implementations/RabbitMQEventBus.cs:126-133`
**Issue**: Events are always ACK'd even on failure. No DLQ configuration for poison messages. Failed events are lost forever.
**Recommendation**: Implement retry with exponential backoff, dead-letter exchange, and poison message handling.
```csharp
// Example: Add retry and DLQ
catch (Exception e)
{
_logger.LogError(e, "Error handling event");
if (retryCount < maxRetries)
{
await channel.BasicNackAsync(@event.DeliveryTag, false, true); // requeue
}
else
{
// Send to DLQ
await channel.BasicNackAsync(@event.DeliveryTag, false, false);
}
}
```
### 2. CORS Configuration is Insecure
**Location**: `FictionArchive.API/Program.cs:24-33`
**Issue**: `AllowAnyOrigin()` allows requests from any domain, unsuitable for production.
**Recommendation**: Configure specific allowed origins via appsettings:
```csharp
builder.Services.AddCors(options =>
{
options.AddPolicy("Production", policy =>
{
policy.WithOrigins(builder.Configuration.GetSection("Cors:AllowedOrigins").Get<string[]>())
.AllowAnyMethod()
.AllowAnyHeader();
});
});
```
### 3. Auto-Migration on Startup
**Location**: `FictionArchive.Service.Shared/Services/Database/FictionArchiveDbContext.cs:23-38`
**Issue**: Running migrations at startup can cause race conditions with multiple instances and potential data corruption during rolling deployments.
**Recommendation**: Use a migration job, init container, or CLI tool instead of startup code.
## Important
### 4. No Circuit Breaker Pattern
**Issue**: External service calls (DeepL, Novelpia, S3) lack resilience patterns.
**Recommendation**: Add Polly for circuit breaker, retry, and timeout policies:
```csharp
builder.Services.AddHttpClient<ISourceAdapter, NovelpiaAdapter>()
.AddPolicyHandler(GetRetryPolicy())
.AddPolicyHandler(GetCircuitBreakerPolicy());
```
### 5. Missing Request Validation/Rate Limiting
**Issue**: No visible rate limiting on GraphQL mutations. `ImportNovel` could be abused.
**Recommendation**: Add rate limiting middleware and input validation.
### 6. Hardcoded Exchange Name
**Location**: `RabbitMQEventBus.cs:24`
**Issue**: `fiction-archive-event-bus` is hardcoded.
**Recommendation**: Move to configuration for environment flexibility.
### 7. No Distributed Tracing
**Issue**: Event correlation exists (`X-Event-Id` header) but not integrated with tracing.
**Recommendation**: Add OpenTelemetry for end-to-end request tracing across services.
### 8. Singleton AuditInterceptor
**Location**: `FictionArchiveDbContext.cs:20`
**Issue**: `new AuditInterceptor()` created per DbContext instance.
**Recommendation**: Register as singleton in DI and inject.
## Minor / Code Quality
### 9. Limited Test Coverage
**Issue**: Only `NovelService.Tests` exists. No integration tests for event handlers.
**Recommendation**: Add unit and integration tests for each service, especially event handlers.
### 10. Inconsistent Port Configuration
**Issue**: Some services use 8080 (HTTP), others 8081 (HTTPS).
**Recommendation**: Standardize on HTTPS with proper cert management.
### 11. No API Versioning
**Issue**: GraphQL schemas have no versioning strategy.
**Recommendation**: Consider schema versioning or deprecation annotations for breaking changes.
### 12. Frontend - No Error Boundary
**Issue**: React app lacks error boundaries for graceful failure handling.
**Recommendation**: Add React Error Boundaries around routes.
### 13. Missing Health Check Depth
**Issue**: Health checks only verify service is running, not dependencies.
**Recommendation**: Add database, RabbitMQ, and S3 health checks:
```csharp
builder.Services.AddHealthChecks()
.AddNpgSql(connectionString)
.AddRabbitMQ()
.AddS3(options => { });
```
### 14. Synchronous File Operations in Event Handlers
**Issue**: File uploads may block event handling thread for large files.
**Recommendation**: Consider async streaming for large files.
## Architectural Suggestions
### 15. Consider Outbox Pattern
**Issue**: Publishing events and saving to DB aren't transactional, could lead to inconsistent state.
**Recommendation**: Implement transactional outbox pattern for guaranteed delivery:
```
1. Save entity + outbox message in same transaction
2. Background worker publishes from outbox
3. Delete outbox message after successful publish
```
### 16. Gateway Schema Build Process
**Issue**: Python script (`build_gateway.py`) for schema composition requires manual execution.
**Recommendation**: Integrate into CI/CD pipeline or consider runtime schema polling.
### 17. Secret Management
**Issue**: Credentials in appsettings files.
**Recommendation**: Use Azure Key Vault, AWS Secrets Manager, HashiCorp Vault, or similar secret management solution.
---
## Key Files Reference
| File | Purpose |
|------|---------|
| `FictionArchive.API/Program.cs` | Gateway setup |
| `FictionArchive.API/build_gateway.py` | Schema composition script |
| `FictionArchive.Service.Shared/Services/EventBus/` | Event bus implementation |
| `FictionArchive.Service.Shared/Extensions/` | Service registration helpers |
| `FictionArchive.Service.Shared/Services/Database/` | DB infrastructure |
| `fictionarchive-web/src/auth/AuthContext.tsx` | Frontend auth state |

220
Documentation/CICD.md Normal file
View File

@@ -0,0 +1,220 @@
# CI/CD Configuration
This document describes the CI/CD pipeline configuration for FictionArchive using Gitea Actions.
## Workflows Overview
| Workflow | File | Trigger | Purpose |
|----------|------|---------|---------|
| CI | `build.yml` | Push/PR to master | Build and test all projects |
| Build Subgraphs | `build-subgraphs.yml` | Push to master (service changes) | Build GraphQL subgraph packages |
| Build Gateway | `build-gateway.yml` | Manual or triggered by subgraphs | Compose gateway and build Docker image |
| Release | `release.yml` | Tag `v*.*.*` | Build and push all Docker images |
## Pipeline Architecture
```
┌─────────────────────────────────────────────────────────────────────┐
│ Push to master │
└─────────────────────────────┬───────────────────────────────────────┘
┌───────────────┴───────────────┐
▼ ▼
┌─────────────────────────┐ ┌─────────────────────────┐
│ build.yml │ │ build-subgraphs.yml │
│ (CI checks - always) │ │ (if service changes) │
└─────────────────────────┘ └────────────┬────────────┘
┌─────────────────────────┐
│ build-gateway.yml │
│ (compose & push API) │
└─────────────────────────┘
┌─────────────────────────────────────────────────────────────────────┐
│ Push tag v*.*.* │
└─────────────────────────────┬───────────────────────────────────────┘
┌─────────────────────────┐
│ release.yml │
│ (build & push all) │
└─────────────────────────┘
```
## Required Configuration
### Repository Secrets
Configure these in **Settings → Actions → Secrets**:
| Secret | Description | Required By |
|--------|-------------|-------------|
| `REGISTRY_TOKEN` | Gitea access token with `write:package` scope | `release.yml`, `build-gateway.yml` |
| `GITEA_TOKEN` | Gitea access token for API calls | `build-subgraphs.yml` |
#### Creating Access Tokens
1. Go to **Settings → Applications → Access Tokens**
2. Create a new token with the following scopes:
- `write:package` - Push container images
- `write:repository` - Trigger workflows via API
3. Copy the token and add it as a repository secret
### Repository Variables
Configure these in **Settings → Actions → Variables**:
| Variable | Description | Example | Required By |
|----------|-------------|---------|-------------|
| `VITE_GRAPHQL_URI` | GraphQL API endpoint URL | `https://api.fictionarchive.example.com/graphql/` | `release.yml` |
| `VITE_OIDC_AUTHORITY` | OIDC provider authority URL | `https://auth.example.com/application/o/fiction-archive/` | `release.yml` |
| `VITE_OIDC_CLIENT_ID` | OIDC client identifier | `your-client-id` | `release.yml` |
| `VITE_OIDC_REDIRECT_URI` | Post-login redirect URL | `https://fictionarchive.example.com/` | `release.yml` |
| `VITE_OIDC_POST_LOGOUT_REDIRECT_URI` | Post-logout redirect URL | `https://fictionarchive.example.com/` | `release.yml` |
## Workflow Details
### CI (`build.yml`)
**Trigger:** Push or pull request to `master`
**Jobs:**
1. `build-backend` - Builds .NET solution and runs tests
2. `build-frontend` - Builds React application with linting
**Requirements:**
- .NET 8.0 SDK
- Python 3.12
- Node.js 20
- HotChocolate Fusion CLI
### Build Subgraphs (`build-subgraphs.yml`)
**Trigger:** Push to `master` with changes in:
- `FictionArchive.Service.*/**`
- `FictionArchive.Common/**`
- `FictionArchive.Service.Shared/**`
**Jobs:**
1. `build-subgraphs` - Matrix job building each service's `.fsp` package
2. `trigger-gateway` - Triggers gateway rebuild via API
**Subgraphs Built:**
- Novel Service
- Translation Service
- Scheduler Service
- User Service
- File Service
**Artifacts:** Each subgraph produces a `.fsp` file retained for 30 days.
### Build Gateway (`build-gateway.yml`)
**Trigger:**
- Manual dispatch (`workflow_dispatch`)
- Push to `master` with changes in `FictionArchive.API/**`
- Triggered by `build-subgraphs.yml` completion
**Process:**
1. Downloads all subgraph `.fsp` artifacts
2. Configures Docker-internal URLs for each subgraph
3. Composes gateway schema using Fusion CLI
4. Builds and pushes API Docker image
**Image Tags:**
- `<registry>/<owner>/fictionarchive-api:latest`
- `<registry>/<owner>/fictionarchive-api:<commit-sha>`
### Release (`release.yml`)
**Trigger:** Push tag matching `v*.*.*` (e.g., `v1.0.0`)
**Jobs:**
1. `build-and-push` - Matrix job building all backend service images
2. `build-frontend` - Builds and pushes frontend image
**Services Built:**
- `fictionarchive-api`
- `fictionarchive-novel-service`
- `fictionarchive-user-service`
- `fictionarchive-translation-service`
- `fictionarchive-file-service`
- `fictionarchive-scheduler-service`
- `fictionarchive-authentication-service`
- `fictionarchive-frontend`
**Image Tags:**
- `<registry>/<owner>/fictionarchive-<service>:<version>`
- `<registry>/<owner>/fictionarchive-<service>:latest`
## Container Registry
Images are pushed to the Gitea Container Registry at:
```
<gitea-server-url>/<repository-owner>/fictionarchive-<service>:<tag>
```
### Pulling Images
```bash
# Login to registry
docker login <gitea-server-url> -u <username> -p <token>
# Pull an image
docker pull <gitea-server-url>/<owner>/fictionarchive-api:latest
```
## Creating a Release
1. Ensure all changes are committed and pushed to `master`
2. Create and push a version tag:
```bash
git tag v1.0.0
git push origin v1.0.0
```
3. The release workflow will automatically build and push all images
4. Monitor progress in **Actions** tab
## Troubleshooting
### Build Failures
**"REGISTRY_TOKEN secret not found"**
- Ensure the `REGISTRY_TOKEN` secret is configured in repository settings
- Verify the token has `write:package` scope
**"Failed to trigger gateway workflow"**
- Ensure `GITEA_TOKEN` secret is configured
- Verify the token has `write:repository` scope
**"No subgraph artifacts found"**
- The gateway build requires subgraph artifacts from a previous `build-subgraphs` run
- Trigger `build-subgraphs.yml` manually or push a change to a service
### Frontend Build Failures
**"VITE_* variables are empty"**
- Ensure all required variables are configured in repository settings
- Variables use `vars.*` context, not `secrets.*`
### Docker Push Failures
**"unauthorized: authentication required"**
- Verify `REGISTRY_TOKEN` has correct permissions
- Check that the token hasn't expired
## Local Testing
To test workflows locally before pushing:
```bash
# Install act (GitHub Actions local runner)
# Note: act has partial Gitea Actions compatibility
# Run CI workflow
act push -W .gitea/workflows/build.yml
# Run with specific event
act push --eventpath .gitea/test-event.json
```

187
Documentation/README.md Normal file
View File

@@ -0,0 +1,187 @@
# FictionArchive
A distributed microservices-based web application for managing fiction and novel content. Features include importing from external sources, multi-language translation, file storage, and user management.
## Architecture
FictionArchive uses a GraphQL Fusion gateway pattern to orchestrate multiple domain services with event-driven communication via RabbitMQ.
More information available in [ARCHITECTURE.md](ARCHITECTURE.md)
## Prerequisites
- .NET SDK 8.0+
- Node.js 20+
- Python 3 (for gateway build script)
- Docker & Docker Compose
- PostgreSQL 16+
- RabbitMQ 3+
**Required CLI Tools**
```bash
# Hot Chocolate Fusion CLI
dotnet tool install -g HotChocolate.Fusion.CommandLine
```
## Getting Started
### Local Development
1. **Clone the repository**
```bash
git clone <repository-url>
cd FictionArchive
```
2. **Start infrastructure** (PostgreSQL, RabbitMQ)
```bash
docker compose up -d postgres rabbitmq
```
3. **Build and run backend**
```bash
dotnet restore
dotnet build FictionArchive.sln
# Start services (in separate terminals or use a process manager)
dotnet run --project FictionArchive.Service.NovelService
dotnet run --project FictionArchive.Service.UserService
dotnet run --project FictionArchive.Service.TranslationService
dotnet run --project FictionArchive.Service.FileService
dotnet run --project FictionArchive.Service.SchedulerService
dotnet run --project FictionArchive.Service.AuthenticationService
# Start the gateway (builds fusion schema automatically)
dotnet run --project FictionArchive.API
```
4. **Build and run frontend**
```bash
cd fictionarchive-web
npm install
npm run codegen # Generate GraphQL types
npm run dev # Start dev server at http://localhost:5173
```
### Docker Deployment
1. **Create environment file**
```bash
cp .env.example .env
# Edit .env with your configuration
```
2. **Start all services**
```bash
docker compose up -d
```
## Configuration
### Environment Variables
Create a `.env` file in the project root:
```bash
# PostgreSQL
POSTGRES_USER=postgres
POSTGRES_PASSWORD=your-secure-password
# RabbitMQ
RABBITMQ_USER=guest
RABBITMQ_PASSWORD=your-secure-password
# External Services
NOVELPIA_USERNAME=your-username
NOVELPIA_PASSWORD=your-password
DEEPL_API_KEY=your-api-key
# S3 Storage
S3_ENDPOINT=https://s3.example.com
S3_BUCKET=fictionarchive
S3_ACCESS_KEY=your-access-key
S3_SECRET_KEY=your-secret-key
# OIDC Authentication
OIDC_AUTHORITY=https://auth.example.com/application/o/fiction-archive/
OIDC_CLIENT_ID=your-client-id
```
### Frontend Environment
Create `fictionarchive-web/.env.local`:
```bash
VITE_GRAPHQL_URI=http://localhost:5234/graphql/
VITE_OIDC_AUTHORITY=https://auth.example.com/application/o/fiction-archive/
VITE_OIDC_CLIENT_ID=your-client-id
VITE_OIDC_REDIRECT_URI=http://localhost:5173/
VITE_OIDC_POST_LOGOUT_REDIRECT_URI=http://localhost:5173/
```
## Building the GraphQL Gateway
The API gateway uses Hot Chocolate Fusion to compose schemas from all subgraphs. The gateway schema is rebuilt automatically when building the API project.
**Manual rebuild:**
```bash
cd FictionArchive.API
python build_gateway.py
```
**Skip specific services** by adding them to `FictionArchive.API/gateway_skip.txt`:
```
FictionArchive.Service.NovelService.Tests
```
## CI/CD
The project uses Gitea Actions with the following workflows:
| Workflow | Trigger | Description |
|----------|---------|-------------|
| `build.yml` | Push/PR to master | CI checks - builds and tests |
| `build-subgraphs.yml` | Service changes on master | Builds subgraph `.fsp` packages |
| `build-gateway.yml` | Gateway changes or subgraph builds | Composes gateway and builds Docker image |
| `release.yml` | Tag `v*.*.*` | Builds and pushes all Docker images |
### Release Process
```bash
git tag v1.0.0
git push origin v1.0.0
```
## Project Structure
```
FictionArchive/
├── FictionArchive.sln
├── FictionArchive.Common/ # Shared enums and extensions
├── FictionArchive.Service.Shared/ # Shared infrastructure (EventBus, DB)
├── FictionArchive.API/ # GraphQL Fusion Gateway
├── FictionArchive.Service.NovelService/
├── FictionArchive.Service.UserService/
├── FictionArchive.Service.TranslationService/
├── FictionArchive.Service.FileService/
├── FictionArchive.Service.SchedulerService/
├── FictionArchive.Service.AuthenticationService/
├── FictionArchive.Service.NovelService.Tests/
├── fictionarchive-web/ # React frontend
├── docker-compose.yml
└── .gitea/workflows/ # CI/CD workflows
```
## Testing
```bash
# Run all tests
dotnet test FictionArchive.sln
# Run specific test project
dotnet test FictionArchive.Service.NovelService.Tests
```
## Documentation
- [ARCHITECTURE.md](ARCHITECTURE.md) - Detailed architecture documentation
- [AGENTS.md](AGENTS.md) - Development guidelines and coding standards

View File

@@ -7,15 +7,23 @@ EXPOSE 8081
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
ARG BUILD_CONFIGURATION=Release
WORKDIR /src
COPY ["FictionArchive.API/FictionArchive.API.csproj", "FictionArchive.API/"]
COPY ["FictionArchive.Common/FictionArchive.Common.csproj", "FictionArchive.Common/"]
COPY ["FictionArchive.Service.Shared/FictionArchive.Service.Shared.csproj", "FictionArchive.Service.Shared/"]
RUN dotnet restore "FictionArchive.API/FictionArchive.API.csproj"
COPY . .
COPY FictionArchive.API/ FictionArchive.API/
COPY FictionArchive.Common/ FictionArchive.Common/
COPY FictionArchive.Service.Shared/ FictionArchive.Service.Shared/
WORKDIR "/src/FictionArchive.API"
RUN dotnet build "./FictionArchive.API.csproj" -c $BUILD_CONFIGURATION -o /app/build
# Skip fusion build - gateway.fgp should be pre-composed in CI
RUN dotnet build "./FictionArchive.API.csproj" -c $BUILD_CONFIGURATION -o /app/build -p:SkipFusionBuild=true
FROM build AS publish
ARG BUILD_CONFIGURATION=Release
RUN dotnet publish "./FictionArchive.API.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
RUN dotnet publish "./FictionArchive.API.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false /p:SkipFusionBuild=true
FROM base AS final
WORKDIR /app

View File

@@ -22,9 +22,9 @@
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
</ItemGroup>
<!-- Builds the Fusion graph file before building the application itself -->
<Target Name="RunFusionBuild" BeforeTargets="BeforeBuild">
<Exec Command="python build_gateway.py" WorkingDirectory="$(ProjectDir)" />
<!-- Builds the Fusion graph file before building the application itself (skipped in CI) -->
<Target Name="RunFusionBuild" BeforeTargets="BeforeBuild" Condition="'$(SkipFusionBuild)' != 'true'">
<Exec Command="python build_gateway.py $(FusionBuildArgs)" WorkingDirectory="$(ProjectDir)" />
</Target>
<ItemGroup>

View File

@@ -1,12 +1,22 @@
#!/usr/bin/env python3
"""
Local development script for building the Fusion gateway.
This script is used for local development only. In CI/CD, subgraphs are built
separately and the gateway is composed from pre-built .fsp artifacts.
Usage:
python build_gateway.py
Requirements:
- .NET 8.0 SDK
- HotChocolate Fusion CLI (dotnet tool install -g HotChocolate.Fusion.CommandLine)
"""
import subprocess
import sys
import os
from pathlib import Path
# ----------------------------------------
# Helpers
# ----------------------------------------
def run(cmd, cwd=None):
"""Run a command and exit on failure."""
@@ -19,7 +29,7 @@ def run(cmd, cwd=None):
def load_skip_list(skip_file: Path):
if not skip_file.exists():
print(f"WARNING: skip-projects.txt not found at {skip_file}")
print(f"WARNING: gateway_skip.txt not found at {skip_file}")
return set()
lines = skip_file.read_text().splitlines()
@@ -53,7 +63,7 @@ print("----------------------------------------")
service_dirs = [
d for d in services_dir.glob("FictionArchive.Service.*")
if d.is_dir()
if d.is_dir() and (d / "subgraph-config.json").exists()
]
selected_services = []

10
README.md Normal file
View File

@@ -0,0 +1,10 @@
# FictionArchive
A distributed microservices-based web application for managing fiction and novel content.
## Documentation
- [README](Documentation/README.md) - Getting started and project overview
- [ARCHITECTURE](Documentation/ARCHITECTURE.md) - System architecture and design
- [CICD](Documentation/CICD.md) - CI/CD pipeline configuration
- [AGENTS](Documentation/AGENTS.md) - Development guidelines and coding standards

177
docker-compose.yml Normal file
View File

@@ -0,0 +1,177 @@
services:
# ===========================================
# Infrastructure
# ===========================================
postgres:
image: postgres:16-alpine
environment:
POSTGRES_USER: ${POSTGRES_USER:-postgres}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
rabbitmq:
image: rabbitmq:3-management-alpine
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-guest}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-guest}
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "check_running"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
# ===========================================
# Backend Services
# ===========================================
novel-service:
build:
context: .
dockerfile: FictionArchive.Service.NovelService/Dockerfile
environment:
ConnectionStrings__DefaultConnection: Host=postgres;Database=FictionArchive_NovelService;Username=${POSTGRES_USER:-postgres};Password=${POSTGRES_PASSWORD:-postgres}
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
Novelpia__Username: ${NOVELPIA_USERNAME}
Novelpia__Password: ${NOVELPIA_PASSWORD}
NovelUpdateService__PendingImageUrl: https://files.fictionarchive.orfl.xyz/api/pendingupload.png
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
restart: unless-stopped
translation-service:
build:
context: .
dockerfile: FictionArchive.Service.TranslationService/Dockerfile
environment:
ConnectionStrings__DefaultConnection: Host=postgres;Database=FictionArchive_TranslationService;Username=${POSTGRES_USER:-postgres};Password=${POSTGRES_PASSWORD:-postgres}
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
DeepL__ApiKey: ${DEEPL_API_KEY}
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
restart: unless-stopped
scheduler-service:
build:
context: .
dockerfile: FictionArchive.Service.SchedulerService/Dockerfile
environment:
ConnectionStrings__DefaultConnection: Host=postgres;Database=FictionArchive_SchedulerService;Username=${POSTGRES_USER:-postgres};Password=${POSTGRES_PASSWORD:-postgres}
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
restart: unless-stopped
user-service:
build:
context: .
dockerfile: FictionArchive.Service.UserService/Dockerfile
environment:
ConnectionStrings__DefaultConnection: Host=postgres;Database=FictionArchive_UserService;Username=${POSTGRES_USER:-postgres};Password=${POSTGRES_PASSWORD:-postgres}
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
restart: unless-stopped
authentication-service:
build:
context: .
dockerfile: FictionArchive.Service.AuthenticationService/Dockerfile
environment:
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
depends_on:
rabbitmq:
condition: service_healthy
restart: unless-stopped
file-service:
build:
context: .
dockerfile: FictionArchive.Service.FileService/Dockerfile
environment:
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
S3__Endpoint: ${S3_ENDPOINT:-https://s3.orfl.xyz}
S3__Bucket: ${S3_BUCKET:-fictionarchive}
S3__AccessKey: ${S3_ACCESS_KEY}
S3__SecretKey: ${S3_SECRET_KEY}
Proxy__BaseUrl: https://files.orfl.xyz/api
labels:
- "traefik.enable=true"
- "traefik.http.routers.file-service.rule=Host(`files.orfl.xyz`)"
- "traefik.http.routers.file-service.entrypoints=websecure"
- "traefik.http.routers.file-service.tls.certresolver=letsencrypt"
- "traefik.http.services.file-service.loadbalancer.server.port=8080"
depends_on:
rabbitmq:
condition: service_healthy
restart: unless-stopped
# ===========================================
# API Gateway
# ===========================================
api-gateway:
build:
context: .
dockerfile: FictionArchive.API/Dockerfile
environment:
ConnectionStrings__RabbitMQ: amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq
labels:
- "traefik.enable=true"
- "traefik.http.routers.api-gateway.rule=Host(`api.fictionarchive.orfl.xyz`)"
- "traefik.http.routers.api-gateway.entrypoints=websecure"
- "traefik.http.routers.api-gateway.tls.certresolver=letsencrypt"
- "traefik.http.services.api-gateway.loadbalancer.server.port=8080"
depends_on:
- novel-service
- translation-service
- scheduler-service
- user-service
- authentication-service
- file-service
restart: unless-stopped
# ===========================================
# Frontend
# ===========================================
frontend:
build:
context: ./fictionarchive-web
dockerfile: Dockerfile
args:
VITE_GRAPHQL_URI: https://api.fictionarchive.orfl.xyz/graphql/
VITE_OIDC_AUTHORITY: ${OIDC_AUTHORITY:-https://auth.orfl.xyz/application/o/fiction-archive/}
VITE_OIDC_CLIENT_ID: ${OIDC_CLIENT_ID}
VITE_OIDC_REDIRECT_URI: https://fictionarchive.orfl.xyz/
VITE_OIDC_POST_LOGOUT_REDIRECT_URI: https://fictionarchive.orfl.xyz/
labels:
- "traefik.enable=true"
- "traefik.http.routers.frontend.rule=Host(`fictionarchive.orfl.xyz`)"
- "traefik.http.routers.frontend.entrypoints=websecure"
- "traefik.http.routers.frontend.tls.certresolver=letsencrypt"
- "traefik.http.services.frontend.loadbalancer.server.port=80"
restart: unless-stopped
volumes:
postgres_data:
rabbitmq_data:
letsencrypt:

View File

@@ -0,0 +1,32 @@
FROM node:20-alpine AS build
WORKDIR /app
# Build arguments for Vite environment variables
ARG VITE_GRAPHQL_URI
ARG VITE_OIDC_AUTHORITY
ARG VITE_OIDC_CLIENT_ID
ARG VITE_OIDC_REDIRECT_URI
ARG VITE_OIDC_POST_LOGOUT_REDIRECT_URI
# Set environment variables for build
ENV VITE_GRAPHQL_URI=$VITE_GRAPHQL_URI
ENV VITE_OIDC_AUTHORITY=$VITE_OIDC_AUTHORITY
ENV VITE_OIDC_CLIENT_ID=$VITE_OIDC_CLIENT_ID
ENV VITE_OIDC_REDIRECT_URI=$VITE_OIDC_REDIRECT_URI
ENV VITE_OIDC_POST_LOGOUT_REDIRECT_URI=$VITE_OIDC_POST_LOGOUT_REDIRECT_URI
COPY package*.json ./
RUN npm ci
COPY . .
RUN npm run build
FROM nginx:alpine
COPY --from=build /app/dist /usr/share/nginx/html
COPY nginx.conf /etc/nginx/conf.d/default.conf
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]

View File

@@ -0,0 +1,21 @@
server {
listen 80;
server_name localhost;
root /usr/share/nginx/html;
index index.html;
# Gzip compression
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
# Handle SPA routing - serve index.html for all routes
location / {
try_files $uri $uri/ /index.html;
}
# Cache static assets
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
expires 1y;
add_header Cache-Control "public, immutable";
}
}