Reconfigure CI pipeline for DinD
Some checks are pending
CI/CD Pipeline (Fully Isolated DinD) / Test Backend and Frontend (Fully Isolated DinD) (push) Waiting to run
CI/CD Pipeline (Fully Isolated DinD) / Build and Push Docker Images (DinD) (push) Blocked by required conditions
CI/CD Pipeline (Fully Isolated DinD) / Deploy to Production (push) Blocked by required conditions

This commit is contained in:
continuist 2025-06-29 23:17:34 -04:00
parent 36041e9202
commit ffb055922a
4 changed files with 171 additions and 432 deletions

View file

@ -1,4 +1,4 @@
name: CI/CD Pipeline (DinD)
name: CI/CD Pipeline (Fully Isolated DinD)
on:
push:
@ -11,62 +11,65 @@ env:
IMAGE_NAME: ${{ secrets.APP_NAME || 'sharenet' }}
jobs:
test-backend:
name: Test Backend (DinD)
# Job 1: Testing - All containers run inside DinD for complete isolation
test:
name: Test Backend and Frontend (Fully Isolated DinD)
runs-on: [self-hosted, dind]
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: ${{ secrets.APP_NAME || 'sharenet' }}_test
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup DinD environment
- name: Setup Containerized CI/CD Environment
run: |
# Ensure DinD is running and accessible
docker version
# Start DinD container
cd /opt/${{ secrets.APP_NAME || 'sharenet' }}
docker compose -f ci-cd-compose.yml up -d
# Configure Docker for Harbor registry
echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
# Wait for DinD to be ready
echo "Waiting for DinD container to be ready..."
timeout 60 bash -c 'until docker compose -f ci-cd-compose.yml ps | grep -q "healthy"; do sleep 2; done'
# Wait for Docker to be ready
timeout 30 bash -c 'until docker info; do sleep 1; done'
# Verify DinD is working
docker exec ci-cd-dind docker version
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
# Create testing containers inside DinD for complete isolation
echo "Creating testing containers inside DinD..."
- name: Install SQLx CLI
run: cargo install sqlx-cli --no-default-features --features postgres
# Start PostgreSQL container inside DinD
docker exec ci-cd-dind docker run -d \
--name ci-cd-postgres \
--restart unless-stopped \
-e POSTGRES_DB=${{ secrets.APP_NAME || 'sharenet' }}_test \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-p 5432:5432 \
postgres:15-alpine
- name: Cache Rust dependencies
uses: actions/cache@v3
with:
path: |
~/.cargo/registry
~/.cargo/git
backend/target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-
# Wait for PostgreSQL to be ready
echo "Waiting for PostgreSQL to be ready..."
timeout 60 bash -c 'until docker exec ci-cd-dind docker exec ci-cd-postgres pg_isready -U postgres; do sleep 1; done'
- name: Make scripts executable
run: chmod +x scripts/*.sh
# Start Rust toolchain container inside DinD
docker exec ci-cd-dind docker run -d \
--name ci-cd-rust \
--restart unless-stopped \
-v /workspace/backend:/workspace/backend \
--network host \
rust:1.75-slim \
sleep infinity
# Start Node.js toolchain container inside DinD
docker exec ci-cd-dind docker run -d \
--name ci-cd-node \
--restart unless-stopped \
-v /workspace/frontend:/workspace/frontend \
node:20-slim \
sleep infinity
- name: Install SQLx CLI in Rust container
run: |
docker exec ci-cd-rust cargo install sqlx-cli --no-default-features --features postgres
- name: Validate migration files
env:
@ -77,13 +80,13 @@ jobs:
timeout 60 bash -c 'until pg_isready -h localhost -p 5432 -U postgres; do sleep 1; done'
# Create test database if it doesn't exist
sqlx database create --database-url "$DATABASE_URL" || true
docker exec ci-cd-rust sqlx database create --database-url "$DATABASE_URL" || true
# Run initial migrations to set up the database
sqlx migrate run --database-url "$DATABASE_URL" || true
docker exec ci-cd-rust sqlx migrate run --database-url "$DATABASE_URL" || true
# Validate migration files
./scripts/validate_migrations.sh --verbose
docker exec ci-cd-rust ./scripts/validate_migrations.sh --verbose
- name: Run backend tests
working-directory: ./backend
@ -91,51 +94,35 @@ jobs:
DATABASE_URL: postgres://postgres:postgres@localhost:5432/${{ secrets.APP_NAME || 'sharenet' }}_test
run: |
# Run tests with increased parallelism for Rust
cargo test --all --jobs 4
cargo clippy --all -- -D warnings
cargo fmt --all -- --check
test-frontend:
name: Test Frontend (DinD)
runs-on: [self-hosted, dind]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup DinD environment
run: |
# Ensure DinD is running and accessible
docker version
# Configure Docker for Harbor registry
echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
# Wait for Docker to be ready
timeout 30 bash -c 'until docker info; do sleep 1; done'
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
docker exec ci-cd-rust cargo test --all --jobs 4
docker exec ci-cd-rust cargo clippy --all -- -D warnings
docker exec ci-cd-rust cargo fmt --all -- --check
- name: Install frontend dependencies
working-directory: ./frontend
run: npm ci
run: |
docker exec ci-cd-node npm ci
- name: Run frontend tests
working-directory: ./frontend
run: |
npm run lint
npm run type-check
npm run build
docker exec ci-cd-node npm run lint
docker exec ci-cd-node npm run type-check
docker exec ci-cd-node npm run build
- name: Cleanup Containerized Environment
if: always()
run: |
# Clean up all containers inside DinD
docker exec ci-cd-dind docker stop ci-cd-postgres ci-cd-rust ci-cd-node || true
docker exec ci-cd-dind docker rm ci-cd-postgres ci-cd-rust ci-cd-node || true
# Stop DinD container
cd /opt/${{ secrets.APP_NAME || 'sharenet' }}
docker compose -f ci-cd-compose.yml down
# Job 2: Building - Uses DinD for isolated image building and pushing
build-and-push:
name: Build and Push Docker Images (DinD)
needs: [test-backend, test-frontend]
needs: [test]
runs-on: [self-hosted, dind]
if: github.ref == 'refs/heads/main'
@ -145,15 +132,23 @@ jobs:
- name: Setup DinD environment
run: |
# Ensure DinD is running and accessible
docker version
# Start DinD container using our compose file
cd /opt/${{ secrets.APP_NAME || 'sharenet' }}
docker compose -f ci-cd-compose.yml up -d
# Configure Docker for Harbor registry
echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
# Wait for DinD to be ready
echo "Waiting for DinD container to be ready..."
timeout 60 bash -c 'until docker compose -f ci-cd-compose.yml ps | grep -q "healthy"; do sleep 2; done'
# Wait for Docker to be ready
timeout 30 bash -c 'until docker info; do sleep 1; done'
# Configure Docker for Harbor registry (needed for pushing images)
docker exec ci-cd-dind sh -c 'echo "{\"insecure-registries\": [\"${{ secrets.CI_HOST }}:5000\"]}" > /etc/docker/daemon.json'
docker exec ci-cd-dind sh -c 'kill -HUP 1'
# Wait for Docker daemon to reload
sleep 5
# Verify DinD is working
docker exec ci-cd-dind docker version
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@ -180,39 +175,25 @@ jobs:
cache-to: type=gha,mode=max
platforms: linux/amd64
- name: Cleanup DinD environment
if: always()
run: |
cd /opt/${{ secrets.APP_NAME || 'sharenet' }}
docker compose -f ci-cd-compose.yml down
# Job 3: Deployment - Runs directly on production runner (no DinD needed)
deploy:
name: Deploy to Production
needs: build-and-push
runs-on: [self-hosted, dind]
runs-on: [self-hosted, production]
if: github.ref == 'refs/heads/main'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup DinD environment
- name: Create environment file for deployment
run: |
# Ensure DinD is running and accessible
docker version
- name: Install SQLx CLI
run: cargo install sqlx-cli --no-default-features --features postgres
- name: Make scripts executable
run: chmod +x scripts/*.sh
- name: Deploy to production server
uses: appleboy/ssh-action@v1.0.3
with:
host: ${{ secrets.PROD_HOST }}
username: ${{ secrets.PROD_USER }}
key: ${{ secrets.PROD_SSH_KEY }}
script: |
cd /opt/${{ secrets.APP_NAME || 'sharenet' }}
# Pull latest code from repository (includes scripts and docker-compose.yml)
git pull origin main
# Create environment file for this deployment
echo "IMAGE_TAG=${{ github.sha }}" > .env
echo "REGISTRY=${{ secrets.CI_HOST }}:5000" >> .env
@ -224,15 +205,18 @@ jobs:
echo "NODE_ENV=production" >> .env
echo "RUST_LOG=info" >> .env
# Make scripts executable
chmod +x scripts/*.sh
- name: Make scripts executable
run: chmod +x scripts/*.sh
# Validate migrations before deployment
- name: Validate migrations before deployment
run: |
echo "Validating migration files before deployment..."
./scripts/validate_migrations.sh --verbose || {
echo "ERROR: Migration validation failed. Deployment aborted."
exit 1
}
- name: Deploy application
run: |
# Run deployment using the deployment script
./scripts/deploy.sh deploy

View file

@ -1132,32 +1132,26 @@ sudo journalctl -u forgejo-runner.service -f --no-pager
**Important**: This step sets up a Docker-in-Docker container that provides an isolated environment for CI/CD operations, eliminating resource contention with Harbor and simplifying cleanup.
#### 8.1 Create DinD Container
#### 8.1 Create Containerized CI/CD Environment
```bash
# Create DinD container with persistent storage
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Start DinD container using the repository's CI/CD Docker Compose file
cd /opt/APP_NAME
docker compose -f ci-cd-compose.yml up -d
# Wait for DinD to start
sleep 15
# Wait for DinD to be ready
echo "Waiting for DinD container to be ready..."
timeout 60 bash -c 'until docker compose -f ci-cd-compose.yml ps | grep -q "healthy"; do sleep 2; done'
# Test DinD connectivity
docker exec ci-cd-dind docker version
```
**What this does**:
- **Creates isolated environment**: DinD container runs its own Docker daemon
- **Creates isolated DinD environment**: Provides isolated Docker environment for all CI/CD operations
- **Persistent storage**: `ci-cd-data` volume preserves data between restarts
- **Privileged mode**: Required for Docker-in-Docker functionality
- **Auto-restart**: Container restarts automatically if it crashes
- **Docker socket access**: Allows DinD to communicate with host Docker
- **Health checks**: Ensures DinD is fully ready before proceeding
- **Configuration management**: Uses the repository's `ci-cd-compose.yml` file
#### 8.2 Configure DinD for Harbor Registry
@ -1185,43 +1179,25 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/alpine:latest
- **Tests connectivity**: Verifies DinD can pull, tag, and push images to Harbor
- **Validates setup**: Ensures the complete CI/CD pipeline will work
#### 8.3 DinD Cleanup Script and Testing
#### 8.3 DinD Environment Management
The project includes a comprehensive Docker-in-Docker cleanup script at `scripts/dind-cleanup.sh` that provides a simple way to clean up the DinD environment by restarting the DinD container for a fresh environment.
The DinD container is managed as an isolated environment where all CI/CD operations run inside the DinD container, providing complete isolation from the host system.
**Key Features:**
- **🧹 Complete Environment Reset**: Stops, removes, and recreates the `ci-cd-dind` container
- **📊 Status Monitoring**: Shows current DinD container status, Docker info, images, and containers
- **🚨 Dry Run Mode**: Use `--dry-run` to see what would be done without making changes
- **🎨 Colored Output**: Clear, color-coded logging for better readability
- **🛡️ Error Handling**: Robust error checking and graceful failure handling
**Key Benefits:**
- **🧹 Complete Isolation**: All testing and building runs inside DinD
- **🚫 No Host Contamination**: No containers run directly on host Docker
- **⚡ Consistent Environment**: Same isolation level for all operations
- **🎯 Resource Isolation**: CI/CD operations can't interfere with host services
- **🔄 Parallel Safety**: Multiple operations can run safely
**Usage:**
```bash
# Clean up DinD environment
./scripts/dind-cleanup.sh
# See what would be done without executing
./scripts/dind-cleanup.sh --dry-run
# Show help
./scripts/dind-cleanup.sh --help
```
**Benefits:**
- ✅ Fresh Docker environment for CI/CD
- ✅ No resource contention with Harbor
- ✅ Clean state for Rust testing
- ✅ Isolated CI/CD operations
The script is already part of the project repository and ready to use for maintaining clean CI/CD environments.
**How it works:**
- **Job 1 (Testing)**: Creates PostgreSQL, Rust, and Node.js containers inside DinD
- **Job 2 (Building)**: Uses DinD directly for building and pushing Docker images
- **Job 3 (Deployment)**: Runs on production runner (no DinD needed)
**Testing DinD Setup:**
```bash
# Test DinD cleanup script
./scripts/dind-cleanup.sh --dry-run
# Test DinD functionality
docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD is working!"
@ -1238,25 +1214,8 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest
- DinD container should be running and accessible
- Docker commands should work inside DinD
- Harbor push/pull should work from DinD
- Cleanup script should show proper status
#### 8.4 Set Up Automated DinD Cleanup
```bash
# Create a cron job to run DinD cleanup daily at 2 AM
(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab -
# Verify the cron job was added
crontab -l
```
**What this does:**
- **Automated cleanup**: Restarts DinD container daily for fresh environment
- **Prevents resource buildup**: Clears CI/CD artifacts automatically
- **Maintains performance**: Ensures consistent CI/CD performance
- **Zero Harbor impact**: DinD cleanup doesn't affect Harbor operations
#### 8.5 Monitoring Script
#### 8.4 Monitoring Script
**Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring.

28
ci-cd-compose.yml Normal file
View file

@ -0,0 +1,28 @@
version: '3.8'
services:
ci-cd-dind:
image: docker:dind
container_name: ci-cd-dind
privileged: true
restart: unless-stopped
ports:
- "2376:2376"
volumes:
- ci-cd-data:/var/lib/docker
- /var/run/docker.sock:/var/run/docker.sock
- ./backend:/workspace/backend
- ./frontend:/workspace/frontend
- ./scripts:/workspace/scripts
environment:
- DOCKER_TLS_CERTDIR=/certs
healthcheck:
test: ["CMD", "docker", "version"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
volumes:
ci-cd-data:
driver: local

View file

@ -1,232 +0,0 @@
#!/bin/bash
# Docker-in-Docker Cleanup Script
# This script provides a simple way to clean up the DinD environment
# by restarting the DinD container, which gives a fresh environment.
set -e
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging functions
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
show_help() {
cat << EOF
Docker-in-Docker Cleanup Script
Usage: $0 [OPTIONS]
Options:
--dry-run Show what would be done without executing
--help|-h Show this help message
Examples:
$0 # Clean up DinD environment
$0 --dry-run # Show what would be done
EOF
}
# Parse command line arguments
DRY_RUN="false"
while [[ $# -gt 0 ]]; do
case $1 in
--dry-run)
DRY_RUN="true"
shift
;;
--help|-h)
show_help
exit 0
;;
*)
log_error "Unknown option: $1"
show_help
exit 1
;;
esac
done
# Main cleanup function
cleanup_dind() {
echo
echo "=================================================================================="
echo " 🧹 Docker-in-Docker Cleanup 🧹"
echo "=================================================================================="
echo
# Check if DinD container exists
if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_error "DinD container 'ci-cd-dind' not found!"
log_info "Creating new DinD container..."
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would create DinD container"
return
fi
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
log_success "DinD container created successfully"
return
fi
# Check if DinD container is running
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD container is running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would stop and restart DinD container"
log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment"
return
fi
log_info "Stopping DinD container..."
docker stop ci-cd-dind
log_info "Removing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
else
log_info "DinD container exists but is not running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would remove and recreate DinD container"
return
fi
log_info "Removing existing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
fi
echo
echo "=================================================================================="
log_success "DinD cleanup completed successfully!"
echo "=================================================================================="
echo
log_info "Benefits of this cleanup:"
log_info " ✅ Fresh Docker environment for CI/CD"
log_info " ✅ No resource contention with Harbor"
log_info " ✅ Clean state for Rust testing"
log_info " ✅ Isolated CI/CD operations"
echo
}
# Show current DinD status
show_status() {
echo "=================================================================================="
echo " 📊 DinD Status 📊"
echo "=================================================================================="
echo
if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Container Status:"
docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
echo
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Docker Info:"
docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon"
echo
log_info "DinD Images:"
docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images"
echo
log_info "DinD Containers:"
docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers"
else
log_warning "DinD container is not running"
fi
else
log_warning "DinD container does not exist"
fi
echo "=================================================================================="
}
# Main execution
if [ "$DRY_RUN" = "true" ]; then
echo
echo "=================================================================================="
echo " 🚨 DRY RUN MODE 🚨"
echo " No changes will be made"
echo "=================================================================================="
echo
show_status
cleanup_dind
else
show_status
cleanup_dind
fi