diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index daa963e..9858b8f 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI/CD Pipeline +name: CI/CD Pipeline (DinD) on: push: @@ -12,8 +12,8 @@ env: jobs: test-backend: - name: Test Backend - runs-on: ubuntu-latest + name: Test Backend (DinD) + runs-on: [self-hosted, dind] services: postgres: @@ -33,6 +33,18 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Setup DinD environment + run: | + # Ensure DinD is running and accessible + docker version + + # Configure Docker for Harbor registry + echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + # Wait for Docker to be ready + timeout 30 bash -c 'until docker info; do sleep 1; done' + - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: @@ -78,18 +90,31 @@ jobs: env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/${{ secrets.APP_NAME || 'sharenet' }}_test run: | - cargo test --all + # Run tests with increased parallelism for Rust + cargo test --all --jobs 4 cargo clippy --all -- -D warnings cargo fmt --all -- --check test-frontend: - name: Test Frontend - runs-on: ubuntu-latest + name: Test Frontend (DinD) + runs-on: [self-hosted, dind] steps: - name: Checkout code uses: actions/checkout@v4 + - name: Setup DinD environment + run: | + # Ensure DinD is running and accessible + docker version + + # Configure Docker for Harbor registry + echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + # Wait for Docker to be ready + timeout 30 bash -c 'until docker info; do sleep 1; done' + - name: Setup Node.js uses: actions/setup-node@v4 with: @@ -109,22 +134,29 @@ jobs: npm run build build-and-push: - name: Build and Push Docker Images + name: Build and Push Docker Images (DinD) needs: [test-backend, test-frontend] - runs-on: ubuntu-latest + runs-on: [self-hosted, dind] if: github.ref == 'refs/heads/main' steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Configure Docker for local registry + - name: Setup DinD environment run: | + # Ensure DinD is running and accessible + docker version + + # Configure Docker for Harbor registry echo '{"insecure-registries": ["${{ secrets.CI_HOST }}:5000"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker + + # Wait for Docker to be ready + timeout 30 bash -c 'until docker info; do sleep 1; done' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build and push backend image uses: docker/build-push-action@v5 @@ -135,6 +167,7 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/backend:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max + platforms: linux/amd64 - name: Build and push frontend image uses: docker/build-push-action@v5 @@ -145,17 +178,23 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/frontend:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max + platforms: linux/amd64 deploy: name: Deploy to Production needs: build-and-push - runs-on: ubuntu-latest + runs-on: [self-hosted, dind] if: github.ref == 'refs/heads/main' steps: - name: Checkout code uses: actions/checkout@v4 + - name: Setup DinD environment + run: | + # Ensure DinD is running and accessible + docker version + - name: Install SQLx CLI run: cargo install sqlx-cli --no-default-features --features postgres @@ -195,5 +234,5 @@ jobs: exit 1 } - # Run deployment using the new deployment script + # Run deployment using the deployment script ./scripts/deploy.sh deploy \ No newline at end of file diff --git a/CI_CD_PIPELINE_SETUP_GUIDE.md b/CI_CD_PIPELINE_SETUP_GUIDE.md index b035dc3..8240fd9 100644 --- a/CI_CD_PIPELINE_SETUP_GUIDE.md +++ b/CI_CD_PIPELINE_SETUP_GUIDE.md @@ -1,6 +1,6 @@ # CI/CD Pipeline Setup Guide -This guide covers setting up a complete Continuous Integration/Continuous Deployment (CI/CD) pipeline with a CI/CD Linode and Production Linode for automated builds, testing, and deployments. +This guide covers setting up a complete Continuous Integration/Continuous Deployment (CI/CD) pipeline with a CI/CD Linode and Production Linode for automated builds, testing, and deployments using Docker-in-Docker (DinD) for isolated CI operations. ## Architecture Overview @@ -8,7 +8,7 @@ This guide covers setting up a complete Continuous Integration/Continuous Deploy ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ Forgejo Host │ │ CI/CD Linode │ │ Production Linode│ │ (Repository) │ │ (Actions Runner)│ │ (Docker Deploy) │ -│ │ │ + Harbor Registry│ │ │ +│ │ │ + Harbor Registry│ │ + DinD Container│ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ │ │ │ │ │ @@ -20,12 +20,31 @@ This guide covers setting up a complete Continuous Integration/Continuous Deploy ## Pipeline Flow 1. **Code Push**: Developer pushes code to Forgejo repository -2. **Automated Testing**: CI/CD Linode runs tests on backend and frontend -3. **Image Building**: If tests pass, Docker images are built -4. **Registry Push**: Images are pushed to Harbor registry on CI/CD Linode +2. **Automated Testing**: CI/CD Linode runs tests in isolated DinD environment +3. **Image Building**: If tests pass, Docker images are built within DinD +4. **Registry Push**: Images are pushed to Harbor registry from DinD 5. **Production Deployment**: Production Linode pulls images and deploys 6. **Health Check**: Application is verified and accessible +## Key Benefits of DinD Approach + +### **For Rust Testing:** +- ✅ **Fresh environment** every test run +- ✅ **Parallel execution** capability +- ✅ **Isolated dependencies** - no test pollution +- ✅ **Fast cleanup** - just restart DinD container + +### **For CI/CD Operations:** +- ✅ **Zero resource contention** with Harbor +- ✅ **Simple cleanup** - one-line container restart +- ✅ **Perfect isolation** - CI/CD can't affect Harbor +- ✅ **Consistent environment** - same setup every time + +### **For Maintenance:** +- ✅ **Reduced complexity** - no complex cleanup scripts +- ✅ **Easy debugging** - isolated environment +- ✅ **Reliable operations** - no interference between services + ## Prerequisites - Two Ubuntu 24.04 LTS Linodes with root access @@ -35,21 +54,23 @@ This guide covers setting up a complete Continuous Integration/Continuous Deploy ## Quick Start -1. **Set up CI/CD Linode** (Steps 1-13) -2. **Set up Production Linode** (Steps 14-26) -3. **Configure SSH key exchange** (Step 27) -4. **Set up Forgejo repository secrets** (Step 28) -5. **Test the complete pipeline** (Step 29) +1. **Set up CI/CD Linode** (Steps 1-14) +2. **Set up Production Linode** (Steps 15-27) +3. **Configure SSH key exchange** (Step 28) +4. **Set up Forgejo repository secrets** (Step 29) +5. **Test the complete pipeline** (Step 30) ## What's Included ### CI/CD Linode Features - Forgejo Actions runner for automated builds +- **Docker-in-Docker (DinD) container** for isolated CI operations - Harbor container registry for image storage - Harbor web UI for image management - Built-in vulnerability scanning with Trivy - Role-based access control and audit logs - Secure SSH communication with production +- **Simplified cleanup** - just restart DinD container ### Production Linode Features - Docker-based application deployment @@ -59,11 +80,12 @@ This guide covers setting up a complete Continuous Integration/Continuous Deploy - Firewall and fail2ban protection ### Pipeline Features -- **Automated testing** on every code push -- **Automated image building** and registry push +- **Automated testing** on every code push in isolated environment +- **Automated image building** and registry push from DinD - **Automated deployment** to production - **Rollback capability** with image versioning - **Health monitoring** and logging +- **Zero resource contention** between CI/CD and Harbor ## Security Model and User Separation @@ -994,8 +1016,8 @@ cd ~ forgejo-runner register \ --instance https://your-forgejo-instance \ --token YOUR_REGISTRATION_TOKEN \ - --name "ci-cd-runner" \ - --labels "ubuntu-latest,docker" \ + --name "ci-cd-dind-runner" \ + --labels "ubuntu-latest,docker,dind" \ --no-interactive ``` @@ -1092,7 +1114,7 @@ sudo journalctl -u forgejo-runner.service -f --no-pager # Verify runner appears in Forgejo # Go to your Forgejo repository → Settings → Actions → Runners -# You should see your runner listed as "ci-cd-runner" with status "Online" +# You should see your runner listed as "ci-cd-dind-runner" with status "Online" ``` **Expected Output**: @@ -1105,9 +1127,355 @@ sudo journalctl -u forgejo-runner.service -f --no-pager - Check network: Ensure the runner can reach your Forgejo instance - Restart service: `sudo systemctl restart forgejo-runner.service` -### Step 8: Set Up Monitoring and Cleanup +### Step 8: Set Up Docker-in-Docker (DinD) for CI Operations -#### 8.1 Monitoring Script +**Important**: This step sets up a Docker-in-Docker container that provides an isolated environment for CI/CD operations, eliminating resource contention with Harbor and simplifying cleanup. + +#### 8.1 Create DinD Container + +```bash +# Create DinD container with persistent storage +docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + +# Wait for DinD to start +sleep 15 + +# Test DinD connectivity +docker exec ci-cd-dind docker version +``` + +**What this does**: +- **Creates isolated environment**: DinD container runs its own Docker daemon +- **Persistent storage**: `ci-cd-data` volume preserves data between restarts +- **Privileged mode**: Required for Docker-in-Docker functionality +- **Auto-restart**: Container restarts automatically if it crashes +- **Docker socket access**: Allows DinD to communicate with host Docker + +#### 8.2 Configure DinD for Harbor Registry + +```bash +# Configure Docker daemon in DinD for Harbor registry +docker exec ci-cd-dind sh -c 'echo "{\"insecure-registries\": [\"localhost:5000\"]}" > /etc/docker/daemon.json' + +# Reload Docker daemon in DinD +docker exec ci-cd-dind sh -c 'kill -HUP 1' + +# Wait for Docker daemon to reload +sleep 5 + +# Test Harbor connectivity from DinD +docker exec ci-cd-dind docker pull alpine:latest +docker exec ci-cd-dind docker tag alpine:latest localhost:5000/test/alpine:latest +docker exec ci-cd-dind docker push localhost:5000/test/alpine:latest + +# Clean up test image +docker exec ci-cd-dind docker rmi localhost:5000/test/alpine:latest +``` + +**What this does**: +- **Configures insecure registry**: Allows DinD to push to Harbor without SSL verification +- **Tests connectivity**: Verifies DinD can pull, tag, and push images to Harbor +- **Validates setup**: Ensures the complete CI/CD pipeline will work + +#### 8.3 Create DinD Cleanup Script + +```bash +# Create simplified cleanup script for DinD +cat > /opt/APP_NAME/scripts/dind-cleanup.sh << 'EOF' +#!/bin/bash + +# Docker-in-Docker Cleanup Script +# This script provides a simple way to clean up the DinD environment +# by restarting the DinD container, which gives a fresh environment. + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +show_help() { + cat << EOF +Docker-in-Docker Cleanup Script + +Usage: $0 [OPTIONS] + +Options: + --dry-run Show what would be done without executing + --help|-h Show this help message + +Examples: + $0 # Clean up DinD environment + $0 --dry-run # Show what would be done +EOF +} + +# Parse command line arguments +DRY_RUN="false" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN="true" + shift + ;; + --help|-h) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Main cleanup function +cleanup_dind() { + echo + echo "==================================================================================" + echo " 🧹 Docker-in-Docker Cleanup 🧹" + echo "==================================================================================" + echo + + # Check if DinD container exists + if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_error "DinD container 'ci-cd-dind' not found!" + log_info "Creating new DinD container..." + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would create DinD container" + return + fi + + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + log_success "DinD container created successfully" + return + fi + + # Check if DinD container is running + if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD container is running" + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would stop and restart DinD container" + log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment" + return + fi + + log_info "Stopping DinD container..." + docker stop ci-cd-dind + + log_info "Removing DinD container..." + docker rm ci-cd-dind + + log_info "Creating fresh DinD container..." + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + # Wait for DinD to start + log_info "Waiting for DinD to start..." + sleep 10 + + # Test DinD connectivity + if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then + log_success "DinD container is ready!" + else + log_error "DinD container failed to start properly" + exit 1 + fi + + else + log_info "DinD container exists but is not running" + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would remove and recreate DinD container" + return + fi + + log_info "Removing existing DinD container..." + docker rm ci-cd-dind + + log_info "Creating fresh DinD container..." + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + # Wait for DinD to start + log_info "Waiting for DinD to start..." + sleep 10 + + # Test DinD connectivity + if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then + log_success "DinD container is ready!" + else + log_error "DinD container failed to start properly" + exit 1 + fi + fi + + echo + echo "==================================================================================" + log_success "DinD cleanup completed successfully!" + echo "==================================================================================" + echo + log_info "Benefits of this cleanup:" + log_info " ✅ Fresh Docker environment for CI/CD" + log_info " ✅ No resource contention with Harbor" + log_info " ✅ Clean state for Rust testing" + log_info " ✅ Isolated CI/CD operations" + echo +} + +# Show current DinD status +show_status() { + echo "==================================================================================" + echo " 📊 DinD Status 📊" + echo "==================================================================================" + echo + + if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD Container Status:" + docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo + + if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD Docker Info:" + docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon" + echo + + log_info "DinD Images:" + docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images" + echo + + log_info "DinD Containers:" + docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers" + else + log_warning "DinD container is not running" + fi + else + log_warning "DinD container does not exist" + fi + + echo "==================================================================================" +} + +# Main execution +if [ "$DRY_RUN" = "true" ]; then + echo + echo "==================================================================================" + echo " 🚨 DRY RUN MODE 🚨" + echo " No changes will be made" + echo "==================================================================================" + echo + show_status + cleanup_dind +else + show_status + cleanup_dind +fi +EOF + +# Make the script executable +chmod +x /opt/APP_NAME/scripts/dind-cleanup.sh +``` + +**What this does**: +- **Creates cleanup script**: Simple script to restart DinD container for fresh environment +- **Status monitoring**: Shows current DinD container and Docker state +- **Dry-run mode**: Allows testing without making changes +- **Error handling**: Proper error checking and user feedback + +#### 8.4 Test DinD Setup + +```bash +# Test DinD cleanup script +./scripts/dind-cleanup.sh --dry-run + +# Test DinD functionality +docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD is working!" + +# Test Harbor integration +docker exec ci-cd-dind docker pull alpine:latest +docker exec ci-cd-dind docker tag alpine:latest localhost:5000/test/dind-test:latest +docker exec ci-cd-dind docker push localhost:5000/test/dind-test:latest + +# Clean up test +docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest +``` + +**Expected Output**: +- DinD container should be running and accessible +- Docker commands should work inside DinD +- Harbor push/pull should work from DinD +- Cleanup script should show proper status + +#### 8.5 Set Up Automated DinD Cleanup + +```bash +# Create a cron job to run DinD cleanup daily at 2 AM +(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab - + +# Verify the cron job was added +crontab -l +``` + +**What this does**: +- **Automated cleanup**: Restarts DinD container daily for fresh environment +- **Prevents resource buildup**: Clears CI/CD artifacts automatically +- **Maintains performance**: Ensures consistent CI/CD performance +- **Zero Harbor impact**: DinD cleanup doesn't affect Harbor operations + +### Step 9: Set Up Monitoring and Cleanup + +#### 9.1 Monitoring Script **Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring. @@ -1131,34 +1499,79 @@ chmod +x scripts/monitor.sh **Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information. -#### 8.2 Cleanup Script +#### 9.2 DinD Cleanup Script -**Important**: The repository includes a pre-configured cleanup script in the `scripts/` directory that can be used for both CI/CD and production cleanup operations. +**Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment. -**Repository Script**: -- `scripts/cleanup.sh` - Comprehensive cleanup script with support for both CI/CD and production environments +**DinD Cleanup Script**: +- `scripts/dind-cleanup.sh` - Simple script to restart DinD container for fresh CI environment -**To use the repository cleanup script**: +**To use the DinD cleanup script**: ```bash # The repository is already cloned at /opt/APP_NAME/ cd /opt/APP_NAME # Make the script executable -chmod +x scripts/cleanup.sh +chmod +x scripts/dind-cleanup.sh -# Test CI/CD cleanup (dry run first) -./scripts/cleanup.sh --type ci-cd --dry-run +# Test DinD cleanup (dry run first) +./scripts/dind-cleanup.sh --dry-run -# Run CI/CD cleanup -./scripts/cleanup.sh --type ci-cd - -# Test production cleanup (dry run first) -./scripts/cleanup.sh --type production --dry-run +# Run DinD cleanup +./scripts/dind-cleanup.sh ``` -**Note**: The repository script is more comprehensive and includes proper error handling, colored output, dry-run mode, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate cleanup operations. +**Benefits of DinD cleanup**: +- ✅ **Simple operation**: Just restart the DinD container +- ✅ **Zero Harbor impact**: Harbor registry is completely unaffected +- ✅ **Fresh environment**: Every cleanup gives a completely clean state +- ✅ **Fast execution**: No complex resource scanning needed +- ✅ **Reliable**: No risk of accidentally removing Harbor resources -#### 8.3 Test Cleanup Script +#### 9.3 Test DinD Cleanup Script + +```bash +# Test DinD cleanup with dry run first +./scripts/dind-cleanup.sh --dry-run + +# Run the DinD cleanup script +./scripts/dind-cleanup.sh + +# Verify DinD is working after cleanup +docker exec ci-cd-dind docker version +docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successful!" +``` + +**Expected Output**: +- DinD cleanup script should run without errors +- DinD container should be restarted with fresh environment +- Docker commands should work inside DinD after cleanup +- Harbor registry should remain completely unaffected + +**If something goes wrong**: +- Check script permissions: `ls -la scripts/dind-cleanup.sh` +- Verify DinD container: `docker ps | grep ci-cd-dind` +- Check DinD logs: `docker logs ci-cd-dind` +- Run manually: `bash -x scripts/dind-cleanup.sh` + +#### 9.4 Set Up Automated DinD Cleanup + +```bash +# Create a cron job to run DinD cleanup daily at 2 AM +(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab - + +# Verify the cron job was added +crontab -l +``` + +**What this does:** +- **Runs automatically**: The DinD cleanup script runs every day at 2:00 AM +- **Frequency**: Daily cleanup to prevent CI/CD resource buildup +- **Logging**: All cleanup output is logged to `/tmp/dind-cleanup.log` +- **What it cleans**: Restarts DinD container for fresh CI environment +- **Zero Harbor impact**: Harbor registry operations are completely unaffected + +#### 9.5 Test Cleanup Script ```bash # Create some test images to clean up @@ -1196,7 +1609,7 @@ docker network ls - Check registry access: `cd /opt/APP_NAME/registry && docker compose ps` - Run manually: `bash -x scripts/cleanup.sh` -#### 8.4 Set Up Automated Cleanup +#### 9.6 Set Up Automated Cleanup ```bash # Create a cron job to run cleanup daily at 3 AM using the repository script @@ -1212,7 +1625,7 @@ crontab -l - **Logging**: All cleanup output is logged to `/tmp/cleanup.log` - **What it cleans**: Unused Docker images, volumes, networks, and Harbor images -### Step 9: Configure Firewall +### Step 10: Configure Firewall ```bash sudo ufw --force enable @@ -1227,23 +1640,23 @@ sudo ufw allow 443/tcp # Harbor registry (public read access) - **SSH**: Restricted to your IP addresses - **All other ports**: Blocked -### Step 10: Test CI/CD Setup +### Step 11: Test CI/CD Setup -#### 10.1 Test Docker Installation +#### 11.1 Test Docker Installation ```bash docker --version docker compose --version ``` -#### 10.2 Check Harbor Status +#### 11.2 Check Harbor Status ```bash cd /opt/APP_NAME/registry docker compose ps ``` -#### 10.3 Test Harbor Access +#### 11.3 Test Harbor Access ```bash # Test Harbor API @@ -1253,7 +1666,7 @@ curl -k https://localhost:8080/api/v2.0/health curl -k -I https://localhost ``` -#### 10.4 Get Public Key for Production Server +#### 11.4 Get Public Key for Production Server ```bash cat ~/.ssh/id_ed25519.pub @@ -1265,15 +1678,15 @@ cat ~/.ssh/id_ed25519.pub ## Part 2: Production Linode Setup -### Step 11: Initial System Setup +### Step 12: Initial System Setup -#### 11.1 Update the System +#### 12.1 Update the System ```bash sudo apt update && sudo apt upgrade -y ``` -#### 11.2 Configure Timezone +#### 12.2 Configure Timezone ```bash # Configure timezone interactively @@ -1287,7 +1700,7 @@ date **Expected output**: After selecting your timezone, the `date` command should show the current date and time in your selected timezone. -#### 11.3 Configure /etc/hosts +#### 12.3 Configure /etc/hosts ```bash # Add localhost entries for both IPv4 and IPv6 @@ -1308,7 +1721,7 @@ cat /etc/hosts **Expected output**: The `/etc/hosts` file should show entries for `127.0.0.1`, `::1`, and your Linode's actual IP addresses all mapping to `localhost`. -#### 11.4 Install Essential Packages +#### 12.4 Install Essential Packages ```bash sudo apt install -y \ @@ -1326,9 +1739,9 @@ sudo apt install -y \ python3-certbot-nginx ``` -### Step 12: Create Users +### Step 13: Create Users -#### 12.1 Create the SERVICE_USER User +#### 13.1 Create the SERVICE_USER User ```bash # Create dedicated group for the service account @@ -1339,7 +1752,7 @@ sudo useradd -r -g SERVICE_USER -s /bin/bash -m -d /home/SERVICE_USER SERVICE_US echo "SERVICE_USER:$(openssl rand -base64 32)" | sudo chpasswd ``` -#### 12.2 Create the DEPLOY_USER User +#### 13.2 Create the DEPLOY_USER User ```bash # Create deployment user @@ -1348,7 +1761,7 @@ sudo usermod -aG sudo DEPLOY_USER echo "DEPLOY_USER:$(openssl rand -base64 32)" | sudo chpasswd ``` -#### 12.3 Verify Users +#### 13.3 Verify Users ```bash sudo su - SERVICE_USER @@ -1362,9 +1775,9 @@ pwd exit ``` -### Step 13: Install Docker +### Step 14: Install Docker -#### 13.1 Add Docker Repository +#### 14.1 Add Docker Repository ```bash curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg @@ -1372,28 +1785,28 @@ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docke sudo apt update ``` -#### 13.2 Install Docker Packages +#### 14.2 Install Docker Packages ```bash sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin ``` -#### 13.3 Configure Docker for Service Account +#### 14.3 Configure Docker for Service Account ```bash sudo usermod -aG docker SERVICE_USER ``` -### Step 14: Install Docker Compose +### Step 15: Install Docker Compose ```bash sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose ``` -### Step 15: Configure Security +### Step 16: Configure Security -#### 15.1 Configure Firewall +#### 16.1 Configure Firewall ```bash sudo ufw --force enable @@ -1406,16 +1819,16 @@ sudo ufw allow 3000/tcp sudo ufw allow 3001/tcp ``` -#### 15.2 Configure Fail2ban +#### 16.2 Configure Fail2ban ```bash sudo systemctl enable fail2ban sudo systemctl start fail2ban ``` -### Step 16: Create Application Directory +### Step 17: Create Application Directory -#### 16.1 Create Directory Structure +#### 17.1 Create Directory Structure ```bash sudo mkdir -p /opt/APP_NAME @@ -1424,22 +1837,22 @@ sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME **Note**: Replace `APP_NAME` with your actual application name. This directory name can be controlled via the `APP_NAME` secret in your Forgejo repository settings. If you set the `APP_NAME` secret to `myapp`, the deployment directory will be `/opt/myapp`. -#### 16.2 Create SSL Directory (Optional - for domain users) +#### 17.2 Create SSL Directory (Optional - for domain users) ```bash sudo mkdir -p /opt/APP_NAME/nginx/ssl sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME/nginx/ssl ``` -### Step 17: Clone Repository and Set Up Application Files +### Step 18: Clone Repository and Set Up Application Files -#### 17.1 Switch to SERVICE_USER User +#### 18.1 Switch to SERVICE_USER User ```bash sudo su - SERVICE_USER ``` -#### 17.2 Clone Repository +#### 18.2 Clone Repository ```bash cd /opt/APP_NAME @@ -1452,7 +1865,7 @@ git clone https://your-forgejo-instance/your-username/APP_NAME.git . **Note**: Replace `your-forgejo-instance` and `your-username/APP_NAME` with your actual Forgejo instance URL and repository path. -#### 17.3 Create Environment File +#### 18.3 Create Environment File The repository doesn't include a `.env.example` file for security reasons. The CI/CD pipeline will create the `.env` file dynamically during deployment. However, for manual testing or initial setup, you can create a basic `.env` file: @@ -1478,7 +1891,7 @@ EOF **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address and `your_secure_password_here` with a strong password. -#### 17.4 Configure Docker for Harbor Access +#### 18.4 Configure Docker for Harbor Access ```bash # Add the CI/CD Harbor registry to Docker's insecure registries @@ -1495,9 +1908,9 @@ sudo systemctl restart docker **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address. -### Step 18: Set Up SSH Key Authentication +### Step 19: Set Up SSH Key Authentication -#### 18.1 Add CI/CD Public Key +#### 19.1 Add CI/CD Public Key ```bash # Create .ssh directory for SERVICE_USER @@ -1511,7 +1924,7 @@ chmod 600 ~/.ssh/authorized_keys **Important**: Replace `YOUR_CI_CD_PUBLIC_KEY` with the public key from the CI/CD Linode (the output from `cat ~/.ssh/id_ed25519.pub` on the CI/CD Linode). -#### 18.2 Test SSH Connection +#### 19.2 Test SSH Connection From the CI/CD Linode, test the SSH connection: @@ -1521,16 +1934,16 @@ ssh production **Expected output**: You should be able to SSH to the production server without a password prompt. -### Step 19: Test Production Setup +### Step 20: Test Production Setup -#### 19.1 Test Docker Installation +#### 20.1 Test Docker Installation ```bash docker --version docker compose --version ``` -#### 19.2 Test Harbor Access +#### 20.2 Test Harbor Access ```bash # Test pulling an image from the CI/CD Harbor registry @@ -1539,14 +1952,14 @@ docker pull YOUR_CI_CD_IP:8080/public/backend:latest **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address. -#### 19.3 Test Application Deployment +#### 20.3 Test Application Deployment ```bash cd /opt/APP_NAME docker compose up -d ``` -#### 19.4 Verify Application Status +#### 20.4 Verify Application Status ```bash docker compose ps @@ -1563,9 +1976,9 @@ curl http://localhost:3001/health ## Part 3: Final Configuration and Testing -### Step 20: Configure Forgejo Repository Secrets +### Step 21: Configure Forgejo Repository Secrets -#### 20.1 Required Repository Secrets +#### 21.1 Required Repository Secrets Go to your Forgejo repository and add these secrets in **Settings → Secrets and Variables → Actions**: @@ -1581,16 +1994,16 @@ Go to your Forgejo repository and add these secrets in **Settings → Secrets an - `DOMAIN`: Your domain name (e.g., `example.com`) - `EMAIL`: Your email for SSL certificate notifications -#### 20.2 Configure Forgejo Actions Runner +#### 21.2 Configure Forgejo Actions Runner -##### 20.2.1 Get Runner Token +##### 21.2.1 Get Runner Token 1. Go to your Forgejo repository 2. Navigate to **Settings → Actions → Runners** 3. Click **"New runner"** 4. Copy the registration token -##### 20.2.2 Configure Runner +##### 21.2.2 Configure Runner ```bash # Switch to DEPLOY_USER on CI/CD Linode @@ -1604,19 +2017,19 @@ sudo su - DEPLOY_USER forgejo-runner register \ --instance https://your-forgejo-instance \ --token YOUR_TOKEN \ - --name "ci-cd-runner" \ - --labels "ubuntu-latest,docker" \ + --name "ci-cd-dind-runner" \ + --labels "ubuntu-latest,docker,dind" \ --no-interactive ``` -##### 20.2.3 Start Runner +##### 21.2.3 Start Runner ```bash sudo systemctl start forgejo-runner.service sudo systemctl status forgejo-runner.service ``` -##### 20.2.4 Test Runner Configuration +##### 21.2.4 Test Runner Configuration ```bash # Check if the runner is running @@ -1627,7 +2040,7 @@ sudo journalctl -u forgejo-runner.service -f --no-pager # Verify runner appears in Forgejo # Go to your Forgejo repository → Settings → Actions → Runners -# You should see your runner listed as "ci-cd-runner" with status "Online" +# You should see your runner listed as "ci-cd-dind-runner" with status "Online" ``` **Expected Output**: @@ -1640,9 +2053,9 @@ sudo journalctl -u forgejo-runner.service -f --no-pager - Check network: Ensure the runner can reach your Forgejo instance - Restart service: `sudo systemctl restart forgejo-runner.service` -### Step 21: Set Up Monitoring and Cleanup +### Step 22: Set Up Monitoring and Cleanup -#### 21.1 Monitoring Script +#### 22.1 Monitoring Script **Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring. @@ -1666,108 +2079,100 @@ chmod +x scripts/monitor.sh **Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information. -#### 21.2 Cleanup Script +#### 22.2 DinD Cleanup Script -**Important**: The repository includes a pre-configured cleanup script in the `scripts/` directory that can be used for both CI/CD and production cleanup operations. +**Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment. -**Repository Script**: -- `scripts/cleanup.sh` - Comprehensive cleanup script with support for both CI/CD and production environments +**DinD Cleanup Script**: +- `scripts/dind-cleanup.sh` - Simple script to restart DinD container for fresh CI environment -**To use the repository cleanup script**: +**To use the DinD cleanup script**: ```bash # The repository is already cloned at /opt/APP_NAME/ cd /opt/APP_NAME # Make the script executable -chmod +x scripts/cleanup.sh +chmod +x scripts/dind-cleanup.sh -# Test CI/CD cleanup (dry run first) -./scripts/cleanup.sh --type ci-cd --dry-run +# Test DinD cleanup (dry run first) +./scripts/dind-cleanup.sh --dry-run -# Run CI/CD cleanup -./scripts/cleanup.sh --type ci-cd - -# Test production cleanup (dry run first) -./scripts/cleanup.sh --type production --dry-run +# Run DinD cleanup +./scripts/dind-cleanup.sh ``` -**Note**: The repository script is more comprehensive and includes proper error handling, colored output, dry-run mode, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate cleanup operations. +**Benefits of DinD cleanup**: +- ✅ **Simple operation**: Just restart the DinD container +- ✅ **Zero Harbor impact**: Harbor registry is completely unaffected +- ✅ **Fresh environment**: Every cleanup gives a completely clean state +- ✅ **Fast execution**: No complex resource scanning needed +- ✅ **Reliable**: No risk of accidentally removing Harbor resources -#### 21.3 Test Cleanup Script +#### 22.3 Test DinD Cleanup Script ```bash -# Create some test images to clean up -docker pull alpine:latest -docker pull nginx:latest -docker tag alpine:latest test-cleanup:latest -docker tag nginx:latest test-cleanup2:latest +# Test DinD cleanup with dry run first +./scripts/dind-cleanup.sh --dry-run -# Test cleanup with dry run first -./scripts/cleanup.sh --type ci-cd --dry-run +# Run the DinD cleanup script +./scripts/dind-cleanup.sh -# Run the cleanup script -./scripts/cleanup.sh --type ci-cd - -# Verify cleanup worked -echo "Checking remaining images:" -docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" - -echo "Checking remaining volumes:" -docker volume ls - -echo "Checking remaining networks:" -docker network ls +# Verify DinD is working after cleanup +docker exec ci-cd-dind docker version +docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successful!" ``` **Expected Output**: -- Cleanup script should run without errors -- Test images should be removed -- System should report cleanup completion -- Remaining images should be minimal (only actively used ones) +- DinD cleanup script should run without errors +- DinD container should be restarted with fresh environment +- Docker commands should work inside DinD after cleanup +- Harbor registry should remain completely unaffected **If something goes wrong**: -- Check script permissions: `ls -la scripts/cleanup.sh` -- Verify Docker access: `docker ps` -- Check registry access: `cd /opt/APP_NAME/registry && docker compose ps` -- Run manually: `bash -x scripts/cleanup.sh` +- Check script permissions: `ls -la scripts/dind-cleanup.sh` +- Verify DinD container: `docker ps | grep ci-cd-dind` +- Check DinD logs: `docker logs ci-cd-dind` +- Run manually: `bash -x scripts/dind-cleanup.sh` -#### 21.4 Set Up Automated Cleanup +#### 22.4 Set Up Automated DinD Cleanup ```bash -# Create a cron job to run cleanup daily at 3 AM using the repository script -(crontab -l 2>/dev/null; echo "0 3 * * * cd /opt/APP_NAME && ./scripts/cleanup.sh --type ci-cd >> /tmp/cleanup.log 2>&1") | crontab - +# Create a cron job to run DinD cleanup daily at 2 AM +(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab - # Verify the cron job was added crontab -l ``` **What this does:** -- **Runs automatically**: The cleanup script runs every day at 3:00 AM -- **Frequency**: Daily cleanup to prevent disk space issues -- **Logging**: All cleanup output is logged to `/tmp/cleanup.log` -- **What it cleans**: Unused Docker images, volumes, networks, and Harbor images +- **Runs automatically**: The DinD cleanup script runs every day at 2:00 AM +- **Frequency**: Daily cleanup to prevent CI/CD resource buildup +- **Logging**: All cleanup output is logged to `/tmp/dind-cleanup.log` +- **What it cleans**: Restarts DinD container for fresh CI environment +- **Zero Harbor impact**: Harbor registry operations are completely unaffected -### Step 22: Test Complete Pipeline +### Step 23: Test Complete Pipeline -#### 22.1 Trigger a Test Build +#### 23.1 Trigger a Test Build 1. **Make a small change** to your repository (e.g., update a comment or add a test file) 2. **Commit and push** the changes to trigger the CI/CD pipeline 3. **Monitor the build** in your Forgejo repository → Actions tab -#### 22.2 Verify Pipeline Steps +#### 23.2 Verify Pipeline Steps The pipeline should execute these steps in order: 1. **Checkout**: Clone the repository -2. **Test Backend**: Run backend tests -3. **Test Frontend**: Run frontend tests -4. **Build Backend**: Build backend Docker image -5. **Build Frontend**: Build frontend Docker image -6. **Push to Registry**: Push images to your private registry -7. **Deploy to Production**: Deploy to production server +2. **Setup DinD**: Configure Docker-in-Docker environment +3. **Test Backend**: Run backend tests in isolated environment +4. **Test Frontend**: Run frontend tests in isolated environment +5. **Build Backend**: Build backend Docker image in DinD +6. **Build Frontend**: Build frontend Docker image in DinD +7. **Push to Registry**: Push images to Harbor registry from DinD +8. **Deploy to Production**: Deploy to production server -#### 22.3 Check Harbor +#### 23.3 Check Harbor ```bash # On CI/CD Linode @@ -1781,7 +2186,7 @@ curl -k https://localhost:8080/v2/public/backend/tags/list curl -k https://localhost:8080/v2/public/frontend/tags/list ``` -#### 22.4 Verify Production Deployment +#### 23.4 Verify Production Deployment ```bash # On Production Linode @@ -1799,16 +2204,16 @@ docker compose logs backend docker compose logs frontend ``` -#### 22.5 Test Application Functionality +#### 23.5 Test Application Functionality 1. **Frontend**: Visit your production URL (IP or domain) 2. **Backend API**: Test API endpoints 3. **Database**: Verify database connections 4. **Logs**: Check for any errors in application logs -### Step 23: Set Up SSL/TLS (Optional - Domain Users) +### Step 24: Set Up SSL/TLS (Optional - Domain Users) -#### 23.1 Install SSL Certificate +#### 24.1 Install SSL Certificate If you have a domain pointing to your Production Linode: @@ -1820,7 +2225,7 @@ sudo certbot --nginx -d your-domain.com sudo certbot certificates ``` -#### 23.2 Configure Auto-Renewal +#### 24.2 Configure Auto-Renewal ```bash # Test auto-renewal @@ -1832,9 +2237,9 @@ sudo crontab -e # 0 12 * * * /usr/bin/certbot renew --quiet ``` -### Step 24: Final Verification +### Step 25: Final Verification -#### 24.1 Security Check +#### 25.1 Security Check ```bash # Check firewall status @@ -1847,7 +2252,7 @@ sudo systemctl status fail2ban sudo grep "PasswordAuthentication" /etc/ssh/sshd_config ``` -#### 24.2 Performance Check +#### 25.2 Performance Check ```bash # Check system resources @@ -1860,7 +2265,7 @@ df -h docker system df ``` -#### 24.3 Backup Verification +#### 25.3 Backup Verification ```bash # Test backup script @@ -1871,16 +2276,16 @@ cd /opt/APP_NAME ./scripts/backup.sh ``` -### Step 25: Documentation and Maintenance +### Step 26: Documentation and Maintenance -#### 25.1 Update Documentation +#### 26.1 Update Documentation 1. **Update README.md** with deployment information 2. **Document environment variables** and their purposes 3. **Create troubleshooting guide** for common issues 4. **Document backup and restore procedures** -#### 25.2 Set Up Monitoring Alerts +#### 26.2 Set Up Monitoring Alerts ```bash # Set up monitoring cron job @@ -1890,7 +2295,7 @@ cd /opt/APP_NAME tail -f /tmp/monitor.log ``` -#### 25.3 Regular Maintenance Tasks +#### 26.3 Regular Maintenance Tasks **Daily:** - Check application logs for errors @@ -1913,12 +2318,13 @@ tail -f /tmp/monitor.log You have successfully set up a complete CI/CD pipeline with: -- ✅ **Automated testing** on every code push +- ✅ **Automated testing** on every code push in isolated DinD environment - ✅ **Docker image building** and Harbor registry storage - ✅ **Automated deployment** to production - ✅ **Health monitoring** and logging - ✅ **Backup and cleanup** automation - ✅ **Security hardening** with proper user separation - ✅ **SSL/TLS support** for production (optional) +- ✅ **Zero resource contention** between CI/CD and Harbor -Your application is now ready for continuous deployment with proper security, monitoring, and maintenance procedures in place! +Your application is now ready for continuous deployment with proper security, monitoring, and maintenance procedures in place! \ No newline at end of file