sharenet/.forgejo/workflows/ci.yml

name: CI/CD Pipeline (Fully Isolated DinD)

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]

env:
  REGISTRY: ${{ secrets.CI_HOST }}:443
  IMAGE_NAME: ${{ secrets.APP_NAME || 'sharenet' }}

jobs:
  # Job 1: Testing - Uses DinD with multiple containers for comprehensive testing
  test:
    name: Run Tests (DinD)
    runs-on: [self-hosted, ci]
    if: github.ref == 'refs/heads/main'

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Setup DinD Environment
      run: |
        # Check if DinD container already exists and is running
        if ! docker ps --format "table {{.Names}}" | grep -q "^ci-dind$"; then
          echo "Starting new DinD container..."
          # Start DinD container for isolated CI operations
          docker run -d \
            --name ci-dind \
            --privileged \
            -p 2375:2375 \
            -e DOCKER_TLS_CERTDIR="" \
            docker:dind
          
          # Wait for DinD to be ready
          echo "Waiting for DinD container to be ready..."
          timeout 60 bash -c 'until docker exec ci-dind docker version; do sleep 2; done'
          
          # Copy Harbor certificate to DinD container
          docker cp /etc/ssl/registry/registry.crt ci-dind:/usr/local/share/ca-certificates/
          docker exec ci-dind chown root:root /usr/local/share/ca-certificates/registry.crt
          docker exec ci-dind update-ca-certificates
          
          # Login to Harbor registry (using HTTPS port 443)
          echo "${{ secrets.HARBOR_CI_PASSWORD }}" | docker exec -i ci-dind docker login ${{ secrets.CI_HOST }}:443 -u ${{ secrets.HARBOR_CI_USER }} --password-stdin
          
          echo "DinD container setup complete"
        else
          echo "DinD container already running, reusing existing setup"
          # Verify DinD is still working
          docker exec ci-dind docker version
        fi

    - name: Setup Containerized Testing Environment
      run: |
        # Copy docker-compose.test.yml to DinD container
        docker cp docker-compose.test.yml ci-dind:/workspace/
        docker cp backend ci-dind:/workspace/
        docker cp frontend ci-dind:/workspace/
        docker cp scripts ci-dind:/workspace/
        
        # Start testing environment using dedicated compose file inside DinD
        docker exec ci-dind docker compose -f /workspace/docker-compose.test.yml up -d
        
        # Wait for all services to be ready
        echo "Waiting for testing environment to be ready..."
        timeout 120 bash -c 'until docker exec ci-dind docker compose -f /workspace/docker-compose.test.yml ps | grep -q "healthy" && docker exec ci-dind docker compose -f /workspace/docker-compose.test.yml ps | grep -q "Up"; do sleep 2; done'
        
        # Verify all containers are running
        docker exec ci-dind docker compose -f /workspace/docker-compose.test.yml ps

    - name: Install SQLx CLI in Rust container
      run: |
        docker exec ci-dind docker exec ci-cd-test-rust cargo install sqlx-cli --no-default-features --features postgres

    - name: Validate migration files
      env:
        DATABASE_URL: postgres://postgres:password@localhost:5432/sharenet_test
      run: |
        # Wait for PostgreSQL to be ready
        echo "Waiting for PostgreSQL to be ready..."
        timeout 60 bash -c 'until docker exec ci-dind docker exec ci-cd-test-postgres pg_isready -h localhost -p 5432 -U postgres; do sleep 1; done'
        
        # Create test database if it doesn't exist
        docker exec ci-dind docker exec ci-cd-test-rust sqlx database create --database-url "$DATABASE_URL" || true
        
        # Run initial migrations to set up the database
        docker exec ci-dind docker exec ci-cd-test-rust sqlx migrate run --database-url "$DATABASE_URL" || true
        
        # Validate migration files
        docker exec ci-dind docker exec ci-cd-test-rust ./scripts/validate_migrations.sh --verbose

    - name: Run backend tests
      working-directory: ./backend
      env:
        DATABASE_URL: postgres://postgres:password@localhost:5432/sharenet_test
      run: |
        # Run tests with increased parallelism for Rust
        docker exec ci-dind docker exec ci-cd-test-rust cargo test --all --jobs 4
        docker exec ci-dind docker exec ci-cd-test-rust cargo clippy --all -- -D warnings
        docker exec ci-dind docker exec ci-cd-test-rust cargo fmt --all -- --check

    - name: Install frontend dependencies
      run: |
        docker exec ci-dind docker exec ci-cd-test-node npm ci

    - name: Run frontend tests
      run: |
        docker exec ci-dind docker exec ci-cd-test-node npm run lint
        docker exec ci-dind docker exec ci-cd-test-node npm run type-check
        docker exec ci-dind docker exec ci-cd-test-node npm run build

    - name: Cleanup Testing Environment
      if: always()
      run: |
        # Stop and remove all testing containers (but keep DinD running)
        docker exec ci-dind docker compose -f /workspace/docker-compose.test.yml down

  # Job 2: Building - Build and push Docker images using same DinD
  build-and-push:
    name: Build and Push Docker Images (DinD)
    needs: [test]
    runs-on: [self-hosted, ci]
    if: github.ref == 'refs/heads/main'

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Set up Docker Buildx in DinD
      run: |
        # Set up Docker Buildx inside the existing DinD container
        docker exec ci-dind docker buildx create --use --name ci-builder || true
        docker exec ci-dind docker buildx inspect --bootstrap

    - name: Build and push backend image
      run: |
        # Build and push backend image using DinD
        docker exec ci-dind docker buildx build \
          --platform linux/amd64 \
          --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/backend:${{ github.sha }} \
          --push \
          --cache-from type=gha \
          --cache-to type=gha,mode=max \
          -f ./backend/Dockerfile \
          ./backend

    - name: Build and push frontend image
      run: |
        # Build and push frontend image using DinD
        docker exec ci-dind docker buildx build \
          --platform linux/amd64 \
          --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/frontend:${{ github.sha }} \
          --push \
          --cache-from type=gha \
          --cache-to type=gha,mode=max \
          -f ./frontend/Dockerfile \
          ./frontend

    - name: Cleanup Testing Environment
      if: always()
      run: |
        # Clean up test containers but keep DinD running for reuse
        docker exec ci-dind docker system prune -f || true
        
        # Check if DinD needs restart due to resource accumulation
        DISK_USAGE=$(docker exec ci-dind df -h /var/lib/docker 2>/dev/null | tail -1 | awk '{print $5}' | sed 's/%//' || echo "0")
        echo "DinD disk usage: ${DISK_USAGE}%"
        
        # Restart DinD if disk usage is high (>80%)
        if [ "$DISK_USAGE" -gt 80 ]; then
          echo "WARNING: High disk usage (${DISK_USAGE}%), restarting DinD container..."
          docker restart ci-dind
          echo "DinD container restarted"
        else
          echo "Disk usage acceptable (${DISK_USAGE}%), keeping DinD running"
        fi

  # Job 3: Deployment - Runs directly on production runner (no DinD needed)
  deploy:
    name: Deploy to Production
    needs: build-and-push
    runs-on: [self-hosted, prod]
    if: github.ref == 'refs/heads/main'

    steps:
    - name: Setup deployment directory
      run: |
        # Create deployment directory if it doesn't exist
        sudo mkdir -p /opt/${{ secrets.APP_NAME || 'sharenet' }}
        sudo chown ${{ secrets.PROD_SERVICE_USER || 'prod-service' }}:${{ secrets.PROD_SERVICE_USER || 'prod-service' }} /opt/${{ secrets.APP_NAME || 'sharenet' }}
        sudo chmod 755 /opt/${{ secrets.APP_NAME || 'sharenet' }}

    - name: Checkout code to deployment directory
      uses: actions/checkout@v4
      with:
        path: /opt/${{ secrets.APP_NAME || 'sharenet' }}

    - name: Set proper ownership
      run: |
        # Ensure proper ownership of all files
        sudo chown -R ${{ secrets.PROD_SERVICE_USER || 'prod-service' }}:${{ secrets.PROD_SERVICE_USER || 'prod-service' }} /opt/${{ secrets.APP_NAME || 'sharenet' }}
        
        # Change to deployment directory for all subsequent operations
        cd /opt/${{ secrets.APP_NAME || 'sharenet' }}

    - name: Create environment file for deployment
      run: |
        # Create environment file for this deployment
        echo "IMAGE_TAG=${{ github.sha }}" > .env
        echo "REGISTRY=${{ secrets.CI_HOST }}:443" >> .env
        echo "IMAGE_NAME=${{ secrets.APP_NAME || 'sharenet' }}" >> .env
        echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD || 'your_secure_password_here' }}" >> .env
        echo "POSTGRES_USER=${{ secrets.POSTGRES_USER || 'sharenet' }}" >> .env
        echo "POSTGRES_DB=${{ secrets.POSTGRES_DB || 'sharenet' }}" >> .env
        echo "DATABASE_URL=postgresql://${{ secrets.POSTGRES_USER || 'sharenet' }}:${{ secrets.POSTGRES_PASSWORD || 'your_secure_password_here' }}@postgres:5432/${{ secrets.POSTGRES_DB || 'sharenet' }}" >> .env
        echo "NODE_ENV=production" >> .env
        echo "RUST_LOG=info" >> .env

    - name: Make scripts executable
      run: chmod +x scripts/*.sh

    - name: Configure Docker for Harbor access
      run: |
        # Configure Docker to access Harbor registry on CI Linode (using HTTPS)
        # The Harbor certificate should already be installed on the production server
        # as described in the CI guide Step 13
        
        # Wait for Docker to be ready
        timeout 30 bash -c 'until docker info; do sleep 1; done'

    - name: Validate migration files
      run: |
        echo "Validating migration files before deployment..."
        ./scripts/validate_migrations.sh --verbose || {
          echo "ERROR: Migration validation failed. Deployment aborted."
          exit 1
        }

    - name: Pull and deploy application
      run: |
        # Pull latest images from Harbor registry
        echo "Pulling latest images from Harbor registry..."
        docker compose -f docker-compose.prod.yml pull
        
        # Deploy the application stack
        echo "Deploying application stack..."
        docker compose -f docker-compose.prod.yml up -d
        
        # Wait for all services to be healthy
        echo "Waiting for all services to be healthy..."
        timeout 120 bash -c 'until docker compose -f docker-compose.prod.yml ps | grep -q "healthy" && docker compose -f docker-compose.prod.yml ps | grep -q "Up"; do sleep 2; done'
        
        # Verify deployment
        echo "Verifying deployment..."
        docker compose -f docker-compose.prod.yml ps