From f11348f2099c7b6f9d30e9f35f6b199c47665f66 Mon Sep 17 00:00:00 2001 From: continuist Date: Sun, 29 Jun 2025 22:18:13 -0400 Subject: [PATCH] Consolidate DinD cleanup process --- CI_CD_PIPELINE_SETUP_GUIDE.md | 538 ++++++---------------------------- scripts/dind-cleanup.sh | 232 +++++++++++++++ 2 files changed, 328 insertions(+), 442 deletions(-) create mode 100755 scripts/dind-cleanup.sh diff --git a/CI_CD_PIPELINE_SETUP_GUIDE.md b/CI_CD_PIPELINE_SETUP_GUIDE.md index 8240fd9..61acde0 100644 --- a/CI_CD_PIPELINE_SETUP_GUIDE.md +++ b/CI_CD_PIPELINE_SETUP_GUIDE.md @@ -1184,256 +1184,38 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/alpine:latest - **Tests connectivity**: Verifies DinD can pull, tag, and push images to Harbor - **Validates setup**: Ensures the complete CI/CD pipeline will work -#### 8.3 Create DinD Cleanup Script +#### 8.3 DinD Cleanup Script and Testing +The project includes a comprehensive Docker-in-Docker cleanup script at `scripts/dind-cleanup.sh` that provides a simple way to clean up the DinD environment by restarting the DinD container for a fresh environment. + +**Key Features:** +- **๐Ÿงน Complete Environment Reset**: Stops, removes, and recreates the `ci-cd-dind` container +- **๐Ÿ“Š Status Monitoring**: Shows current DinD container status, Docker info, images, and containers +- **๐Ÿšจ Dry Run Mode**: Use `--dry-run` to see what would be done without making changes +- **๐ŸŽจ Colored Output**: Clear, color-coded logging for better readability +- **๐Ÿ›ก๏ธ Error Handling**: Robust error checking and graceful failure handling + +**Usage:** ```bash -# Create simplified cleanup script for DinD -cat > /opt/APP_NAME/scripts/dind-cleanup.sh << 'EOF' -#!/bin/bash +# Clean up DinD environment +./scripts/dind-cleanup.sh -# Docker-in-Docker Cleanup Script -# This script provides a simple way to clean up the DinD environment -# by restarting the DinD container, which gives a fresh environment. +# See what would be done without executing +./scripts/dind-cleanup.sh --dry-run -set -e - -# Color codes for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -show_help() { - cat << EOF -Docker-in-Docker Cleanup Script - -Usage: $0 [OPTIONS] - -Options: - --dry-run Show what would be done without executing - --help|-h Show this help message - -Examples: - $0 # Clean up DinD environment - $0 --dry-run # Show what would be done -EOF -} - -# Parse command line arguments -DRY_RUN="false" - -while [[ $# -gt 0 ]]; do - case $1 in - --dry-run) - DRY_RUN="true" - shift - ;; - --help|-h) - show_help - exit 0 - ;; - *) - log_error "Unknown option: $1" - show_help - exit 1 - ;; - esac -done - -# Main cleanup function -cleanup_dind() { - echo - echo "==================================================================================" - echo " ๐Ÿงน Docker-in-Docker Cleanup ๐Ÿงน" - echo "==================================================================================" - echo - - # Check if DinD container exists - if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then - log_error "DinD container 'ci-cd-dind' not found!" - log_info "Creating new DinD container..." - - if [ "$DRY_RUN" = "true" ]; then - log_info "DRY RUN: Would create DinD container" - return - fi - - docker run -d \ - --name ci-cd-dind \ - --privileged \ - --restart unless-stopped \ - -p 2376:2376 \ - -v ci-cd-data:/var/lib/docker \ - -v /var/run/docker.sock:/var/run/docker.sock \ - docker:dind - - log_success "DinD container created successfully" - return - fi - - # Check if DinD container is running - if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then - log_info "DinD container is running" - - if [ "$DRY_RUN" = "true" ]; then - log_info "DRY RUN: Would stop and restart DinD container" - log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment" - return - fi - - log_info "Stopping DinD container..." - docker stop ci-cd-dind - - log_info "Removing DinD container..." - docker rm ci-cd-dind - - log_info "Creating fresh DinD container..." - docker run -d \ - --name ci-cd-dind \ - --privileged \ - --restart unless-stopped \ - -p 2376:2376 \ - -v ci-cd-data:/var/lib/docker \ - -v /var/run/docker.sock:/var/run/docker.sock \ - docker:dind - - # Wait for DinD to start - log_info "Waiting for DinD to start..." - sleep 10 - - # Test DinD connectivity - if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then - log_success "DinD container is ready!" - else - log_error "DinD container failed to start properly" - exit 1 - fi - - else - log_info "DinD container exists but is not running" - - if [ "$DRY_RUN" = "true" ]; then - log_info "DRY RUN: Would remove and recreate DinD container" - return - fi - - log_info "Removing existing DinD container..." - docker rm ci-cd-dind - - log_info "Creating fresh DinD container..." - docker run -d \ - --name ci-cd-dind \ - --privileged \ - --restart unless-stopped \ - -p 2376:2376 \ - -v ci-cd-data:/var/lib/docker \ - -v /var/run/docker.sock:/var/run/docker.sock \ - docker:dind - - # Wait for DinD to start - log_info "Waiting for DinD to start..." - sleep 10 - - # Test DinD connectivity - if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then - log_success "DinD container is ready!" - else - log_error "DinD container failed to start properly" - exit 1 - fi - fi - - echo - echo "==================================================================================" - log_success "DinD cleanup completed successfully!" - echo "==================================================================================" - echo - log_info "Benefits of this cleanup:" - log_info " โœ… Fresh Docker environment for CI/CD" - log_info " โœ… No resource contention with Harbor" - log_info " โœ… Clean state for Rust testing" - log_info " โœ… Isolated CI/CD operations" - echo -} - -# Show current DinD status -show_status() { - echo "==================================================================================" - echo " ๐Ÿ“Š DinD Status ๐Ÿ“Š" - echo "==================================================================================" - echo - - if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then - log_info "DinD Container Status:" - docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" - echo - - if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then - log_info "DinD Docker Info:" - docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon" - echo - - log_info "DinD Images:" - docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images" - echo - - log_info "DinD Containers:" - docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers" - else - log_warning "DinD container is not running" - fi - else - log_warning "DinD container does not exist" - fi - - echo "==================================================================================" -} - -# Main execution -if [ "$DRY_RUN" = "true" ]; then - echo - echo "==================================================================================" - echo " ๐Ÿšจ DRY RUN MODE ๐Ÿšจ" - echo " No changes will be made" - echo "==================================================================================" - echo - show_status - cleanup_dind -else - show_status - cleanup_dind -fi -EOF - -# Make the script executable -chmod +x /opt/APP_NAME/scripts/dind-cleanup.sh +# Show help +./scripts/dind-cleanup.sh --help ``` -**What this does**: -- **Creates cleanup script**: Simple script to restart DinD container for fresh environment -- **Status monitoring**: Shows current DinD container and Docker state -- **Dry-run mode**: Allows testing without making changes -- **Error handling**: Proper error checking and user feedback +**Benefits:** +- โœ… Fresh Docker environment for CI/CD +- โœ… No resource contention with Harbor +- โœ… Clean state for Rust testing +- โœ… Isolated CI/CD operations -#### 8.4 Test DinD Setup +The script is already part of the project repository and ready to use for maintaining clean CI/CD environments. + +**Testing DinD Setup:** ```bash # Test DinD cleanup script @@ -1457,7 +1239,7 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest - Harbor push/pull should work from DinD - Cleanup script should show proper status -#### 8.5 Set Up Automated DinD Cleanup +#### 8.4 Set Up Automated DinD Cleanup ```bash # Create a cron job to run DinD cleanup daily at 2 AM @@ -1467,15 +1249,13 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest crontab -l ``` -**What this does**: +**What this does:** - **Automated cleanup**: Restarts DinD container daily for fresh environment - **Prevents resource buildup**: Clears CI/CD artifacts automatically - **Maintains performance**: Ensures consistent CI/CD performance - **Zero Harbor impact**: DinD cleanup doesn't affect Harbor operations -### Step 9: Set Up Monitoring and Cleanup - -#### 9.1 Monitoring Script +#### 8.5 Monitoring Script **Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring. @@ -1499,133 +1279,7 @@ chmod +x scripts/monitor.sh **Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information. -#### 9.2 DinD Cleanup Script - -**Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment. - -**DinD Cleanup Script**: -- `scripts/dind-cleanup.sh` - Simple script to restart DinD container for fresh CI environment - -**To use the DinD cleanup script**: -```bash -# The repository is already cloned at /opt/APP_NAME/ -cd /opt/APP_NAME - -# Make the script executable -chmod +x scripts/dind-cleanup.sh - -# Test DinD cleanup (dry run first) -./scripts/dind-cleanup.sh --dry-run - -# Run DinD cleanup -./scripts/dind-cleanup.sh -``` - -**Benefits of DinD cleanup**: -- โœ… **Simple operation**: Just restart the DinD container -- โœ… **Zero Harbor impact**: Harbor registry is completely unaffected -- โœ… **Fresh environment**: Every cleanup gives a completely clean state -- โœ… **Fast execution**: No complex resource scanning needed -- โœ… **Reliable**: No risk of accidentally removing Harbor resources - -#### 9.3 Test DinD Cleanup Script - -```bash -# Test DinD cleanup with dry run first -./scripts/dind-cleanup.sh --dry-run - -# Run the DinD cleanup script -./scripts/dind-cleanup.sh - -# Verify DinD is working after cleanup -docker exec ci-cd-dind docker version -docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successful!" -``` - -**Expected Output**: -- DinD cleanup script should run without errors -- DinD container should be restarted with fresh environment -- Docker commands should work inside DinD after cleanup -- Harbor registry should remain completely unaffected - -**If something goes wrong**: -- Check script permissions: `ls -la scripts/dind-cleanup.sh` -- Verify DinD container: `docker ps | grep ci-cd-dind` -- Check DinD logs: `docker logs ci-cd-dind` -- Run manually: `bash -x scripts/dind-cleanup.sh` - -#### 9.4 Set Up Automated DinD Cleanup - -```bash -# Create a cron job to run DinD cleanup daily at 2 AM -(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab - - -# Verify the cron job was added -crontab -l -``` - -**What this does:** -- **Runs automatically**: The DinD cleanup script runs every day at 2:00 AM -- **Frequency**: Daily cleanup to prevent CI/CD resource buildup -- **Logging**: All cleanup output is logged to `/tmp/dind-cleanup.log` -- **What it cleans**: Restarts DinD container for fresh CI environment -- **Zero Harbor impact**: Harbor registry operations are completely unaffected - -#### 9.5 Test Cleanup Script - -```bash -# Create some test images to clean up -docker pull alpine:latest -docker pull nginx:latest -docker tag alpine:latest test-cleanup:latest -docker tag nginx:latest test-cleanup2:latest - -# Test cleanup with dry run first -./scripts/cleanup.sh --type ci-cd --dry-run - -# Run the cleanup script -./scripts/cleanup.sh --type ci-cd - -# Verify cleanup worked -echo "Checking remaining images:" -docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" - -echo "Checking remaining volumes:" -docker volume ls - -echo "Checking remaining networks:" -docker network ls -``` - -**Expected Output**: -- Cleanup script should run without errors -- Test images should be removed -- System should report cleanup completion -- Remaining images should be minimal (only actively used ones) - -**If something goes wrong**: -- Check script permissions: `ls -la scripts/cleanup.sh` -- Verify Docker access: `docker ps` -- Check registry access: `cd /opt/APP_NAME/registry && docker compose ps` -- Run manually: `bash -x scripts/cleanup.sh` - -#### 9.6 Set Up Automated Cleanup - -```bash -# Create a cron job to run cleanup daily at 3 AM using the repository script -(crontab -l 2>/dev/null; echo "0 3 * * * cd /opt/APP_NAME && ./scripts/cleanup.sh --type ci-cd >> /tmp/cleanup.log 2>&1") | crontab - - -# Verify the cron job was added -crontab -l -``` - -**What this does:** -- **Runs automatically**: The cleanup script runs every day at 3:00 AM -- **Frequency**: Daily cleanup to prevent disk space issues -- **Logging**: All cleanup output is logged to `/tmp/cleanup.log` -- **What it cleans**: Unused Docker images, volumes, networks, and Harbor images - -### Step 10: Configure Firewall +### Step 9: Configure Firewall ```bash sudo ufw --force enable @@ -1640,23 +1294,23 @@ sudo ufw allow 443/tcp # Harbor registry (public read access) - **SSH**: Restricted to your IP addresses - **All other ports**: Blocked -### Step 11: Test CI/CD Setup +### Step 10: Test CI/CD Setup -#### 11.1 Test Docker Installation +#### 10.1 Test Docker Installation ```bash docker --version docker compose --version ``` -#### 11.2 Check Harbor Status +#### 10.2 Check Harbor Status ```bash cd /opt/APP_NAME/registry docker compose ps ``` -#### 11.3 Test Harbor Access +#### 10.3 Test Harbor Access ```bash # Test Harbor API @@ -1666,7 +1320,7 @@ curl -k https://localhost:8080/api/v2.0/health curl -k -I https://localhost ``` -#### 11.4 Get Public Key for Production Server +#### 10.4 Get Public Key for Production Server ```bash cat ~/.ssh/id_ed25519.pub @@ -1678,15 +1332,15 @@ cat ~/.ssh/id_ed25519.pub ## Part 2: Production Linode Setup -### Step 12: Initial System Setup +### Step 11: Initial System Setup -#### 12.1 Update the System +#### 11.1 Update the System ```bash sudo apt update && sudo apt upgrade -y ``` -#### 12.2 Configure Timezone +#### 11.2 Configure Timezone ```bash # Configure timezone interactively @@ -1700,7 +1354,7 @@ date **Expected output**: After selecting your timezone, the `date` command should show the current date and time in your selected timezone. -#### 12.3 Configure /etc/hosts +#### 11.3 Configure /etc/hosts ```bash # Add localhost entries for both IPv4 and IPv6 @@ -1721,7 +1375,7 @@ cat /etc/hosts **Expected output**: The `/etc/hosts` file should show entries for `127.0.0.1`, `::1`, and your Linode's actual IP addresses all mapping to `localhost`. -#### 12.4 Install Essential Packages +#### 11.4 Install Essential Packages ```bash sudo apt install -y \ @@ -1739,9 +1393,9 @@ sudo apt install -y \ python3-certbot-nginx ``` -### Step 13: Create Users +### Step 12: Create Users -#### 13.1 Create the SERVICE_USER User +#### 12.1 Create the SERVICE_USER User ```bash # Create dedicated group for the service account @@ -1752,7 +1406,7 @@ sudo useradd -r -g SERVICE_USER -s /bin/bash -m -d /home/SERVICE_USER SERVICE_US echo "SERVICE_USER:$(openssl rand -base64 32)" | sudo chpasswd ``` -#### 13.2 Create the DEPLOY_USER User +#### 12.2 Create the DEPLOY_USER User ```bash # Create deployment user @@ -1761,7 +1415,7 @@ sudo usermod -aG sudo DEPLOY_USER echo "DEPLOY_USER:$(openssl rand -base64 32)" | sudo chpasswd ``` -#### 13.3 Verify Users +#### 12.3 Verify Users ```bash sudo su - SERVICE_USER @@ -1775,9 +1429,9 @@ pwd exit ``` -### Step 14: Install Docker +### Step 13: Install Docker -#### 14.1 Add Docker Repository +#### 13.1 Add Docker Repository ```bash curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg @@ -1785,28 +1439,28 @@ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docke sudo apt update ``` -#### 14.2 Install Docker Packages +#### 13.2 Install Docker Packages ```bash sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin ``` -#### 14.3 Configure Docker for Service Account +#### 13.3 Configure Docker for Service Account ```bash sudo usermod -aG docker SERVICE_USER ``` -### Step 15: Install Docker Compose +### Step 14: Install Docker Compose ```bash sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose ``` -### Step 16: Configure Security +### Step 15: Configure Security -#### 16.1 Configure Firewall +#### 15.1 Configure Firewall ```bash sudo ufw --force enable @@ -1819,16 +1473,16 @@ sudo ufw allow 3000/tcp sudo ufw allow 3001/tcp ``` -#### 16.2 Configure Fail2ban +#### 15.2 Configure Fail2ban ```bash sudo systemctl enable fail2ban sudo systemctl start fail2ban ``` -### Step 17: Create Application Directory +### Step 16: Create Application Directory -#### 17.1 Create Directory Structure +#### 16.1 Create Directory Structure ```bash sudo mkdir -p /opt/APP_NAME @@ -1837,22 +1491,22 @@ sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME **Note**: Replace `APP_NAME` with your actual application name. This directory name can be controlled via the `APP_NAME` secret in your Forgejo repository settings. If you set the `APP_NAME` secret to `myapp`, the deployment directory will be `/opt/myapp`. -#### 17.2 Create SSL Directory (Optional - for domain users) +#### 16.2 Create SSL Directory (Optional - for domain users) ```bash sudo mkdir -p /opt/APP_NAME/nginx/ssl sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME/nginx/ssl ``` -### Step 18: Clone Repository and Set Up Application Files +### Step 17: Clone Repository and Set Up Application Files -#### 18.1 Switch to SERVICE_USER User +#### 17.1 Switch to SERVICE_USER User ```bash sudo su - SERVICE_USER ``` -#### 18.2 Clone Repository +#### 17.2 Clone Repository ```bash cd /opt/APP_NAME @@ -1865,7 +1519,7 @@ git clone https://your-forgejo-instance/your-username/APP_NAME.git . **Note**: Replace `your-forgejo-instance` and `your-username/APP_NAME` with your actual Forgejo instance URL and repository path. -#### 18.3 Create Environment File +#### 17.3 Create Environment File The repository doesn't include a `.env.example` file for security reasons. The CI/CD pipeline will create the `.env` file dynamically during deployment. However, for manual testing or initial setup, you can create a basic `.env` file: @@ -1891,7 +1545,7 @@ EOF **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address and `your_secure_password_here` with a strong password. -#### 18.4 Configure Docker for Harbor Access +#### 17.4 Configure Docker for Harbor Access ```bash # Add the CI/CD Harbor registry to Docker's insecure registries @@ -1908,9 +1562,9 @@ sudo systemctl restart docker **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address. -### Step 19: Set Up SSH Key Authentication +### Step 18: Set Up SSH Key Authentication -#### 19.1 Add CI/CD Public Key +#### 18.1 Add CI/CD Public Key ```bash # Create .ssh directory for SERVICE_USER @@ -1924,7 +1578,7 @@ chmod 600 ~/.ssh/authorized_keys **Important**: Replace `YOUR_CI_CD_PUBLIC_KEY` with the public key from the CI/CD Linode (the output from `cat ~/.ssh/id_ed25519.pub` on the CI/CD Linode). -#### 19.2 Test SSH Connection +#### 18.2 Test SSH Connection From the CI/CD Linode, test the SSH connection: @@ -1934,16 +1588,16 @@ ssh production **Expected output**: You should be able to SSH to the production server without a password prompt. -### Step 20: Test Production Setup +### Step 19: Test Production Setup -#### 20.1 Test Docker Installation +#### 19.1 Test Docker Installation ```bash docker --version docker compose --version ``` -#### 20.2 Test Harbor Access +#### 19.2 Test Harbor Access ```bash # Test pulling an image from the CI/CD Harbor registry @@ -1952,14 +1606,14 @@ docker pull YOUR_CI_CD_IP:8080/public/backend:latest **Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address. -#### 20.3 Test Application Deployment +#### 19.3 Test Application Deployment ```bash cd /opt/APP_NAME docker compose up -d ``` -#### 20.4 Verify Application Status +#### 19.4 Verify Application Status ```bash docker compose ps @@ -1976,9 +1630,9 @@ curl http://localhost:3001/health ## Part 3: Final Configuration and Testing -### Step 21: Configure Forgejo Repository Secrets +### Step 20: Configure Forgejo Repository Secrets -#### 21.1 Required Repository Secrets +#### 20.1 Required Repository Secrets Go to your Forgejo repository and add these secrets in **Settings โ†’ Secrets and Variables โ†’ Actions**: @@ -1994,16 +1648,16 @@ Go to your Forgejo repository and add these secrets in **Settings โ†’ Secrets an - `DOMAIN`: Your domain name (e.g., `example.com`) - `EMAIL`: Your email for SSL certificate notifications -#### 21.2 Configure Forgejo Actions Runner +#### 20.2 Configure Forgejo Actions Runner -##### 21.2.1 Get Runner Token +##### 20.2.1 Get Runner Token 1. Go to your Forgejo repository 2. Navigate to **Settings โ†’ Actions โ†’ Runners** 3. Click **"New runner"** 4. Copy the registration token -##### 21.2.2 Configure Runner +##### 20.2.2 Configure Runner ```bash # Switch to DEPLOY_USER on CI/CD Linode @@ -2022,14 +1676,14 @@ forgejo-runner register \ --no-interactive ``` -##### 21.2.3 Start Runner +##### 20.2.3 Start Runner ```bash sudo systemctl start forgejo-runner.service sudo systemctl status forgejo-runner.service ``` -##### 21.2.4 Test Runner Configuration +##### 20.2.4 Test Runner Configuration ```bash # Check if the runner is running @@ -2053,9 +1707,9 @@ sudo journalctl -u forgejo-runner.service -f --no-pager - Check network: Ensure the runner can reach your Forgejo instance - Restart service: `sudo systemctl restart forgejo-runner.service` -### Step 22: Set Up Monitoring and Cleanup +### Step 21: Set Up Monitoring and Cleanup -#### 22.1 Monitoring Script +#### 21.1 Monitoring Script **Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring. @@ -2079,7 +1733,7 @@ chmod +x scripts/monitor.sh **Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information. -#### 22.2 DinD Cleanup Script +#### 21.2 DinD Cleanup Script **Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment. @@ -2108,7 +1762,7 @@ chmod +x scripts/dind-cleanup.sh - โœ… **Fast execution**: No complex resource scanning needed - โœ… **Reliable**: No risk of accidentally removing Harbor resources -#### 22.3 Test DinD Cleanup Script +#### 21.3 Test DinD Cleanup Script ```bash # Test DinD cleanup with dry run first @@ -2134,7 +1788,7 @@ docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successf - Check DinD logs: `docker logs ci-cd-dind` - Run manually: `bash -x scripts/dind-cleanup.sh` -#### 22.4 Set Up Automated DinD Cleanup +#### 21.4 Set Up Automated DinD Cleanup ```bash # Create a cron job to run DinD cleanup daily at 2 AM @@ -2151,15 +1805,15 @@ crontab -l - **What it cleans**: Restarts DinD container for fresh CI environment - **Zero Harbor impact**: Harbor registry operations are completely unaffected -### Step 23: Test Complete Pipeline +### Step 22: Test Complete Pipeline -#### 23.1 Trigger a Test Build +#### 22.1 Trigger a Test Build 1. **Make a small change** to your repository (e.g., update a comment or add a test file) 2. **Commit and push** the changes to trigger the CI/CD pipeline 3. **Monitor the build** in your Forgejo repository โ†’ Actions tab -#### 23.2 Verify Pipeline Steps +#### 22.2 Verify Pipeline Steps The pipeline should execute these steps in order: @@ -2172,7 +1826,7 @@ The pipeline should execute these steps in order: 7. **Push to Registry**: Push images to Harbor registry from DinD 8. **Deploy to Production**: Deploy to production server -#### 23.3 Check Harbor +#### 22.3 Check Harbor ```bash # On CI/CD Linode @@ -2186,7 +1840,7 @@ curl -k https://localhost:8080/v2/public/backend/tags/list curl -k https://localhost:8080/v2/public/frontend/tags/list ``` -#### 23.4 Verify Production Deployment +#### 22.4 Verify Production Deployment ```bash # On Production Linode @@ -2204,16 +1858,16 @@ docker compose logs backend docker compose logs frontend ``` -#### 23.5 Test Application Functionality +#### 22.5 Test Application Functionality 1. **Frontend**: Visit your production URL (IP or domain) 2. **Backend API**: Test API endpoints 3. **Database**: Verify database connections 4. **Logs**: Check for any errors in application logs -### Step 24: Set Up SSL/TLS (Optional - Domain Users) +### Step 23: Set Up SSL/TLS (Optional - Domain Users) -#### 24.1 Install SSL Certificate +#### 23.1 Install SSL Certificate If you have a domain pointing to your Production Linode: @@ -2225,7 +1879,7 @@ sudo certbot --nginx -d your-domain.com sudo certbot certificates ``` -#### 24.2 Configure Auto-Renewal +#### 23.2 Configure Auto-Renewal ```bash # Test auto-renewal @@ -2237,9 +1891,9 @@ sudo crontab -e # 0 12 * * * /usr/bin/certbot renew --quiet ``` -### Step 25: Final Verification +### Step 24: Final Verification -#### 25.1 Security Check +#### 24.1 Security Check ```bash # Check firewall status @@ -2252,7 +1906,7 @@ sudo systemctl status fail2ban sudo grep "PasswordAuthentication" /etc/ssh/sshd_config ``` -#### 25.2 Performance Check +#### 24.2 Performance Check ```bash # Check system resources @@ -2265,7 +1919,7 @@ df -h docker system df ``` -#### 25.3 Backup Verification +#### 24.3 Backup Verification ```bash # Test backup script @@ -2276,16 +1930,16 @@ cd /opt/APP_NAME ./scripts/backup.sh ``` -### Step 26: Documentation and Maintenance +### Step 25: Documentation and Maintenance -#### 26.1 Update Documentation +#### 25.1 Update Documentation 1. **Update README.md** with deployment information 2. **Document environment variables** and their purposes 3. **Create troubleshooting guide** for common issues 4. **Document backup and restore procedures** -#### 26.2 Set Up Monitoring Alerts +#### 25.2 Set Up Monitoring Alerts ```bash # Set up monitoring cron job @@ -2295,7 +1949,7 @@ cd /opt/APP_NAME tail -f /tmp/monitor.log ``` -#### 26.3 Regular Maintenance Tasks +#### 25.3 Regular Maintenance Tasks **Daily:** - Check application logs for errors diff --git a/scripts/dind-cleanup.sh b/scripts/dind-cleanup.sh new file mode 100755 index 0000000..eeeefa2 --- /dev/null +++ b/scripts/dind-cleanup.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +# Docker-in-Docker Cleanup Script +# This script provides a simple way to clean up the DinD environment +# by restarting the DinD container, which gives a fresh environment. + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +show_help() { + cat << EOF +Docker-in-Docker Cleanup Script + +Usage: $0 [OPTIONS] + +Options: + --dry-run Show what would be done without executing + --help|-h Show this help message + +Examples: + $0 # Clean up DinD environment + $0 --dry-run # Show what would be done +EOF +} + +# Parse command line arguments +DRY_RUN="false" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN="true" + shift + ;; + --help|-h) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Main cleanup function +cleanup_dind() { + echo + echo "==================================================================================" + echo " ๐Ÿงน Docker-in-Docker Cleanup ๐Ÿงน" + echo "==================================================================================" + echo + + # Check if DinD container exists + if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_error "DinD container 'ci-cd-dind' not found!" + log_info "Creating new DinD container..." + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would create DinD container" + return + fi + + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + log_success "DinD container created successfully" + return + fi + + # Check if DinD container is running + if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD container is running" + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would stop and restart DinD container" + log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment" + return + fi + + log_info "Stopping DinD container..." + docker stop ci-cd-dind + + log_info "Removing DinD container..." + docker rm ci-cd-dind + + log_info "Creating fresh DinD container..." + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + # Wait for DinD to start + log_info "Waiting for DinD to start..." + sleep 10 + + # Test DinD connectivity + if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then + log_success "DinD container is ready!" + else + log_error "DinD container failed to start properly" + exit 1 + fi + + else + log_info "DinD container exists but is not running" + + if [ "$DRY_RUN" = "true" ]; then + log_info "DRY RUN: Would remove and recreate DinD container" + return + fi + + log_info "Removing existing DinD container..." + docker rm ci-cd-dind + + log_info "Creating fresh DinD container..." + docker run -d \ + --name ci-cd-dind \ + --privileged \ + --restart unless-stopped \ + -p 2376:2376 \ + -v ci-cd-data:/var/lib/docker \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker:dind + + # Wait for DinD to start + log_info "Waiting for DinD to start..." + sleep 10 + + # Test DinD connectivity + if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then + log_success "DinD container is ready!" + else + log_error "DinD container failed to start properly" + exit 1 + fi + fi + + echo + echo "==================================================================================" + log_success "DinD cleanup completed successfully!" + echo "==================================================================================" + echo + log_info "Benefits of this cleanup:" + log_info " โœ… Fresh Docker environment for CI/CD" + log_info " โœ… No resource contention with Harbor" + log_info " โœ… Clean state for Rust testing" + log_info " โœ… Isolated CI/CD operations" + echo +} + +# Show current DinD status +show_status() { + echo "==================================================================================" + echo " ๐Ÿ“Š DinD Status ๐Ÿ“Š" + echo "==================================================================================" + echo + + if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD Container Status:" + docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + echo + + if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then + log_info "DinD Docker Info:" + docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon" + echo + + log_info "DinD Images:" + docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images" + echo + + log_info "DinD Containers:" + docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers" + else + log_warning "DinD container is not running" + fi + else + log_warning "DinD container does not exist" + fi + + echo "==================================================================================" +} + +# Main execution +if [ "$DRY_RUN" = "true" ]; then + echo + echo "==================================================================================" + echo " ๐Ÿšจ DRY RUN MODE ๐Ÿšจ" + echo " No changes will be made" + echo "==================================================================================" + echo + show_status + cleanup_dind +else + show_status + cleanup_dind +fi \ No newline at end of file