Consolidate DinD cleanup process
Some checks are pending
CI/CD Pipeline (DinD) / Test Backend (DinD) (push) Waiting to run
CI/CD Pipeline (DinD) / Test Frontend (DinD) (push) Waiting to run
CI/CD Pipeline (DinD) / Build and Push Docker Images (DinD) (push) Blocked by required conditions
CI/CD Pipeline (DinD) / Deploy to Production (push) Blocked by required conditions

This commit is contained in:
continuist 2025-06-29 22:18:13 -04:00
parent 86bb5e37b8
commit f11348f209
2 changed files with 328 additions and 442 deletions

View file

@ -1184,256 +1184,38 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/alpine:latest
- **Tests connectivity**: Verifies DinD can pull, tag, and push images to Harbor
- **Validates setup**: Ensures the complete CI/CD pipeline will work
#### 8.3 Create DinD Cleanup Script
#### 8.3 DinD Cleanup Script and Testing
The project includes a comprehensive Docker-in-Docker cleanup script at `scripts/dind-cleanup.sh` that provides a simple way to clean up the DinD environment by restarting the DinD container for a fresh environment.
**Key Features:**
- **🧹 Complete Environment Reset**: Stops, removes, and recreates the `ci-cd-dind` container
- **📊 Status Monitoring**: Shows current DinD container status, Docker info, images, and containers
- **🚨 Dry Run Mode**: Use `--dry-run` to see what would be done without making changes
- **🎨 Colored Output**: Clear, color-coded logging for better readability
- **🛡️ Error Handling**: Robust error checking and graceful failure handling
**Usage:**
```bash
# Create simplified cleanup script for DinD
cat > /opt/APP_NAME/scripts/dind-cleanup.sh << 'EOF'
#!/bin/bash
# Clean up DinD environment
./scripts/dind-cleanup.sh
# Docker-in-Docker Cleanup Script
# This script provides a simple way to clean up the DinD environment
# by restarting the DinD container, which gives a fresh environment.
# See what would be done without executing
./scripts/dind-cleanup.sh --dry-run
set -e
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging functions
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
show_help() {
cat << EOF
Docker-in-Docker Cleanup Script
Usage: $0 [OPTIONS]
Options:
--dry-run Show what would be done without executing
--help|-h Show this help message
Examples:
$0 # Clean up DinD environment
$0 --dry-run # Show what would be done
EOF
}
# Parse command line arguments
DRY_RUN="false"
while [[ $# -gt 0 ]]; do
case $1 in
--dry-run)
DRY_RUN="true"
shift
;;
--help|-h)
show_help
exit 0
;;
*)
log_error "Unknown option: $1"
show_help
exit 1
;;
esac
done
# Main cleanup function
cleanup_dind() {
echo
echo "=================================================================================="
echo " 🧹 Docker-in-Docker Cleanup 🧹"
echo "=================================================================================="
echo
# Check if DinD container exists
if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_error "DinD container 'ci-cd-dind' not found!"
log_info "Creating new DinD container..."
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would create DinD container"
return
fi
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
log_success "DinD container created successfully"
return
fi
# Check if DinD container is running
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD container is running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would stop and restart DinD container"
log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment"
return
fi
log_info "Stopping DinD container..."
docker stop ci-cd-dind
log_info "Removing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
else
log_info "DinD container exists but is not running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would remove and recreate DinD container"
return
fi
log_info "Removing existing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
fi
echo
echo "=================================================================================="
log_success "DinD cleanup completed successfully!"
echo "=================================================================================="
echo
log_info "Benefits of this cleanup:"
log_info " ✅ Fresh Docker environment for CI/CD"
log_info " ✅ No resource contention with Harbor"
log_info " ✅ Clean state for Rust testing"
log_info " ✅ Isolated CI/CD operations"
echo
}
# Show current DinD status
show_status() {
echo "=================================================================================="
echo " 📊 DinD Status 📊"
echo "=================================================================================="
echo
if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Container Status:"
docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
echo
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Docker Info:"
docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon"
echo
log_info "DinD Images:"
docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images"
echo
log_info "DinD Containers:"
docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers"
else
log_warning "DinD container is not running"
fi
else
log_warning "DinD container does not exist"
fi
echo "=================================================================================="
}
# Main execution
if [ "$DRY_RUN" = "true" ]; then
echo
echo "=================================================================================="
echo " 🚨 DRY RUN MODE 🚨"
echo " No changes will be made"
echo "=================================================================================="
echo
show_status
cleanup_dind
else
show_status
cleanup_dind
fi
EOF
# Make the script executable
chmod +x /opt/APP_NAME/scripts/dind-cleanup.sh
# Show help
./scripts/dind-cleanup.sh --help
```
**What this does**:
- **Creates cleanup script**: Simple script to restart DinD container for fresh environment
- **Status monitoring**: Shows current DinD container and Docker state
- **Dry-run mode**: Allows testing without making changes
- **Error handling**: Proper error checking and user feedback
**Benefits:**
- ✅ Fresh Docker environment for CI/CD
- ✅ No resource contention with Harbor
- ✅ Clean state for Rust testing
- ✅ Isolated CI/CD operations
#### 8.4 Test DinD Setup
The script is already part of the project repository and ready to use for maintaining clean CI/CD environments.
**Testing DinD Setup:**
```bash
# Test DinD cleanup script
@ -1457,7 +1239,7 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest
- Harbor push/pull should work from DinD
- Cleanup script should show proper status
#### 8.5 Set Up Automated DinD Cleanup
#### 8.4 Set Up Automated DinD Cleanup
```bash
# Create a cron job to run DinD cleanup daily at 2 AM
@ -1467,15 +1249,13 @@ docker exec ci-cd-dind docker rmi localhost:5000/test/dind-test:latest
crontab -l
```
**What this does**:
**What this does:**
- **Automated cleanup**: Restarts DinD container daily for fresh environment
- **Prevents resource buildup**: Clears CI/CD artifacts automatically
- **Maintains performance**: Ensures consistent CI/CD performance
- **Zero Harbor impact**: DinD cleanup doesn't affect Harbor operations
### Step 9: Set Up Monitoring and Cleanup
#### 9.1 Monitoring Script
#### 8.5 Monitoring Script
**Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring.
@ -1499,133 +1279,7 @@ chmod +x scripts/monitor.sh
**Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information.
#### 9.2 DinD Cleanup Script
**Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment.
**DinD Cleanup Script**:
- `scripts/dind-cleanup.sh` - Simple script to restart DinD container for fresh CI environment
**To use the DinD cleanup script**:
```bash
# The repository is already cloned at /opt/APP_NAME/
cd /opt/APP_NAME
# Make the script executable
chmod +x scripts/dind-cleanup.sh
# Test DinD cleanup (dry run first)
./scripts/dind-cleanup.sh --dry-run
# Run DinD cleanup
./scripts/dind-cleanup.sh
```
**Benefits of DinD cleanup**:
- ✅ **Simple operation**: Just restart the DinD container
- ✅ **Zero Harbor impact**: Harbor registry is completely unaffected
- ✅ **Fresh environment**: Every cleanup gives a completely clean state
- ✅ **Fast execution**: No complex resource scanning needed
- ✅ **Reliable**: No risk of accidentally removing Harbor resources
#### 9.3 Test DinD Cleanup Script
```bash
# Test DinD cleanup with dry run first
./scripts/dind-cleanup.sh --dry-run
# Run the DinD cleanup script
./scripts/dind-cleanup.sh
# Verify DinD is working after cleanup
docker exec ci-cd-dind docker version
docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successful!"
```
**Expected Output**:
- DinD cleanup script should run without errors
- DinD container should be restarted with fresh environment
- Docker commands should work inside DinD after cleanup
- Harbor registry should remain completely unaffected
**If something goes wrong**:
- Check script permissions: `ls -la scripts/dind-cleanup.sh`
- Verify DinD container: `docker ps | grep ci-cd-dind`
- Check DinD logs: `docker logs ci-cd-dind`
- Run manually: `bash -x scripts/dind-cleanup.sh`
#### 9.4 Set Up Automated DinD Cleanup
```bash
# Create a cron job to run DinD cleanup daily at 2 AM
(crontab -l 2>/dev/null; echo "0 2 * * * cd /opt/APP_NAME && ./scripts/dind-cleanup.sh >> /tmp/dind-cleanup.log 2>&1") | crontab -
# Verify the cron job was added
crontab -l
```
**What this does:**
- **Runs automatically**: The DinD cleanup script runs every day at 2:00 AM
- **Frequency**: Daily cleanup to prevent CI/CD resource buildup
- **Logging**: All cleanup output is logged to `/tmp/dind-cleanup.log`
- **What it cleans**: Restarts DinD container for fresh CI environment
- **Zero Harbor impact**: Harbor registry operations are completely unaffected
#### 9.5 Test Cleanup Script
```bash
# Create some test images to clean up
docker pull alpine:latest
docker pull nginx:latest
docker tag alpine:latest test-cleanup:latest
docker tag nginx:latest test-cleanup2:latest
# Test cleanup with dry run first
./scripts/cleanup.sh --type ci-cd --dry-run
# Run the cleanup script
./scripts/cleanup.sh --type ci-cd
# Verify cleanup worked
echo "Checking remaining images:"
docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}"
echo "Checking remaining volumes:"
docker volume ls
echo "Checking remaining networks:"
docker network ls
```
**Expected Output**:
- Cleanup script should run without errors
- Test images should be removed
- System should report cleanup completion
- Remaining images should be minimal (only actively used ones)
**If something goes wrong**:
- Check script permissions: `ls -la scripts/cleanup.sh`
- Verify Docker access: `docker ps`
- Check registry access: `cd /opt/APP_NAME/registry && docker compose ps`
- Run manually: `bash -x scripts/cleanup.sh`
#### 9.6 Set Up Automated Cleanup
```bash
# Create a cron job to run cleanup daily at 3 AM using the repository script
(crontab -l 2>/dev/null; echo "0 3 * * * cd /opt/APP_NAME && ./scripts/cleanup.sh --type ci-cd >> /tmp/cleanup.log 2>&1") | crontab -
# Verify the cron job was added
crontab -l
```
**What this does:**
- **Runs automatically**: The cleanup script runs every day at 3:00 AM
- **Frequency**: Daily cleanup to prevent disk space issues
- **Logging**: All cleanup output is logged to `/tmp/cleanup.log`
- **What it cleans**: Unused Docker images, volumes, networks, and Harbor images
### Step 10: Configure Firewall
### Step 9: Configure Firewall
```bash
sudo ufw --force enable
@ -1640,23 +1294,23 @@ sudo ufw allow 443/tcp # Harbor registry (public read access)
- **SSH**: Restricted to your IP addresses
- **All other ports**: Blocked
### Step 11: Test CI/CD Setup
### Step 10: Test CI/CD Setup
#### 11.1 Test Docker Installation
#### 10.1 Test Docker Installation
```bash
docker --version
docker compose --version
```
#### 11.2 Check Harbor Status
#### 10.2 Check Harbor Status
```bash
cd /opt/APP_NAME/registry
docker compose ps
```
#### 11.3 Test Harbor Access
#### 10.3 Test Harbor Access
```bash
# Test Harbor API
@ -1666,7 +1320,7 @@ curl -k https://localhost:8080/api/v2.0/health
curl -k -I https://localhost
```
#### 11.4 Get Public Key for Production Server
#### 10.4 Get Public Key for Production Server
```bash
cat ~/.ssh/id_ed25519.pub
@ -1678,15 +1332,15 @@ cat ~/.ssh/id_ed25519.pub
## Part 2: Production Linode Setup
### Step 12: Initial System Setup
### Step 11: Initial System Setup
#### 12.1 Update the System
#### 11.1 Update the System
```bash
sudo apt update && sudo apt upgrade -y
```
#### 12.2 Configure Timezone
#### 11.2 Configure Timezone
```bash
# Configure timezone interactively
@ -1700,7 +1354,7 @@ date
**Expected output**: After selecting your timezone, the `date` command should show the current date and time in your selected timezone.
#### 12.3 Configure /etc/hosts
#### 11.3 Configure /etc/hosts
```bash
# Add localhost entries for both IPv4 and IPv6
@ -1721,7 +1375,7 @@ cat /etc/hosts
**Expected output**: The `/etc/hosts` file should show entries for `127.0.0.1`, `::1`, and your Linode's actual IP addresses all mapping to `localhost`.
#### 12.4 Install Essential Packages
#### 11.4 Install Essential Packages
```bash
sudo apt install -y \
@ -1739,9 +1393,9 @@ sudo apt install -y \
python3-certbot-nginx
```
### Step 13: Create Users
### Step 12: Create Users
#### 13.1 Create the SERVICE_USER User
#### 12.1 Create the SERVICE_USER User
```bash
# Create dedicated group for the service account
@ -1752,7 +1406,7 @@ sudo useradd -r -g SERVICE_USER -s /bin/bash -m -d /home/SERVICE_USER SERVICE_US
echo "SERVICE_USER:$(openssl rand -base64 32)" | sudo chpasswd
```
#### 13.2 Create the DEPLOY_USER User
#### 12.2 Create the DEPLOY_USER User
```bash
# Create deployment user
@ -1761,7 +1415,7 @@ sudo usermod -aG sudo DEPLOY_USER
echo "DEPLOY_USER:$(openssl rand -base64 32)" | sudo chpasswd
```
#### 13.3 Verify Users
#### 12.3 Verify Users
```bash
sudo su - SERVICE_USER
@ -1775,9 +1429,9 @@ pwd
exit
```
### Step 14: Install Docker
### Step 13: Install Docker
#### 14.1 Add Docker Repository
#### 13.1 Add Docker Repository
```bash
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
@ -1785,28 +1439,28 @@ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docke
sudo apt update
```
#### 14.2 Install Docker Packages
#### 13.2 Install Docker Packages
```bash
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
```
#### 14.3 Configure Docker for Service Account
#### 13.3 Configure Docker for Service Account
```bash
sudo usermod -aG docker SERVICE_USER
```
### Step 15: Install Docker Compose
### Step 14: Install Docker Compose
```bash
sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
```
### Step 16: Configure Security
### Step 15: Configure Security
#### 16.1 Configure Firewall
#### 15.1 Configure Firewall
```bash
sudo ufw --force enable
@ -1819,16 +1473,16 @@ sudo ufw allow 3000/tcp
sudo ufw allow 3001/tcp
```
#### 16.2 Configure Fail2ban
#### 15.2 Configure Fail2ban
```bash
sudo systemctl enable fail2ban
sudo systemctl start fail2ban
```
### Step 17: Create Application Directory
### Step 16: Create Application Directory
#### 17.1 Create Directory Structure
#### 16.1 Create Directory Structure
```bash
sudo mkdir -p /opt/APP_NAME
@ -1837,22 +1491,22 @@ sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME
**Note**: Replace `APP_NAME` with your actual application name. This directory name can be controlled via the `APP_NAME` secret in your Forgejo repository settings. If you set the `APP_NAME` secret to `myapp`, the deployment directory will be `/opt/myapp`.
#### 17.2 Create SSL Directory (Optional - for domain users)
#### 16.2 Create SSL Directory (Optional - for domain users)
```bash
sudo mkdir -p /opt/APP_NAME/nginx/ssl
sudo chown SERVICE_USER:SERVICE_USER /opt/APP_NAME/nginx/ssl
```
### Step 18: Clone Repository and Set Up Application Files
### Step 17: Clone Repository and Set Up Application Files
#### 18.1 Switch to SERVICE_USER User
#### 17.1 Switch to SERVICE_USER User
```bash
sudo su - SERVICE_USER
```
#### 18.2 Clone Repository
#### 17.2 Clone Repository
```bash
cd /opt/APP_NAME
@ -1865,7 +1519,7 @@ git clone https://your-forgejo-instance/your-username/APP_NAME.git .
**Note**: Replace `your-forgejo-instance` and `your-username/APP_NAME` with your actual Forgejo instance URL and repository path.
#### 18.3 Create Environment File
#### 17.3 Create Environment File
The repository doesn't include a `.env.example` file for security reasons. The CI/CD pipeline will create the `.env` file dynamically during deployment. However, for manual testing or initial setup, you can create a basic `.env` file:
@ -1891,7 +1545,7 @@ EOF
**Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address and `your_secure_password_here` with a strong password.
#### 18.4 Configure Docker for Harbor Access
#### 17.4 Configure Docker for Harbor Access
```bash
# Add the CI/CD Harbor registry to Docker's insecure registries
@ -1908,9 +1562,9 @@ sudo systemctl restart docker
**Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address.
### Step 19: Set Up SSH Key Authentication
### Step 18: Set Up SSH Key Authentication
#### 19.1 Add CI/CD Public Key
#### 18.1 Add CI/CD Public Key
```bash
# Create .ssh directory for SERVICE_USER
@ -1924,7 +1578,7 @@ chmod 600 ~/.ssh/authorized_keys
**Important**: Replace `YOUR_CI_CD_PUBLIC_KEY` with the public key from the CI/CD Linode (the output from `cat ~/.ssh/id_ed25519.pub` on the CI/CD Linode).
#### 19.2 Test SSH Connection
#### 18.2 Test SSH Connection
From the CI/CD Linode, test the SSH connection:
@ -1934,16 +1588,16 @@ ssh production
**Expected output**: You should be able to SSH to the production server without a password prompt.
### Step 20: Test Production Setup
### Step 19: Test Production Setup
#### 20.1 Test Docker Installation
#### 19.1 Test Docker Installation
```bash
docker --version
docker compose --version
```
#### 20.2 Test Harbor Access
#### 19.2 Test Harbor Access
```bash
# Test pulling an image from the CI/CD Harbor registry
@ -1952,14 +1606,14 @@ docker pull YOUR_CI_CD_IP:8080/public/backend:latest
**Important**: Replace `YOUR_CI_CD_IP` with your actual CI/CD Linode IP address.
#### 20.3 Test Application Deployment
#### 19.3 Test Application Deployment
```bash
cd /opt/APP_NAME
docker compose up -d
```
#### 20.4 Verify Application Status
#### 19.4 Verify Application Status
```bash
docker compose ps
@ -1976,9 +1630,9 @@ curl http://localhost:3001/health
## Part 3: Final Configuration and Testing
### Step 21: Configure Forgejo Repository Secrets
### Step 20: Configure Forgejo Repository Secrets
#### 21.1 Required Repository Secrets
#### 20.1 Required Repository Secrets
Go to your Forgejo repository and add these secrets in **Settings → Secrets and Variables → Actions**:
@ -1994,16 +1648,16 @@ Go to your Forgejo repository and add these secrets in **Settings → Secrets an
- `DOMAIN`: Your domain name (e.g., `example.com`)
- `EMAIL`: Your email for SSL certificate notifications
#### 21.2 Configure Forgejo Actions Runner
#### 20.2 Configure Forgejo Actions Runner
##### 21.2.1 Get Runner Token
##### 20.2.1 Get Runner Token
1. Go to your Forgejo repository
2. Navigate to **Settings → Actions → Runners**
3. Click **"New runner"**
4. Copy the registration token
##### 21.2.2 Configure Runner
##### 20.2.2 Configure Runner
```bash
# Switch to DEPLOY_USER on CI/CD Linode
@ -2022,14 +1676,14 @@ forgejo-runner register \
--no-interactive
```
##### 21.2.3 Start Runner
##### 20.2.3 Start Runner
```bash
sudo systemctl start forgejo-runner.service
sudo systemctl status forgejo-runner.service
```
##### 21.2.4 Test Runner Configuration
##### 20.2.4 Test Runner Configuration
```bash
# Check if the runner is running
@ -2053,9 +1707,9 @@ sudo journalctl -u forgejo-runner.service -f --no-pager
- Check network: Ensure the runner can reach your Forgejo instance
- Restart service: `sudo systemctl restart forgejo-runner.service`
### Step 22: Set Up Monitoring and Cleanup
### Step 21: Set Up Monitoring and Cleanup
#### 22.1 Monitoring Script
#### 21.1 Monitoring Script
**Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring.
@ -2079,7 +1733,7 @@ chmod +x scripts/monitor.sh
**Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information.
#### 22.2 DinD Cleanup Script
#### 21.2 DinD Cleanup Script
**Important**: With the DinD setup, CI/CD operations are isolated in the DinD container. This means we can use a much simpler cleanup approach - just restart the DinD container for a fresh environment.
@ -2108,7 +1762,7 @@ chmod +x scripts/dind-cleanup.sh
- ✅ **Fast execution**: No complex resource scanning needed
- ✅ **Reliable**: No risk of accidentally removing Harbor resources
#### 22.3 Test DinD Cleanup Script
#### 21.3 Test DinD Cleanup Script
```bash
# Test DinD cleanup with dry run first
@ -2134,7 +1788,7 @@ docker exec ci-cd-dind docker run --rm alpine:latest echo "DinD cleanup successf
- Check DinD logs: `docker logs ci-cd-dind`
- Run manually: `bash -x scripts/dind-cleanup.sh`
#### 22.4 Set Up Automated DinD Cleanup
#### 21.4 Set Up Automated DinD Cleanup
```bash
# Create a cron job to run DinD cleanup daily at 2 AM
@ -2151,15 +1805,15 @@ crontab -l
- **What it cleans**: Restarts DinD container for fresh CI environment
- **Zero Harbor impact**: Harbor registry operations are completely unaffected
### Step 23: Test Complete Pipeline
### Step 22: Test Complete Pipeline
#### 23.1 Trigger a Test Build
#### 22.1 Trigger a Test Build
1. **Make a small change** to your repository (e.g., update a comment or add a test file)
2. **Commit and push** the changes to trigger the CI/CD pipeline
3. **Monitor the build** in your Forgejo repository → Actions tab
#### 23.2 Verify Pipeline Steps
#### 22.2 Verify Pipeline Steps
The pipeline should execute these steps in order:
@ -2172,7 +1826,7 @@ The pipeline should execute these steps in order:
7. **Push to Registry**: Push images to Harbor registry from DinD
8. **Deploy to Production**: Deploy to production server
#### 23.3 Check Harbor
#### 22.3 Check Harbor
```bash
# On CI/CD Linode
@ -2186,7 +1840,7 @@ curl -k https://localhost:8080/v2/public/backend/tags/list
curl -k https://localhost:8080/v2/public/frontend/tags/list
```
#### 23.4 Verify Production Deployment
#### 22.4 Verify Production Deployment
```bash
# On Production Linode
@ -2204,16 +1858,16 @@ docker compose logs backend
docker compose logs frontend
```
#### 23.5 Test Application Functionality
#### 22.5 Test Application Functionality
1. **Frontend**: Visit your production URL (IP or domain)
2. **Backend API**: Test API endpoints
3. **Database**: Verify database connections
4. **Logs**: Check for any errors in application logs
### Step 24: Set Up SSL/TLS (Optional - Domain Users)
### Step 23: Set Up SSL/TLS (Optional - Domain Users)
#### 24.1 Install SSL Certificate
#### 23.1 Install SSL Certificate
If you have a domain pointing to your Production Linode:
@ -2225,7 +1879,7 @@ sudo certbot --nginx -d your-domain.com
sudo certbot certificates
```
#### 24.2 Configure Auto-Renewal
#### 23.2 Configure Auto-Renewal
```bash
# Test auto-renewal
@ -2237,9 +1891,9 @@ sudo crontab -e
# 0 12 * * * /usr/bin/certbot renew --quiet
```
### Step 25: Final Verification
### Step 24: Final Verification
#### 25.1 Security Check
#### 24.1 Security Check
```bash
# Check firewall status
@ -2252,7 +1906,7 @@ sudo systemctl status fail2ban
sudo grep "PasswordAuthentication" /etc/ssh/sshd_config
```
#### 25.2 Performance Check
#### 24.2 Performance Check
```bash
# Check system resources
@ -2265,7 +1919,7 @@ df -h
docker system df
```
#### 25.3 Backup Verification
#### 24.3 Backup Verification
```bash
# Test backup script
@ -2276,16 +1930,16 @@ cd /opt/APP_NAME
./scripts/backup.sh
```
### Step 26: Documentation and Maintenance
### Step 25: Documentation and Maintenance
#### 26.1 Update Documentation
#### 25.1 Update Documentation
1. **Update README.md** with deployment information
2. **Document environment variables** and their purposes
3. **Create troubleshooting guide** for common issues
4. **Document backup and restore procedures**
#### 26.2 Set Up Monitoring Alerts
#### 25.2 Set Up Monitoring Alerts
```bash
# Set up monitoring cron job
@ -2295,7 +1949,7 @@ cd /opt/APP_NAME
tail -f /tmp/monitor.log
```
#### 26.3 Regular Maintenance Tasks
#### 25.3 Regular Maintenance Tasks
**Daily:**
- Check application logs for errors

232
scripts/dind-cleanup.sh Executable file
View file

@ -0,0 +1,232 @@
#!/bin/bash
# Docker-in-Docker Cleanup Script
# This script provides a simple way to clean up the DinD environment
# by restarting the DinD container, which gives a fresh environment.
set -e
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging functions
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
show_help() {
cat << EOF
Docker-in-Docker Cleanup Script
Usage: $0 [OPTIONS]
Options:
--dry-run Show what would be done without executing
--help|-h Show this help message
Examples:
$0 # Clean up DinD environment
$0 --dry-run # Show what would be done
EOF
}
# Parse command line arguments
DRY_RUN="false"
while [[ $# -gt 0 ]]; do
case $1 in
--dry-run)
DRY_RUN="true"
shift
;;
--help|-h)
show_help
exit 0
;;
*)
log_error "Unknown option: $1"
show_help
exit 1
;;
esac
done
# Main cleanup function
cleanup_dind() {
echo
echo "=================================================================================="
echo " 🧹 Docker-in-Docker Cleanup 🧹"
echo "=================================================================================="
echo
# Check if DinD container exists
if ! docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_error "DinD container 'ci-cd-dind' not found!"
log_info "Creating new DinD container..."
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would create DinD container"
return
fi
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
log_success "DinD container created successfully"
return
fi
# Check if DinD container is running
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD container is running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would stop and restart DinD container"
log_info "DRY RUN: This would clear all CI/CD artifacts and give fresh environment"
return
fi
log_info "Stopping DinD container..."
docker stop ci-cd-dind
log_info "Removing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
else
log_info "DinD container exists but is not running"
if [ "$DRY_RUN" = "true" ]; then
log_info "DRY RUN: Would remove and recreate DinD container"
return
fi
log_info "Removing existing DinD container..."
docker rm ci-cd-dind
log_info "Creating fresh DinD container..."
docker run -d \
--name ci-cd-dind \
--privileged \
--restart unless-stopped \
-p 2376:2376 \
-v ci-cd-data:/var/lib/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
docker:dind
# Wait for DinD to start
log_info "Waiting for DinD to start..."
sleep 10
# Test DinD connectivity
if timeout 30 bash -c 'until docker exec ci-cd-dind docker version >/dev/null 2>&1; do sleep 1; done'; then
log_success "DinD container is ready!"
else
log_error "DinD container failed to start properly"
exit 1
fi
fi
echo
echo "=================================================================================="
log_success "DinD cleanup completed successfully!"
echo "=================================================================================="
echo
log_info "Benefits of this cleanup:"
log_info " ✅ Fresh Docker environment for CI/CD"
log_info " ✅ No resource contention with Harbor"
log_info " ✅ Clean state for Rust testing"
log_info " ✅ Isolated CI/CD operations"
echo
}
# Show current DinD status
show_status() {
echo "=================================================================================="
echo " 📊 DinD Status 📊"
echo "=================================================================================="
echo
if docker ps -a --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Container Status:"
docker ps -a --filter "name=ci-cd-dind" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
echo
if docker ps --format "{{.Names}}" | grep -q "^ci-cd-dind$"; then
log_info "DinD Docker Info:"
docker exec ci-cd-dind docker info --format "{{.ServerVersion}}" 2>/dev/null || log_warning "Cannot connect to DinD Docker daemon"
echo
log_info "DinD Images:"
docker exec ci-cd-dind docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" 2>/dev/null || log_warning "Cannot list DinD images"
echo
log_info "DinD Containers:"
docker exec ci-cd-dind docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || log_warning "Cannot list DinD containers"
else
log_warning "DinD container is not running"
fi
else
log_warning "DinD container does not exist"
fi
echo "=================================================================================="
}
# Main execution
if [ "$DRY_RUN" = "true" ]; then
echo
echo "=================================================================================="
echo " 🚨 DRY RUN MODE 🚨"
echo " No changes will be made"
echo "=================================================================================="
echo
show_status
cleanup_dind
else
show_status
cleanup_dind
fi