From f607d93d21f3e0ebc98629e0062137e513f3cda2 Mon Sep 17 00:00:00 2001 From: continuist Date: Sat, 23 Aug 2025 23:07:29 -0400 Subject: [PATCH] Changes to podman config for running outside home folder --- .forgejo/workflows/ci.yml | 2 +- CI_CD_PIPELINE_SETUP_GUIDE.md | 248 ++++++++++++++++--------------- registry/README.md | 23 ++- registry/docker-registry.service | 16 +- registry/nginx.conf | 43 +++--- registry/registry-pod.yaml | 4 +- 6 files changed, 183 insertions(+), 153 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index b8fba32..a6732ca 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - name: Checkout code to workspace run: | - # Use the pre-configured workspace directory (created in CI guide Step 7.3) + # Use the pre-configured workspace directory (created in CI guide Step 6.3) # Clone the repository to workspace rm -rf /tmp/ci-workspace /tmp/ci-workspace/.* 2>/dev/null || true diff --git a/CI_CD_PIPELINE_SETUP_GUIDE.md b/CI_CD_PIPELINE_SETUP_GUIDE.md index 356f425..90afab5 100644 --- a/CI_CD_PIPELINE_SETUP_GUIDE.md +++ b/CI_CD_PIPELINE_SETUP_GUIDE.md @@ -581,6 +581,7 @@ sudo apt install -y \ ```bash # Install Podman and related tools +# TODO: Remove podman-compose from CI guide if it's determined it's no longer needed on the CI server sudo apt install -y podman podman-compose # Verify installation @@ -651,29 +652,11 @@ ls -la /opt/APP_NAME/registry/ - CI_SERVICE_USER owns all the files for security - Registry configuration files are now available at `/opt/APP_NAME/registry/` -### Step 4: Configure Podman for CI Service Account - -#### 4.1 Verify Podman Installation - -```bash -# Podman should already be installed from Step 1.5 -podman --version -podman-compose --version -``` - -#### 4.2 Configure Podman for CI Service Account - -```bash -# Podman can run rootless, so no group membership needed -# But we'll configure it for the CI service account -sudo usermod -aG podman CI_SERVICE_USER -``` - -### Step 5: Set Up Docker Registry v2 with nginx +### Step 4: Set Up Docker Registry v2 with nginx We'll set up a basic Docker Registry v2 with nginx as a reverse proxy, configured to allow unauthenticated pulls but require authentication for pushes. -#### 5.1 Configure FHS-Compliant Registry Directories +#### 4.1 Configure FHS-Compliant Registry Directories ```bash # Create FHS-compliant directories for registry data and certificates @@ -688,15 +671,12 @@ sudo chmod 755 /etc/registry/certs sudo chmod 755 /var/log/registry ``` -#### 5.2 Create Docker Registry v2 Pod Setup +#### 4.2 Create Docker Registry v2 Pod Setup ```bash # Navigate to the cloned application directory cd /opt/APP_NAME/registry -# Update nginx.conf with your actual IP address -sudo sed -i "s/YOUR_CI_CD_IP/YOUR_ACTUAL_IP_ADDRESS/g" /opt/APP_NAME/registry/nginx.conf - # Update openssl.conf with your actual IP address and registry name sudo sed -i "s/YOUR_CI_CD_IP/YOUR_ACTUAL_IP_ADDRESS/g" /opt/APP_NAME/registry/openssl.conf sudo sed -i "s/YOUR_REGISTRY_NAME/APP_NAME-Registry/g" /opt/APP_NAME/registry/openssl.conf @@ -704,7 +684,7 @@ sudo sed -i "s/YOUR_REGISTRY_NAME/APP_NAME-Registry/g" /opt/APP_NAME/registry/op -#### 5.2.1 Security Features Applied +#### 4.2.1 Security Features Applied The Docker Registry v2 setup includes comprehensive security hardening: @@ -732,7 +712,7 @@ The Docker Registry v2 setup includes comprehensive security hardening: - ✅ Container policy enforcement via containers-policy.json - ✅ Volume mounts with read-only where possible -#### 5.2.2 Set Up Authentication and Permissions +#### 4.2.2 Set Up Authentication and Permissions ```bash # Create FHS-compliant authentication directory @@ -762,7 +742,7 @@ sudo chmod 644 /opt/APP_NAME/registry/registry-pod.yaml sudo chmod 644 /opt/APP_NAME/registry/containers-policy.json ``` -#### 5.2.3 Create FHS-Compliant Directory Structure +#### 4.2.3 Create FHS-Compliant Directory Structure ```bash # Create FHS-compliant certificate directory structure @@ -771,23 +751,22 @@ sudo mkdir -p /etc/registry/certs/requests sudo mkdir -p /etc/registry/certs/ca sudo mkdir -p /var/lib/registry/data -# Set proper ownership for certificate and auth directories -sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs -sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/auth +# Set proper ownership for registry data directory sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /var/lib/registry/data -# Set proper permissions for certificate directories -sudo chmod 755 /etc/registry/certs +# Set proper ownership and permissions for certificate subdirectories +sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/private +sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/requests +sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/ca sudo chmod 700 /etc/registry/certs/private # Private keys - restricted access sudo chmod 755 /etc/registry/certs/requests # Certificate requests sudo chmod 755 /etc/registry/certs/ca # CA certificates sudo chmod 755 /var/lib/registry/data # Registry data -# Create registry data directory symlink for docker-compose -sudo ln -sf /var/lib/registry/data /opt/APP_NAME/registry/registry + ``` -#### 5.2.4 Generate TLS Certificate and Install for Podman +#### 4.2.4 Generate TLS Certificate and Install for Podman **Choose one of the following options based on whether you have a domain name:** @@ -838,10 +817,16 @@ sudo chmod 644 requests/registry.csr requests/openssl.conf # Requests - world r # Verify certificate creation sudo -u CI_SERVICE_USER openssl x509 -in /etc/registry/certs/registry.crt -text -noout | grep -E "(Subject:|DNS:|IP Address:)" -# 2. Install CA certificate for Podman -sudo mkdir -p /etc/containers/certs.d/YOUR_ACTUAL_IP_ADDRESS -sudo cp /etc/registry/certs/ca/ca.crt /etc/containers/certs.d/YOUR_ACTUAL_IP_ADDRESS/ca.crt +# Create required directories for containers +sudo mkdir -p /var/log/nginx +sudo mkdir -p /tmp/registry-tmp /tmp/nginx-tmp +sudo chown CI_SERVICE_USER:CI_SERVICE_USER /var/log/nginx /tmp/registry-tmp /tmp/nginx-tmp +sudo chmod 755 /var/log/nginx /tmp/registry-tmp /tmp/nginx-tmp + +# 2. Install CA certificate in system trust store (for curl, wget, etc.) sudo cp /etc/registry/certs/ca/ca.crt /usr/local/share/ca-certificates/registry-ca.crt + +# This step should complete with "1 added, 0 removed". If it does not, there could be a problem with the certificate you generated, or you might already have a certificate in the trust store sudo update-ca-certificates ``` @@ -863,14 +848,14 @@ sudo certbot certonly --standalone \ sudo certbot certificates sudo cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/fullchain.pem /etc/registry/certs/registry.crt sudo cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/privkey.pem /etc/registry/certs/registry.key -sudo chown CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/registry.crt -sudo chown CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/registry.key +sudo chown CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/registry.crt /etc/registry/certs/registry.key sudo chmod 644 /etc/registry/certs/registry.crt sudo chmod 600 /etc/registry/certs/registry.key -# 2. Install certificate for Podman -sudo mkdir -p /etc/containers/certs.d/YOUR_DOMAIN_NAME -sudo cp /etc/registry/certs/registry.crt /etc/containers/certs.d/YOUR_DOMAIN_NAME/ca.crt +# 2. Install certificate in system trust store (for curl, wget, etc.) +# Note: Let's Encrypt certificates are already trusted by the system, but we install for consistency +sudo cp /etc/registry/certs/registry.crt /usr/local/share/ca-certificates/registry-ca.crt +sudo update-ca-certificates ``` --- @@ -881,32 +866,30 @@ sudo cp /etc/registry/certs/registry.crt /etc/containers/certs.d/YOUR_DOMAIN_NAM --- -**After completing the steps for your chosen option, continue with Step 5.5 (Set Up Systemd Service for Docker Registry v2).** +**After completing the steps for your chosen option, continue with Step 4.5 (Set Up Systemd Service for Docker Registry v2).** -#### 5.3 Install Certificate for Podman (Option B Only) +#### 4.3 Install Certificate for Podman (Option B Only) -**Important**: This step adds the Let's Encrypt certificate for Podman. Since Let's Encrypt is a trusted CA, Podman will automatically trust this certificate. +**Important**: This step adds the Let's Encrypt certificate to the system trust store. Since Let's Encrypt is a trusted CA, system tools will automatically trust this certificate. ```bash -# Create Podman certificates directory for your domain -sudo mkdir -p /etc/containers/certs.d/YOUR_DOMAIN_NAME - -# Copy certificate for Podman -sudo cp /opt/registry/certs/registry.crt /etc/containers/certs.d/YOUR_DOMAIN_NAME/ca.crt +# Install certificate in system trust store (for curl, wget, etc.) +sudo cp /etc/registry/certs/registry.crt /usr/local/share/ca-certificates/registry-ca.crt +sudo update-ca-certificates # Verify certificate installation -if [ -f "/etc/containers/certs.d/YOUR_DOMAIN_NAME/ca.crt" ]; then - echo "✅ Let's Encrypt certificate installed for Podman" +if [ -f "/usr/local/share/ca-certificates/registry-ca.crt" ]; then + echo "✅ Let's Encrypt certificate installed in system trust store" else - echo "❌ Failed to install certificate for Podman" + echo "❌ Failed to install certificate in system trust store" exit 1 fi echo "Certificate installation completed successfully!" -echo "Podman can now connect to the registry securely using your domain name" +echo "System tools can now connect to the registry securely using your domain name" ``` -#### 5.4 Set Up Automatic Certificate Renewal (Option B Only) +#### 4.4 Set Up Automatic Certificate Renewal (Option B Only) **Important**: Let's Encrypt certificates expire after 90 days, so we need to set up automatic renewal. @@ -917,39 +900,67 @@ sudo certbot renew --dry-run # Set up automatic renewal cron job sudo crontab -e # Add this line to renew certificates twice daily (Let's Encrypt allows renewal 30 days before expiry): -# 0 12,18 * * * /usr/bin/certbot renew --quiet --post-hook "cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/fullchain.pem /etc/registry/certs/registry.crt && cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/privkey.pem /etc/registry/certs/registry.key && chown CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/registry.* && chmod 644 /etc/registry/certs/registry.crt && chmod 600 /etc/registry/certs/registry.key && systemctl restart docker-registry.service" +# 0 12,18 * * * /usr/bin/certbot renew --quiet --post-hook "cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/fullchain.pem /etc/registry/certs/registry.crt && cp /etc/letsencrypt/live/YOUR_DOMAIN_NAME/privkey.pem /etc/registry/certs/registry.key && chown CI_SERVICE_USER:CI_SERVICE_USER /etc/registry/certs/registry.* && chmod 644 /etc/registry/certs/registry.crt && chmod 600 /etc/registry/certs/registry.key && cp /etc/registry/certs/registry.crt /usr/local/share/ca-certificates/registry-ca.crt && update-ca-certificates && systemctl restart docker-registry.service" echo "Automatic certificate renewal configured!" echo "Certificates will be renewed automatically and the registry service will be restarted" ``` -#### 5.5 Set Up Systemd Service for Docker Registry v2 +#### 4.5 Set Up Systemd Service for Docker Registry v2 ```bash -# Create system-wide Podman configuration to avoid permission issues +# Create system-wide Podman configuration sudo mkdir -p /etc/containers sudo tee /etc/containers/registries.conf > /dev/null << 'EOF' unqualified-search-registries = ["docker.io"] - -[[registry]] -prefix = "localhost" -location = "localhost" -insecure = true EOF # Set proper permissions for system-wide Podman config +sudo chown CI_SERVICE_USER:CI_SERVICE_USER /etc/containers/registries.conf sudo chmod 644 /etc/containers/registries.conf +# Enable lingering for CI_SERVICE_USER to allow systemd services +sudo loginctl enable-linger CI_SERVICE_USER + +# Create Podman rootless directories outside home +sudo mkdir -p /var/tmp/podman-$(id -u CI_SERVICE_USER)/{root,run,tmp,xdg-data,xdg-config} +sudo chown -R CI_SERVICE_USER:CI_SERVICE_USER /var/tmp/podman-$(id -u CI_SERVICE_USER) +sudo chmod 755 /var/tmp/podman-$(id -u CI_SERVICE_USER) + +# Create runtime directory for user +sudo mkdir -p /run/user/$(id -u CI_SERVICE_USER)/podman-run +sudo chown CI_SERVICE_USER:CI_SERVICE_USER /run/user/$(id -u CI_SERVICE_USER)/podman-run +sudo chmod 755 /run/user/$(id -u CI_SERVICE_USER)/podman-run + +# Initialize Podman with rootless configuration (no home directory access) +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman system migrate" +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman info" + # Install systemd service and configuration files from repository sudo cp /opt/APP_NAME/registry/docker-registry.service /etc/systemd/system/docker-registry.service sudo cp /opt/APP_NAME/registry/containers-policy.json /etc/containers/policy.json +# Set proper permissions for policy file +sudo chown CI_SERVICE_USER:CI_SERVICE_USER /etc/containers/policy.json +sudo chmod 644 /etc/containers/policy.json + # Replace APP_NAME placeholder with actual application name sudo sed -i "s/APP_NAME/YOUR_ACTUAL_APP_NAME/g" /etc/systemd/system/docker-registry.service # Replace CI_SERVICE_USER placeholder with actual CI service user name sudo sed -i "s/CI_SERVICE_USER/YOUR_ACTUAL_CI_SERVICE_USER/g" /etc/systemd/system/docker-registry.service +# Note: The service is configured to use rootless Podman with all state outside home directory +# - PODMAN_ROOT=/var/tmp/podman-%u/root +# - PODMAN_RUNROOT=/run/user/%u/podman-run +# - PODMAN_TMPDIR=/var/tmp/podman-%u/tmp +# - XDG_DATA_HOME=/var/tmp/podman-%u/xdg-data +# - XDG_CONFIG_HOME=/var/tmp/podman-%u/xdg-config + +# Configure firewall for Docker Registry v2 ports +sudo ufw allow 443/tcp # Docker Registry via nginx (public read access) +sudo ufw allow 4443/tcp # Docker Registry via nginx (authenticated operations) + # Enable and start Docker Registry v2 service sudo systemctl daemon-reload sudo systemctl enable docker-registry.service @@ -960,25 +971,28 @@ sudo systemctl status docker-registry.service # Check service logs for any issues sudo journalctl -u docker-registry.service -f --no-pager -n 50 + +# Verify Podman is using non-home paths +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman info --format '{{.Store.GraphRoot}} {{.Store.RunRoot}}'" ``` -#### 5.6 Verify Docker Registry v2 Service +#### 4.6 Verify Docker Registry v2 Service ```bash # Check that the service is running properly sudo systemctl status docker-registry.service # Check that pods are running -sudo su - CI_SERVICE_USER -c "podman pod ps" +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman pod ps" # Check nginx logs -sudo su - CI_SERVICE_USER -c "podman logs registry-pod-nginx" +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman logs registry-pod-nginx" # Check Registry logs -sudo su - CI_SERVICE_USER -c "podman logs registry-pod-registry" +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman logs registry-pod-registry" ``` -#### 5.7 Test Registry Setup +#### 4.7 Test Registry Setup **For Option A (Self-signed certificates):** @@ -990,27 +1004,27 @@ sudo su - CI_SERVICE_USER cd /opt/APP_NAME # Test authenticated push using the project's registry configuration (port 4443) -echo "your-secure-registry-password" | podman login YOUR_ACTUAL_IP_ADDRESS:4443 -u registry-user --password-stdin +echo "your-secure-registry-password" | env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman login YOUR_ACTUAL_IP_ADDRESS:4443 -u registry-user --password-stdin # Create and push test image to authenticated endpoint echo "FROM alpine:latest" > /tmp/test.Dockerfile -podman build -f /tmp/test.Dockerfile -t YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest /tmp -podman push YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman build -f /tmp/test.Dockerfile -t YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest /tmp +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman push YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest # Test unauthenticated pull from standard HTTPS endpoint (port 443) -podman logout YOUR_ACTUAL_IP_ADDRESS:4443 -podman pull YOUR_ACTUAL_IP_ADDRESS/APP_NAME/test:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman logout YOUR_ACTUAL_IP_ADDRESS:4443 +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman pull YOUR_ACTUAL_IP_ADDRESS/APP_NAME/test:latest # Test that unauthorized push to authenticated endpoint is blocked echo "FROM alpine:latest" > /tmp/unauthorized.Dockerfile -podman build -f /tmp/unauthorized.Dockerfile -t YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest /tmp -podman push YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman build -f /tmp/unauthorized.Dockerfile -t YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest /tmp +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman push YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest # Expected: This should fail with authentication error # Clean up -podman rmi YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest 2>/dev/null || true -podman rmi YOUR_ACTUAL_IP_ADDRESS/APP_NAME/test:latest 2>/dev/null || true -podman rmi YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest 2>/dev/null || true +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman rmi YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/test:latest 2>/dev/null || true +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman rmi YOUR_ACTUAL_IP_ADDRESS/APP_NAME/test:latest 2>/dev/null || true +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman rmi YOUR_ACTUAL_IP_ADDRESS:4443/APP_NAME/unauthorized:latest 2>/dev/null || true exit ``` @@ -1024,26 +1038,26 @@ sudo su - CI_SERVICE_USER cd /opt/APP_NAME # Test Podman login and push (now using Let's Encrypt certificate with domain) -echo "your-secure-registry-password" | podman login YOUR_DOMAIN_NAME -u registry-user --password-stdin +echo "your-secure-registry-password" | env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman login YOUR_DOMAIN_NAME -u registry-user --password-stdin # Create and push test image echo "FROM alpine:latest" > /tmp/test.Dockerfile -podman build -f /tmp/test.Dockerfile -t YOUR_DOMAIN_NAME/APP_NAME/test:latest /tmp -podman push YOUR_DOMAIN_NAME/APP_NAME/test:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman build -f /tmp/test.Dockerfile -t YOUR_DOMAIN_NAME/APP_NAME/test:latest /tmp +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman push YOUR_DOMAIN_NAME/APP_NAME/test:latest # Test public pull (no authentication) -docker logout YOUR_DOMAIN_NAME -docker pull YOUR_DOMAIN_NAME/APP_NAME/test:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman logout YOUR_DOMAIN_NAME +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman pull YOUR_DOMAIN_NAME/APP_NAME/test:latest # Test that unauthorized push is blocked echo "FROM alpine:latest" > /tmp/unauthorized.Dockerfile -docker build -f /tmp/unauthorized.Dockerfile -t YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest /tmp -docker push YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman build -f /tmp/unauthorized.Dockerfile -t YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest /tmp +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman push YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest # Expected: This should fail with authentication error # Clean up -docker rmi YOUR_DOMAIN_NAME/APP_NAME/test:latest -docker rmi YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman rmi YOUR_DOMAIN_NAME/APP_NAME/test:latest +env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman rmi YOUR_DOMAIN_NAME/APP_NAME/unauthorized:latest exit ``` @@ -1064,8 +1078,7 @@ exit If you get a TLS error like `remote error: tls: internal error` when using self-signed certificates, verify the certificate installation and Docker configuration: ```bash -# Verify the certificate was installed correctly for Podman -ls -la /etc/containers/certs.d/YOUR_ACTUAL_IP_ADDRESS/ +# Verify the certificate was installed correctly in system trust store ls -la /usr/local/share/ca-certificates/registry-ca.crt # Verify certificate chain is valid @@ -1075,18 +1088,16 @@ openssl verify -CAfile /etc/registry/certs/ca/ca.crt /etc/registry/certs/registr openssl s_client -connect YOUR_ACTUAL_IP_ADDRESS:4443 -servername YOUR_ACTUAL_IP_ADDRESS < /dev/null # Verify nginx is using the correct certificates -docker exec nginx ls -la /etc/registry/certs/ +sudo su - CI_SERVICE_USER -c "env PODMAN_ROOT=/var/tmp/podman-\$(id -u)/root PODMAN_RUNROOT=/run/user/\$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-\$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-\$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-\$(id -u)/xdg-config podman exec registry-pod-nginx ls -la /etc/registry/certs/" -# If issues persist, restart Docker daemon to reload certificates -sudo systemctl restart docker +# If issues persist, restart the Docker Registry v2 service to reload certificates +sudo systemctl restart docker-registry.service # Wait for Docker Registry to restart, then test again sleep 10 -cd /opt/APP_NAME/registry -podman play kube registry-pod.yaml # Test Podman login to authenticated endpoint -echo "your-secure-registry-password" | podman login YOUR_ACTUAL_IP_ADDRESS:4443 -u registry-user --password-stdin +echo "your-secure-registry-password" | env PODMAN_ROOT=/var/tmp/podman-$(id -u)/root PODMAN_RUNROOT=/run/user/$(id -u)/podman-run PODMAN_TMPDIR=/var/tmp/podman-$(id -u)/tmp XDG_DATA_HOME=/var/tmp/podman-$(id -u)/xdg-data XDG_CONFIG_HOME=/var/tmp/podman-$(id -u)/xdg-config podman login YOUR_ACTUAL_IP_ADDRESS:4443 -u registry-user --password-stdin ``` **Certificate Structure Summary:** @@ -1100,11 +1111,11 @@ The project uses a two-port configuration: - **CA Certificates**: `/etc/registry/certs/ca/` (mode 644) - **Certificate Requests**: `/etc/registry/certs/requests/` (mode 644) - **Server Certificates**: `/etc/registry/certs/` (mode 644) -- **Podman Certificates**: `/etc/containers/certs.d/YOUR_ACTUAL_IP_ADDRESS/` +- **System Trust Store**: `/usr/local/share/ca-certificates/registry-ca.crt` -### Step 6: Install Forgejo Actions Runner +### Step 5: Install Forgejo Actions Runner -#### 6.1 Download Runner +#### 5.1 Download Runner **Important**: Run this step as the **CI_DEPLOY_USER** (not root or CI_SERVICE_USER). The CI_DEPLOY_USER handles deployment tasks including downloading and installing the Forgejo runner. @@ -1141,7 +1152,7 @@ sudo mv forgejo-runner-${VERSION#v}-linux-amd64 /usr/bin/forgejo-runner **Production Recommendation**: Use version pinning in production environments to ensure consistency and avoid unexpected breaking changes. -#### 6.2 Register Runner +#### 5.2 Register Runner **Important**: The runner must be registered with your Forgejo instance before it can start. This creates the required `.runner` configuration file. @@ -1299,7 +1310,7 @@ sudo systemctl enable forgejo-runner.service - Enables the service to start automatically on boot - Sets up proper restart behavior for reliability -#### 6.3 Start Service +#### 5.3 Start Service ```bash # Start the Forgejo runner service @@ -1311,7 +1322,7 @@ sudo systemctl status forgejo-runner.service **Expected Output**: The service should show "active (running)" status. -#### 6.4 Test Runner Configuration +#### 5.4 Test Runner Configuration ```bash # Check if the runner is running @@ -1335,11 +1346,11 @@ sudo journalctl -u forgejo-runner.service -f --no-pager - Check network: Ensure the runner can reach your Forgejo instance - Restart service: `sudo systemctl restart forgejo-runner.service` -### Step 7: Set Up Podman-in-Podman (PiP) for CI Operations +### Step 6: Set Up Podman-in-Podman (PiP) for CI Operations **Important**: This step sets up a Podman-in-Podman container that provides an isolated environment for CI/CD operations, eliminating resource contention with Podman Registry and simplifying cleanup. -#### 7.1 Create Containerized CI/CD Environment +#### 6.1 Create Containerized CI/CD Environment ```bash # Switch to CI_SERVICE_USER (who has Podman access) @@ -1369,7 +1380,7 @@ podman exec ci-pip podman version **Why CI_SERVICE_USER**: The CI_SERVICE_USER has Podman access and runs the CI pipeline, so it needs direct access to the PiP container for seamless CI/CD operations. -#### 7.2 Configure PiP for Docker Registry v2 +#### 6.2 Configure PiP for Docker Registry v2 ```bash # Navigate to the application directory @@ -1390,7 +1401,7 @@ podman exec ci-pip podman pull YOUR_CI_CD_IP/APP_NAME/test:latest podman exec ci-pip podman rmi YOUR_CI_CD_IP:4443/APP_NAME/test:latest podman exec ci-pip podman rmi YOUR_CI_CD_IP/APP_NAME/test:latest -#### 7.3 Set Up Workspace Directory +#### 6.3 Set Up Workspace Directory **Important**: The CI workflow needs a workspace directory for code checkout. This directory will be used by the Forgejo Actions runner. @@ -1459,7 +1470,7 @@ The Docker Registry setup now follows the Filesystem Hierarchy Standard (FHS) fo - **Tests connectivity**: Verifies DinD can pull, tag, and push images to Docker Registry - **Validates setup**: Ensures the complete CI/CD pipeline will work -#### 7.3 CI/CD Workflow Architecture +#### 6.4 CI/CD Workflow Architecture The CI/CD pipeline uses a three-stage approach with dedicated environments for each stage: @@ -1526,7 +1537,7 @@ docker exec ci-dind docker rmi YOUR_CI_CD_IP/APP_NAME/dind-test:latest - Docker commands should work inside DinD - Docker Registry push/pull should work from DinD -#### 7.4 Production Deployment Architecture +#### 6.5 Production Deployment Architecture The production deployment uses a separate Docker Compose file (`docker-compose.prod.yml`) that pulls built images from the Docker Registry and deploys the complete application stack. @@ -1549,7 +1560,7 @@ The production deployment uses a separate Docker Compose file (`docker-compose.p - **🛡️ Rollback Capability**: Can easily rollback to previous image versions - **📊 Health Monitoring**: Built-in health checks for all services -#### 7.5 Monitoring Script +#### 6.6 Monitoring Script **Important**: The repository includes a pre-configured monitoring script in the `scripts/` directory that can be used for both CI/CD and production monitoring. @@ -1573,9 +1584,9 @@ chmod +x scripts/monitor.sh **Note**: The repository script is more comprehensive and includes proper error handling, colored output, and support for both CI/CD and production environments. It automatically detects the environment and provides appropriate monitoring information. -### Step 8: Configure Firewall +### Step 7: Configure Firewall -#### 8.1 Configure UFW Firewall +#### 7.1 Configure UFW Firewall ```bash sudo ufw --force enable @@ -1590,23 +1601,23 @@ sudo ufw allow 443/tcp # Docker Registry via nginx (public read access) - **SSH**: Restricted to your IP addresses - **All other ports**: Blocked -### Step 9: Test CI/CD Setup +### Step 8: Test CI/CD Setup -#### 9.1 Test Podman Installation +#### 8.1 Test Podman Installation ```bash podman --version podman-compose --version ``` -#### 9.2 Check Docker Registry v2 Status +#### 8.2 Check Docker Registry v2 Status ```bash cd /opt/APP_NAME/registry podman pod ps ``` -#### 9.3 Test Docker Registry v2 Access +#### 8.3 Test Docker Registry v2 Access ```bash # Test Docker Registry v2 API @@ -1871,9 +1882,8 @@ podman-compose --version #### 12.2 Configure Podman for Production Service Account ```bash -# Podman can run rootless, so no group membership needed -# But we'll configure it for the production service account -sudo usermod -aG podman PROD_SERVICE_USER +# Podman runs rootless by default, no group membership needed +# The subuid/subgid ranges configured in Step 1.5 enable rootless operation ``` #### 12.4 Create Application Directory @@ -2458,7 +2468,7 @@ sudo rm -rf /opt/APP_NAME/registry/certs/requests/openssl.conf - **Protected with proper permissions** (600 - owner read/write only) - **Rotated regularly** by updating the password and regenerating the htpasswd file -### Step 8.6 CI/CD Workflow Summary Table +### Step 7.4 CI/CD Workflow Summary Table | Stage | What Runs | How/Where | |---------|--------------------------|--------------------------| diff --git a/registry/README.md b/registry/README.md index 364db70..b1c8f63 100644 --- a/registry/README.md +++ b/registry/README.md @@ -34,16 +34,27 @@ The registry setup uses: - **Method-based restrictions**: Push operations require authentication - **Path validation**: Prevents method spoofing by validating both URL patterns and HTTP methods - **Security headers**: X-Content-Type-Options, X-Frame-Options for additional protection +- **Rootless Podman**: All state stored outside home directory for complete isolation ## Configuration The setup is configured through: -1. **Environment Variables**: Stored in `.env` file (created during setup) for authentication -2. **Docker Compose Environment**: Registry configuration via environment variables -3. **nginx.conf**: Handles HTTPS and authentication -4. **Docker Compose**: Orchestrates the registry and nginx services -5. **Systemd Service**: Manages the Docker Registry service lifecycle -6. **User/Permissions**: All files and services are owned and run by `CI_SERVICE_USER` for consistency and security +1. **nginx.conf**: Handles HTTPS and authentication +2. **registry-pod.yaml**: Kubernetes-style pod definition for Podman +3. **docker-registry.service**: Systemd service with rootless Podman configuration +4. **User/Permissions**: All files and services are owned and run by `CI_SERVICE_USER` for consistency and security + +## Podman Configuration + +The registry uses rootless Podman with all state stored outside the user's home directory: + +- **PODMAN_ROOT**: `/var/tmp/podman-%u/root` - Container storage +- **PODMAN_RUNROOT**: `/run/user/%u/podman-run` - Runtime state +- **PODMAN_TMPDIR**: `/var/tmp/podman-%u/tmp` - Temporary files +- **XDG_DATA_HOME**: `/var/tmp/podman-%u/xdg-data` - Data directory +- **XDG_CONFIG_HOME**: `/var/tmp/podman-%u/xdg-config` - Configuration + +This ensures complete isolation from the user's home directory while maintaining rootless security. ## Usage diff --git a/registry/docker-registry.service b/registry/docker-registry.service index 1edebd6..807e565 100644 --- a/registry/docker-registry.service +++ b/registry/docker-registry.service @@ -8,9 +8,17 @@ RemainAfterExit=yes User=CI_SERVICE_USER Group=CI_SERVICE_USER WorkingDirectory=/opt/APP_NAME/registry -ExecStart=/usr/bin/podman play kube registry-pod.yaml -ExecStop=/usr/bin/podman pod stop registry-pod -ExecReload=/usr/bin/podman pod restart registry-pod + +# Podman rootless configuration - all state outside home +Environment=PODMAN_ROOT=/var/tmp/podman-%u/root +Environment=PODMAN_RUNROOT=/run/user/%u/podman-run +Environment=PODMAN_TMPDIR=/var/tmp/podman-%u/tmp +Environment=XDG_DATA_HOME=/var/tmp/podman-%u/xdg-data +Environment=XDG_CONFIG_HOME=/var/tmp/podman-%u/xdg-config + +ExecStart=/usr/bin/podman --root=${PODMAN_ROOT} --runroot=${PODMAN_RUNROOT} --tmpdir=${PODMAN_TMPDIR} --events-backend=file play kube registry-pod.yaml +ExecStop=/usr/bin/podman --root=${PODMAN_ROOT} --runroot=${PODMAN_RUNROOT} --tmpdir=${PODMAN_TMPDIR} --events-backend=file pod stop registry-pod +ExecReload=/usr/bin/podman --root=${PODMAN_ROOT} --runroot=${PODMAN_RUNROOT} --tmpdir=${PODMAN_TMPDIR} --events-backend=file pod restart registry-pod TimeoutStartSec=0 # Security settings @@ -18,7 +26,7 @@ NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict ProtectHome=true -ReadWritePaths=/opt/APP_NAME/registry /etc/registry /var/lib/registry /var/log/registry +ReadWritePaths=/opt/APP_NAME/registry /etc/registry /var/lib/registry /var/log/registry /var/tmp/podman-%u [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/registry/nginx.conf b/registry/nginx.conf index 934806e..e53c63c 100644 --- a/registry/nginx.conf +++ b/registry/nginx.conf @@ -2,23 +2,30 @@ # Port 443: Unauthenticated pulls (GET requests only) # Port 4443: Authenticated operations (login, logout, push, delete, etc.) -# Rate limiting with different zones for different operations -limit_req_zone $binary_remote_addr zone=registry:10m rate=10r/s; -limit_req_zone $binary_remote_addr zone=push:10m rate=5r/s; -limit_req_zone $binary_remote_addr zone=login:10m rate=2r/s; - -# Connection limiting -limit_conn_zone $binary_remote_addr zone=conn_limit_per_ip:10m; - -# Upstream Docker Registry -upstream registry { - server registry:5000; - keepalive 32; - # Health check - keepalive_requests 100; - keepalive_timeout 60s; +events { + worker_connections 1024; + use epoll; + multi_accept on; } +http { + # Rate limiting with different zones for different operations + limit_req_zone $binary_remote_addr zone=registry:10m rate=10r/s; + limit_req_zone $binary_remote_addr zone=push:10m rate=5r/s; + limit_req_zone $binary_remote_addr zone=login:10m rate=2r/s; + + # Connection limiting + limit_conn_zone $binary_remote_addr zone=conn_limit_per_ip:10m; + + # Upstream Docker Registry + upstream registry { + server localhost:5000; + keepalive 32; + # Health check + keepalive_requests 100; + keepalive_timeout 60s; + } + # HTTP server for unauthenticated pulls on port 443 server { listen 443 ssl http2; @@ -210,10 +217,4 @@ server { access_log /var/log/nginx/registry_auth_access.log; error_log /var/log/nginx/registry_auth_error.log; } - -# Redirect HTTP to HTTPS (optional - for port 80 if needed) -server { - listen 80; - server_name _; - return 301 https://$server_name$request_uri; } diff --git a/registry/registry-pod.yaml b/registry/registry-pod.yaml index bc31841..c50a419 100644 --- a/registry/registry-pod.yaml +++ b/registry/registry-pod.yaml @@ -28,7 +28,7 @@ spec: value: "0" containers: - name: registry - image: registry@sha256:8be26f81ffea54106bae012c6f349df70f4d5e7e2ec01b143c46e2c03b9e551d + image: registry:2 securityContext: runAsNonRoot: true runAsUser: 1000 @@ -83,7 +83,7 @@ spec: periodSeconds: 5 - name: nginx - image: nginx@sha256:6650513efd1d27c1f8a5351cbd33edf85cc7e3b73dc4d4d4e8f8c0b3d0b3d0b3d + image: nginx:alpine securityContext: runAsNonRoot: true runAsUser: 1000