Sanitized mirror from private repository - 2026-03-18 10:31:50 UTC
This commit is contained in:
Gitea Mirror Bot
501
ansible/automation/playbooks/container_update_orchestrator.yml
Normal file
501
ansible/automation/playbooks/container_update_orchestrator.yml
Normal file
@@ -0,0 +1,501 @@
|
||||
---
|
||||
- name: Container Update Orchestrator
|
||||
hosts: all
|
||||
gather_facts: yes
|
||||
vars:
|
||||
update_timestamp: "{{ ansible_date_time.iso8601 }}"
|
||||
update_report_dir: "/tmp/update_reports"
|
||||
rollback_enabled: true
|
||||
update_timeout: 600
|
||||
health_check_retries: 5
|
||||
health_check_delay: 10
|
||||
|
||||
tasks:
|
||||
- name: Create update reports directory
|
||||
file:
|
||||
path: "{{ update_report_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
|
||||
- name: Check if Docker is available
|
||||
shell: command -v docker >/dev/null 2>&1
|
||||
register: docker_available
|
||||
changed_when: false
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Skip Docker tasks if not available
|
||||
set_fact:
|
||||
skip_docker: "{{ docker_available.rc != 0 }}"
|
||||
|
||||
- name: Pre-update system check
|
||||
shell: |
|
||||
echo "=== PRE-UPDATE SYSTEM CHECK ==="
|
||||
|
||||
# System resources
|
||||
echo "System Resources:"
|
||||
echo "Memory: $(free -h | awk 'NR==2{print $3"/"$2" ("$3*100/$2"%)"}')"
|
||||
echo "Disk: $(df -h / | awk 'NR==2{print $3"/"$2" ("$5")"}')"
|
||||
echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
|
||||
echo ""
|
||||
|
||||
# Docker status
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker Status:"
|
||||
echo "Running containers: $(docker ps -q 2>/dev/null | wc -l)"
|
||||
echo "Total containers: $(docker ps -aq 2>/dev/null | wc -l)"
|
||||
echo "Images: $(docker images -q 2>/dev/null | wc -l)"
|
||||
echo "Docker daemon: $(docker info >/dev/null 2>&1 && echo 'OK' || echo 'ERROR')"
|
||||
else
|
||||
echo "Docker not available"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Network connectivity
|
||||
echo "Network Connectivity:"
|
||||
ping -c 1 8.8.8.8 >/dev/null 2>&1 && echo "Internet: OK" || echo "Internet: FAILED"
|
||||
|
||||
# Tailscale connectivity
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
tailscale status >/dev/null 2>&1 && echo "Tailscale: OK" || echo "Tailscale: FAILED"
|
||||
fi
|
||||
register: pre_update_check
|
||||
changed_when: false
|
||||
|
||||
- name: Discover updatable containers
|
||||
shell: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== CONTAINER UPDATE DISCOVERY ==="
|
||||
|
||||
# Get current container information
|
||||
echo "Current Container Status:"
|
||||
docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.RunningFor}}" 2>/dev/null
|
||||
echo ""
|
||||
|
||||
# Check for available image updates
|
||||
echo "Checking for image updates:"
|
||||
docker images --format "{{.Repository}}:{{.Tag}}" 2>/dev/null | grep -v "<none>" | while read image; do
|
||||
if [ -n "$image" ]; then
|
||||
echo "Checking: $image"
|
||||
|
||||
# Pull latest image to compare
|
||||
if docker pull "$image" >/dev/null 2>&1; then
|
||||
# Compare image IDs
|
||||
current_id=$(docker images "$image" --format "{{.ID}}" | head -1)
|
||||
echo " Current ID: $current_id"
|
||||
|
||||
# Check if any containers are using this image
|
||||
containers_using=$(docker ps --filter "ancestor=$image" --format "{{.Names}}" 2>/dev/null | tr '\n' ' ')
|
||||
if [ -n "$containers_using" ]; then
|
||||
echo " Used by containers: $containers_using"
|
||||
else
|
||||
echo " No running containers using this image"
|
||||
fi
|
||||
else
|
||||
echo " ❌ Failed to pull latest image"
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
register: container_discovery
|
||||
changed_when: false
|
||||
when: not skip_docker
|
||||
|
||||
- name: Create container backup snapshots
|
||||
shell: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== CREATING CONTAINER SNAPSHOTS ==="
|
||||
|
||||
# Create snapshots of running containers
|
||||
docker ps --format "{{.Names}}" 2>/dev/null | while read container; do
|
||||
if [ -n "$container" ]; then
|
||||
echo "Creating snapshot for: $container"
|
||||
|
||||
# Commit container to backup image
|
||||
backup_image="${container}_backup_$(date +%Y%m%d_%H%M%S)"
|
||||
if docker commit "$container" "$backup_image" >/dev/null 2>&1; then
|
||||
echo " ✅ Snapshot created: $backup_image"
|
||||
else
|
||||
echo " ❌ Failed to create snapshot"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# Export Docker Compose configurations
|
||||
echo "Backing up Docker Compose files:"
|
||||
find /opt /home -name "docker-compose*.yml" -o -name "compose*.yml" 2>/dev/null | while read compose_file; do
|
||||
if [ -f "$compose_file" ]; then
|
||||
backup_file="/tmp/$(basename "$compose_file").backup.$(date +%Y%m%d_%H%M%S)"
|
||||
cp "$compose_file" "$backup_file" 2>/dev/null && echo " ✅ Backed up: $compose_file -> $backup_file"
|
||||
fi
|
||||
done
|
||||
register: backup_snapshots
|
||||
changed_when: false
|
||||
when: not skip_docker and rollback_enabled
|
||||
|
||||
- name: Orchestrated container updates
|
||||
block:
|
||||
- name: Update containers by priority groups
|
||||
shell: |
|
||||
echo "=== ORCHESTRATED CONTAINER UPDATES ==="
|
||||
|
||||
# Define update priority groups
|
||||
# Priority 1: Infrastructure services (databases, caches)
|
||||
# Priority 2: Application services
|
||||
# Priority 3: Monitoring and auxiliary services
|
||||
|
||||
priority_1="postgres mysql mariadb redis mongo elasticsearch rabbitmq"
|
||||
priority_2="nginx apache traefik caddy"
|
||||
priority_3="grafana prometheus node-exporter"
|
||||
|
||||
update_group() {
|
||||
local group_name="$1"
|
||||
local containers="$2"
|
||||
|
||||
echo "Updating $group_name containers..."
|
||||
|
||||
for pattern in $containers; do
|
||||
matching_containers=$(docker ps --format "{{.Names}}" 2>/dev/null | grep -i "$pattern" || true)
|
||||
|
||||
for container in $matching_containers; do
|
||||
if [ -n "$container" ]; then
|
||||
echo " Updating: $container"
|
||||
|
||||
# Get current image
|
||||
current_image=$(docker inspect "$container" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
|
||||
# Pull latest image
|
||||
if docker pull "$current_image" >/dev/null 2>&1; then
|
||||
echo " ✅ Image updated: $current_image"
|
||||
|
||||
# Recreate container with new image
|
||||
if docker-compose -f "$(find /opt /home -name "*compose*.yml" -exec grep -l "$container" {} \; | head -1)" up -d "$container" >/dev/null 2>&1; then
|
||||
echo " ✅ Container recreated successfully"
|
||||
|
||||
# Wait for container to be healthy
|
||||
sleep {{ health_check_delay }}
|
||||
|
||||
# Check container health
|
||||
if [ "$(docker inspect "$container" --format '{{.State.Status}}' 2>/dev/null)" = "running" ]; then
|
||||
echo " ✅ Container is running"
|
||||
else
|
||||
echo " ❌ Container failed to start"
|
||||
fi
|
||||
else
|
||||
echo " ❌ Failed to recreate container"
|
||||
fi
|
||||
else
|
||||
echo " ⚠️ No image update available"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# Execute updates by priority
|
||||
update_group "Priority 1 (Infrastructure)" "$priority_1"
|
||||
sleep 30 # Wait between priority groups
|
||||
|
||||
update_group "Priority 2 (Applications)" "$priority_2"
|
||||
sleep 30
|
||||
|
||||
update_group "Priority 3 (Monitoring)" "$priority_3"
|
||||
|
||||
echo "Orchestrated updates completed"
|
||||
register: orchestrated_updates
|
||||
when: update_mode is defined and update_mode == "orchestrated"
|
||||
|
||||
- name: Update specific container
|
||||
shell: |
|
||||
echo "=== UPDATING SPECIFIC CONTAINER ==="
|
||||
|
||||
container="{{ target_container }}"
|
||||
|
||||
if ! docker ps --format "{{.Names}}" | grep -q "^${container}$"; then
|
||||
echo "❌ Container '$container' not found or not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Updating container: $container"
|
||||
|
||||
# Get current image
|
||||
current_image=$(docker inspect "$container" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
echo "Current image: $current_image"
|
||||
|
||||
# Pull latest image
|
||||
echo "Pulling latest image..."
|
||||
if docker pull "$current_image"; then
|
||||
echo "✅ Image pulled successfully"
|
||||
|
||||
# Find compose file
|
||||
compose_file=$(find /opt /home -name "*compose*.yml" -exec grep -l "$container" {} \; | head -1)
|
||||
|
||||
if [ -n "$compose_file" ]; then
|
||||
echo "Using compose file: $compose_file"
|
||||
|
||||
# Update container using compose
|
||||
if docker-compose -f "$compose_file" up -d "$container"; then
|
||||
echo "✅ Container updated successfully"
|
||||
|
||||
# Health check
|
||||
echo "Performing health check..."
|
||||
sleep {{ health_check_delay }}
|
||||
|
||||
retries={{ health_check_retries }}
|
||||
while [ $retries -gt 0 ]; do
|
||||
if [ "$(docker inspect "$container" --format '{{.State.Status}}' 2>/dev/null)" = "running" ]; then
|
||||
echo "✅ Container is healthy"
|
||||
break
|
||||
else
|
||||
echo "⏳ Waiting for container to be ready... ($retries retries left)"
|
||||
sleep {{ health_check_delay }}
|
||||
retries=$((retries - 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $retries -eq 0 ]; then
|
||||
echo "❌ Container failed health check"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "❌ Failed to update container"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "⚠️ No compose file found, using direct Docker commands"
|
||||
docker restart "$container"
|
||||
fi
|
||||
else
|
||||
echo "❌ Failed to pull image"
|
||||
exit 1
|
||||
fi
|
||||
register: specific_update
|
||||
when: target_container is defined
|
||||
|
||||
when: not skip_docker
|
||||
|
||||
- name: Post-update verification
|
||||
shell: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== POST-UPDATE VERIFICATION ==="
|
||||
|
||||
# Check all containers are running
|
||||
echo "Container Status Check:"
|
||||
failed_containers=""
|
||||
docker ps -a --format "{{.Names}}\t{{.Status}}" 2>/dev/null | while IFS=$'\t' read name status; do
|
||||
if [ -n "$name" ]; then
|
||||
if echo "$status" | grep -q "Up"; then
|
||||
echo "✅ $name: $status"
|
||||
else
|
||||
echo "❌ $name: $status"
|
||||
failed_containers="$failed_containers $name"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check system resources after update
|
||||
echo ""
|
||||
echo "System Resources After Update:"
|
||||
echo "Memory: $(free -h | awk 'NR==2{print $3"/"$2" ("$3*100/$2"%)"}')"
|
||||
echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
|
||||
|
||||
# Check for any error logs
|
||||
echo ""
|
||||
echo "Recent Error Logs:"
|
||||
docker ps --format "{{.Names}}" 2>/dev/null | head -5 | while read container; do
|
||||
if [ -n "$container" ]; then
|
||||
errors=$(docker logs "$container" --since="5m" 2>&1 | grep -i error | wc -l)
|
||||
if [ "$errors" -gt "0" ]; then
|
||||
echo "⚠️ $container: $errors error(s) in last 5 minutes"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
register: post_update_verification
|
||||
changed_when: false
|
||||
when: not skip_docker
|
||||
|
||||
- name: Rollback on failure
|
||||
shell: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== ROLLBACK PROCEDURE ==="
|
||||
|
||||
# Check if rollback is needed
|
||||
failed_containers=$(docker ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null | head -5)
|
||||
|
||||
if [ -n "$failed_containers" ]; then
|
||||
echo "Failed containers detected: $failed_containers"
|
||||
echo "Initiating rollback..."
|
||||
|
||||
for container in $failed_containers; do
|
||||
echo "Rolling back: $container"
|
||||
|
||||
# Find backup image
|
||||
backup_image=$(docker images --format "{{.Repository}}" | grep "${container}_backup_" | head -1)
|
||||
|
||||
if [ -n "$backup_image" ]; then
|
||||
echo " Found backup image: $backup_image"
|
||||
|
||||
# Stop current container
|
||||
docker stop "$container" 2>/dev/null || true
|
||||
docker rm "$container" 2>/dev/null || true
|
||||
|
||||
# Start container from backup image
|
||||
if docker run -d --name "$container" "$backup_image"; then
|
||||
echo " ✅ Rollback successful"
|
||||
else
|
||||
echo " ❌ Rollback failed"
|
||||
fi
|
||||
else
|
||||
echo " ⚠️ No backup image found"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "No rollback needed - all containers are healthy"
|
||||
fi
|
||||
register: rollback_result
|
||||
when: not skip_docker and rollback_enabled and (orchestrated_updates.rc is defined and orchestrated_updates.rc != 0) or (specific_update.rc is defined and specific_update.rc != 0)
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Cleanup old backup images
|
||||
shell: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker not available"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== CLEANUP OLD BACKUPS ==="
|
||||
|
||||
# Remove backup images older than 7 days
|
||||
old_backups=$(docker images --format "{{.Repository}}\t{{.CreatedAt}}" | grep "_backup_" | awk '$2 < "'$(date -d '7 days ago' '+%Y-%m-%d')'"' | cut -f1)
|
||||
|
||||
if [ -n "$old_backups" ]; then
|
||||
echo "Removing old backup images:"
|
||||
for backup in $old_backups; do
|
||||
echo " Removing: $backup"
|
||||
docker rmi "$backup" 2>/dev/null || echo " Failed to remove $backup"
|
||||
done
|
||||
else
|
||||
echo "No old backup images to clean up"
|
||||
fi
|
||||
|
||||
# Clean up temporary backup files
|
||||
find /tmp -name "*.backup.*" -mtime +7 -delete 2>/dev/null || true
|
||||
register: cleanup_result
|
||||
when: not skip_docker
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Create update report
|
||||
set_fact:
|
||||
update_report:
|
||||
timestamp: "{{ update_timestamp }}"
|
||||
hostname: "{{ inventory_hostname }}"
|
||||
docker_available: "{{ not skip_docker }}"
|
||||
pre_update_check: "{{ pre_update_check.stdout }}"
|
||||
container_discovery: "{{ container_discovery.stdout if not skip_docker else 'Docker not available' }}"
|
||||
backup_snapshots: "{{ backup_snapshots.stdout if not skip_docker and rollback_enabled else 'Snapshots disabled' }}"
|
||||
orchestrated_updates: "{{ orchestrated_updates.stdout if orchestrated_updates is defined else 'Not performed' }}"
|
||||
specific_update: "{{ specific_update.stdout if specific_update is defined else 'Not performed' }}"
|
||||
post_update_verification: "{{ post_update_verification.stdout if not skip_docker else 'Docker not available' }}"
|
||||
rollback_result: "{{ rollback_result.stdout if rollback_result is defined else 'Not needed' }}"
|
||||
cleanup_result: "{{ cleanup_result.stdout if not skip_docker else 'Docker not available' }}"
|
||||
|
||||
- name: Display update report
|
||||
debug:
|
||||
msg: |
|
||||
|
||||
==========================================
|
||||
🔄 CONTAINER UPDATE REPORT - {{ inventory_hostname }}
|
||||
==========================================
|
||||
|
||||
📊 DOCKER AVAILABLE: {{ 'Yes' if update_report.docker_available else 'No' }}
|
||||
|
||||
🔍 PRE-UPDATE CHECK:
|
||||
{{ update_report.pre_update_check }}
|
||||
|
||||
🔍 CONTAINER DISCOVERY:
|
||||
{{ update_report.container_discovery }}
|
||||
|
||||
💾 BACKUP SNAPSHOTS:
|
||||
{{ update_report.backup_snapshots }}
|
||||
|
||||
🔄 ORCHESTRATED UPDATES:
|
||||
{{ update_report.orchestrated_updates }}
|
||||
|
||||
🎯 SPECIFIC UPDATE:
|
||||
{{ update_report.specific_update }}
|
||||
|
||||
✅ POST-UPDATE VERIFICATION:
|
||||
{{ update_report.post_update_verification }}
|
||||
|
||||
↩️ ROLLBACK RESULT:
|
||||
{{ update_report.rollback_result }}
|
||||
|
||||
🧹 CLEANUP RESULT:
|
||||
{{ update_report.cleanup_result }}
|
||||
|
||||
==========================================
|
||||
|
||||
- name: Generate JSON update report
|
||||
copy:
|
||||
content: |
|
||||
{
|
||||
"timestamp": "{{ update_report.timestamp }}",
|
||||
"hostname": "{{ update_report.hostname }}",
|
||||
"docker_available": {{ update_report.docker_available | lower }},
|
||||
"pre_update_check": {{ update_report.pre_update_check | to_json }},
|
||||
"container_discovery": {{ update_report.container_discovery | to_json }},
|
||||
"backup_snapshots": {{ update_report.backup_snapshots | to_json }},
|
||||
"orchestrated_updates": {{ update_report.orchestrated_updates | to_json }},
|
||||
"specific_update": {{ update_report.specific_update | to_json }},
|
||||
"post_update_verification": {{ update_report.post_update_verification | to_json }},
|
||||
"rollback_result": {{ update_report.rollback_result | to_json }},
|
||||
"cleanup_result": {{ update_report.cleanup_result | to_json }},
|
||||
"recommendations": [
|
||||
"Test updates in staging environment first",
|
||||
"Monitor container health after updates",
|
||||
"Maintain regular backup snapshots",
|
||||
"Keep rollback procedures tested and ready",
|
||||
"Schedule updates during maintenance windows"
|
||||
]
|
||||
}
|
||||
dest: "{{ update_report_dir }}/{{ inventory_hostname }}_container_updates_{{ ansible_date_time.epoch }}.json"
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Summary message
|
||||
debug:
|
||||
msg: |
|
||||
|
||||
🔄 Container update orchestration complete for {{ inventory_hostname }}
|
||||
📄 Report saved to: {{ update_report_dir }}/{{ inventory_hostname }}_container_updates_{{ ansible_date_time.epoch }}.json
|
||||
|
||||
{% if target_container is defined %}
|
||||
🎯 Updated container: {{ target_container }}
|
||||
{% endif %}
|
||||
|
||||
{% if update_mode is defined %}
|
||||
🔄 Update mode: {{ update_mode }}
|
||||
{% endif %}
|
||||
|
||||
💡 Use -e target_container=<name> to update specific containers
|
||||
💡 Use -e update_mode=orchestrated for priority-based updates
|
||||
💡 Use -e rollback_enabled=false to disable automatic rollback
|
||||
Reference in New Issue
Block a user