502 lines
19 KiB
YAML
502 lines
19 KiB
YAML
---
|
|
- name: Container Update Orchestrator
|
|
hosts: all
|
|
gather_facts: yes
|
|
vars:
|
|
update_timestamp: "{{ ansible_date_time.iso8601 }}"
|
|
update_report_dir: "/tmp/update_reports"
|
|
rollback_enabled: true
|
|
update_timeout: 600
|
|
health_check_retries: 5
|
|
health_check_delay: 10
|
|
|
|
tasks:
|
|
- name: Create update reports directory
|
|
file:
|
|
path: "{{ update_report_dir }}"
|
|
state: directory
|
|
mode: '0755'
|
|
delegate_to: localhost
|
|
run_once: true
|
|
|
|
- name: Check if Docker is available
|
|
shell: command -v docker >/dev/null 2>&1
|
|
register: docker_available
|
|
changed_when: false
|
|
ignore_errors: yes
|
|
|
|
- name: Skip Docker tasks if not available
|
|
set_fact:
|
|
skip_docker: "{{ docker_available.rc != 0 }}"
|
|
|
|
- name: Pre-update system check
|
|
shell: |
|
|
echo "=== PRE-UPDATE SYSTEM CHECK ==="
|
|
|
|
# System resources
|
|
echo "System Resources:"
|
|
echo "Memory: $(free -h | awk 'NR==2{print $3"/"$2" ("$3*100/$2"%)"}')"
|
|
echo "Disk: $(df -h / | awk 'NR==2{print $3"/"$2" ("$5")"}')"
|
|
echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
|
|
echo ""
|
|
|
|
# Docker status
|
|
if command -v docker >/dev/null 2>&1; then
|
|
echo "Docker Status:"
|
|
echo "Running containers: $(docker ps -q 2>/dev/null | wc -l)"
|
|
echo "Total containers: $(docker ps -aq 2>/dev/null | wc -l)"
|
|
echo "Images: $(docker images -q 2>/dev/null | wc -l)"
|
|
echo "Docker daemon: $(docker info >/dev/null 2>&1 && echo 'OK' || echo 'ERROR')"
|
|
else
|
|
echo "Docker not available"
|
|
fi
|
|
echo ""
|
|
|
|
# Network connectivity
|
|
echo "Network Connectivity:"
|
|
ping -c 1 8.8.8.8 >/dev/null 2>&1 && echo "Internet: OK" || echo "Internet: FAILED"
|
|
|
|
# Tailscale connectivity
|
|
if command -v tailscale >/dev/null 2>&1; then
|
|
tailscale status >/dev/null 2>&1 && echo "Tailscale: OK" || echo "Tailscale: FAILED"
|
|
fi
|
|
register: pre_update_check
|
|
changed_when: false
|
|
|
|
- name: Discover updatable containers
|
|
shell: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "Docker not available"
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== CONTAINER UPDATE DISCOVERY ==="
|
|
|
|
# Get current container information
|
|
echo "Current Container Status:"
|
|
docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.RunningFor}}" 2>/dev/null
|
|
echo ""
|
|
|
|
# Check for available image updates
|
|
echo "Checking for image updates:"
|
|
docker images --format "{{.Repository}}:{{.Tag}}" 2>/dev/null | grep -v "<none>" | while read image; do
|
|
if [ -n "$image" ]; then
|
|
echo "Checking: $image"
|
|
|
|
# Pull latest image to compare
|
|
if docker pull "$image" >/dev/null 2>&1; then
|
|
# Compare image IDs
|
|
current_id=$(docker images "$image" --format "{{.ID}}" | head -1)
|
|
echo " Current ID: $current_id"
|
|
|
|
# Check if any containers are using this image
|
|
containers_using=$(docker ps --filter "ancestor=$image" --format "{{.Names}}" 2>/dev/null | tr '\n' ' ')
|
|
if [ -n "$containers_using" ]; then
|
|
echo " Used by containers: $containers_using"
|
|
else
|
|
echo " No running containers using this image"
|
|
fi
|
|
else
|
|
echo " ❌ Failed to pull latest image"
|
|
fi
|
|
echo ""
|
|
fi
|
|
done
|
|
register: container_discovery
|
|
changed_when: false
|
|
when: not skip_docker
|
|
|
|
- name: Create container backup snapshots
|
|
shell: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "Docker not available"
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== CREATING CONTAINER SNAPSHOTS ==="
|
|
|
|
# Create snapshots of running containers
|
|
docker ps --format "{{.Names}}" 2>/dev/null | while read container; do
|
|
if [ -n "$container" ]; then
|
|
echo "Creating snapshot for: $container"
|
|
|
|
# Commit container to backup image
|
|
backup_image="${container}_backup_$(date +%Y%m%d_%H%M%S)"
|
|
if docker commit "$container" "$backup_image" >/dev/null 2>&1; then
|
|
echo " ✅ Snapshot created: $backup_image"
|
|
else
|
|
echo " ❌ Failed to create snapshot"
|
|
fi
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
# Export Docker Compose configurations
|
|
echo "Backing up Docker Compose files:"
|
|
find /opt /home -name "docker-compose*.yml" -o -name "compose*.yml" 2>/dev/null | while read compose_file; do
|
|
if [ -f "$compose_file" ]; then
|
|
backup_file="/tmp/$(basename "$compose_file").backup.$(date +%Y%m%d_%H%M%S)"
|
|
cp "$compose_file" "$backup_file" 2>/dev/null && echo " ✅ Backed up: $compose_file -> $backup_file"
|
|
fi
|
|
done
|
|
register: backup_snapshots
|
|
changed_when: false
|
|
when: not skip_docker and rollback_enabled
|
|
|
|
- name: Orchestrated container updates
|
|
block:
|
|
- name: Update containers by priority groups
|
|
shell: |
|
|
echo "=== ORCHESTRATED CONTAINER UPDATES ==="
|
|
|
|
# Define update priority groups
|
|
# Priority 1: Infrastructure services (databases, caches)
|
|
# Priority 2: Application services
|
|
# Priority 3: Monitoring and auxiliary services
|
|
|
|
priority_1="postgres mysql mariadb redis mongo elasticsearch rabbitmq"
|
|
priority_2="nginx apache traefik caddy"
|
|
priority_3="grafana prometheus node-exporter"
|
|
|
|
update_group() {
|
|
local group_name="$1"
|
|
local containers="$2"
|
|
|
|
echo "Updating $group_name containers..."
|
|
|
|
for pattern in $containers; do
|
|
matching_containers=$(docker ps --format "{{.Names}}" 2>/dev/null | grep -i "$pattern" || true)
|
|
|
|
for container in $matching_containers; do
|
|
if [ -n "$container" ]; then
|
|
echo " Updating: $container"
|
|
|
|
# Get current image
|
|
current_image=$(docker inspect "$container" --format '{{.Config.Image}}' 2>/dev/null)
|
|
|
|
# Pull latest image
|
|
if docker pull "$current_image" >/dev/null 2>&1; then
|
|
echo " ✅ Image updated: $current_image"
|
|
|
|
# Recreate container with new image
|
|
if docker-compose -f "$(find /opt /home -name "*compose*.yml" -exec grep -l "$container" {} \; | head -1)" up -d "$container" >/dev/null 2>&1; then
|
|
echo " ✅ Container recreated successfully"
|
|
|
|
# Wait for container to be healthy
|
|
sleep {{ health_check_delay }}
|
|
|
|
# Check container health
|
|
if [ "$(docker inspect "$container" --format '{{.State.Status}}' 2>/dev/null)" = "running" ]; then
|
|
echo " ✅ Container is running"
|
|
else
|
|
echo " ❌ Container failed to start"
|
|
fi
|
|
else
|
|
echo " ❌ Failed to recreate container"
|
|
fi
|
|
else
|
|
echo " ⚠️ No image update available"
|
|
fi
|
|
|
|
echo ""
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
# Execute updates by priority
|
|
update_group "Priority 1 (Infrastructure)" "$priority_1"
|
|
sleep 30 # Wait between priority groups
|
|
|
|
update_group "Priority 2 (Applications)" "$priority_2"
|
|
sleep 30
|
|
|
|
update_group "Priority 3 (Monitoring)" "$priority_3"
|
|
|
|
echo "Orchestrated updates completed"
|
|
register: orchestrated_updates
|
|
when: update_mode is defined and update_mode == "orchestrated"
|
|
|
|
- name: Update specific container
|
|
shell: |
|
|
echo "=== UPDATING SPECIFIC CONTAINER ==="
|
|
|
|
container="{{ target_container }}"
|
|
|
|
if ! docker ps --format "{{.Names}}" | grep -q "^${container}$"; then
|
|
echo "❌ Container '$container' not found or not running"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Updating container: $container"
|
|
|
|
# Get current image
|
|
current_image=$(docker inspect "$container" --format '{{.Config.Image}}' 2>/dev/null)
|
|
echo "Current image: $current_image"
|
|
|
|
# Pull latest image
|
|
echo "Pulling latest image..."
|
|
if docker pull "$current_image"; then
|
|
echo "✅ Image pulled successfully"
|
|
|
|
# Find compose file
|
|
compose_file=$(find /opt /home -name "*compose*.yml" -exec grep -l "$container" {} \; | head -1)
|
|
|
|
if [ -n "$compose_file" ]; then
|
|
echo "Using compose file: $compose_file"
|
|
|
|
# Update container using compose
|
|
if docker-compose -f "$compose_file" up -d "$container"; then
|
|
echo "✅ Container updated successfully"
|
|
|
|
# Health check
|
|
echo "Performing health check..."
|
|
sleep {{ health_check_delay }}
|
|
|
|
retries={{ health_check_retries }}
|
|
while [ $retries -gt 0 ]; do
|
|
if [ "$(docker inspect "$container" --format '{{.State.Status}}' 2>/dev/null)" = "running" ]; then
|
|
echo "✅ Container is healthy"
|
|
break
|
|
else
|
|
echo "⏳ Waiting for container to be ready... ($retries retries left)"
|
|
sleep {{ health_check_delay }}
|
|
retries=$((retries - 1))
|
|
fi
|
|
done
|
|
|
|
if [ $retries -eq 0 ]; then
|
|
echo "❌ Container failed health check"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "❌ Failed to update container"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "⚠️ No compose file found, using direct Docker commands"
|
|
docker restart "$container"
|
|
fi
|
|
else
|
|
echo "❌ Failed to pull image"
|
|
exit 1
|
|
fi
|
|
register: specific_update
|
|
when: target_container is defined
|
|
|
|
when: not skip_docker
|
|
|
|
- name: Post-update verification
|
|
shell: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "Docker not available"
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== POST-UPDATE VERIFICATION ==="
|
|
|
|
# Check all containers are running
|
|
echo "Container Status Check:"
|
|
failed_containers=""
|
|
docker ps -a --format "{{.Names}}\t{{.Status}}" 2>/dev/null | while IFS=$'\t' read name status; do
|
|
if [ -n "$name" ]; then
|
|
if echo "$status" | grep -q "Up"; then
|
|
echo "✅ $name: $status"
|
|
else
|
|
echo "❌ $name: $status"
|
|
failed_containers="$failed_containers $name"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Check system resources after update
|
|
echo ""
|
|
echo "System Resources After Update:"
|
|
echo "Memory: $(free -h | awk 'NR==2{print $3"/"$2" ("$3*100/$2"%)"}')"
|
|
echo "Load: $(uptime | awk -F'load average:' '{print $2}')"
|
|
|
|
# Check for any error logs
|
|
echo ""
|
|
echo "Recent Error Logs:"
|
|
docker ps --format "{{.Names}}" 2>/dev/null | head -5 | while read container; do
|
|
if [ -n "$container" ]; then
|
|
errors=$(docker logs "$container" --since="5m" 2>&1 | grep -i error | wc -l)
|
|
if [ "$errors" -gt "0" ]; then
|
|
echo "⚠️ $container: $errors error(s) in last 5 minutes"
|
|
fi
|
|
fi
|
|
done
|
|
register: post_update_verification
|
|
changed_when: false
|
|
when: not skip_docker
|
|
|
|
- name: Rollback on failure
|
|
shell: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "Docker not available"
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== ROLLBACK PROCEDURE ==="
|
|
|
|
# Check if rollback is needed
|
|
failed_containers=$(docker ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null | head -5)
|
|
|
|
if [ -n "$failed_containers" ]; then
|
|
echo "Failed containers detected: $failed_containers"
|
|
echo "Initiating rollback..."
|
|
|
|
for container in $failed_containers; do
|
|
echo "Rolling back: $container"
|
|
|
|
# Find backup image
|
|
backup_image=$(docker images --format "{{.Repository}}" | grep "${container}_backup_" | head -1)
|
|
|
|
if [ -n "$backup_image" ]; then
|
|
echo " Found backup image: $backup_image"
|
|
|
|
# Stop current container
|
|
docker stop "$container" 2>/dev/null || true
|
|
docker rm "$container" 2>/dev/null || true
|
|
|
|
# Start container from backup image
|
|
if docker run -d --name "$container" "$backup_image"; then
|
|
echo " ✅ Rollback successful"
|
|
else
|
|
echo " ❌ Rollback failed"
|
|
fi
|
|
else
|
|
echo " ⚠️ No backup image found"
|
|
fi
|
|
done
|
|
else
|
|
echo "No rollback needed - all containers are healthy"
|
|
fi
|
|
register: rollback_result
|
|
when: not skip_docker and rollback_enabled and (orchestrated_updates.rc is defined and orchestrated_updates.rc != 0) or (specific_update.rc is defined and specific_update.rc != 0)
|
|
ignore_errors: yes
|
|
|
|
- name: Cleanup old backup images
|
|
shell: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "Docker not available"
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== CLEANUP OLD BACKUPS ==="
|
|
|
|
# Remove backup images older than 7 days
|
|
old_backups=$(docker images --format "{{.Repository}}\t{{.CreatedAt}}" | grep "_backup_" | awk '$2 < "'$(date -d '7 days ago' '+%Y-%m-%d')'"' | cut -f1)
|
|
|
|
if [ -n "$old_backups" ]; then
|
|
echo "Removing old backup images:"
|
|
for backup in $old_backups; do
|
|
echo " Removing: $backup"
|
|
docker rmi "$backup" 2>/dev/null || echo " Failed to remove $backup"
|
|
done
|
|
else
|
|
echo "No old backup images to clean up"
|
|
fi
|
|
|
|
# Clean up temporary backup files
|
|
find /tmp -name "*.backup.*" -mtime +7 -delete 2>/dev/null || true
|
|
register: cleanup_result
|
|
when: not skip_docker
|
|
ignore_errors: yes
|
|
|
|
- name: Create update report
|
|
set_fact:
|
|
update_report:
|
|
timestamp: "{{ update_timestamp }}"
|
|
hostname: "{{ inventory_hostname }}"
|
|
docker_available: "{{ not skip_docker }}"
|
|
pre_update_check: "{{ pre_update_check.stdout }}"
|
|
container_discovery: "{{ container_discovery.stdout if not skip_docker else 'Docker not available' }}"
|
|
backup_snapshots: "{{ backup_snapshots.stdout if not skip_docker and rollback_enabled else 'Snapshots disabled' }}"
|
|
orchestrated_updates: "{{ orchestrated_updates.stdout if orchestrated_updates is defined else 'Not performed' }}"
|
|
specific_update: "{{ specific_update.stdout if specific_update is defined else 'Not performed' }}"
|
|
post_update_verification: "{{ post_update_verification.stdout if not skip_docker else 'Docker not available' }}"
|
|
rollback_result: "{{ rollback_result.stdout if rollback_result is defined else 'Not needed' }}"
|
|
cleanup_result: "{{ cleanup_result.stdout if not skip_docker else 'Docker not available' }}"
|
|
|
|
- name: Display update report
|
|
debug:
|
|
msg: |
|
|
|
|
==========================================
|
|
🔄 CONTAINER UPDATE REPORT - {{ inventory_hostname }}
|
|
==========================================
|
|
|
|
📊 DOCKER AVAILABLE: {{ 'Yes' if update_report.docker_available else 'No' }}
|
|
|
|
🔍 PRE-UPDATE CHECK:
|
|
{{ update_report.pre_update_check }}
|
|
|
|
🔍 CONTAINER DISCOVERY:
|
|
{{ update_report.container_discovery }}
|
|
|
|
💾 BACKUP SNAPSHOTS:
|
|
{{ update_report.backup_snapshots }}
|
|
|
|
🔄 ORCHESTRATED UPDATES:
|
|
{{ update_report.orchestrated_updates }}
|
|
|
|
🎯 SPECIFIC UPDATE:
|
|
{{ update_report.specific_update }}
|
|
|
|
✅ POST-UPDATE VERIFICATION:
|
|
{{ update_report.post_update_verification }}
|
|
|
|
↩️ ROLLBACK RESULT:
|
|
{{ update_report.rollback_result }}
|
|
|
|
🧹 CLEANUP RESULT:
|
|
{{ update_report.cleanup_result }}
|
|
|
|
==========================================
|
|
|
|
- name: Generate JSON update report
|
|
copy:
|
|
content: |
|
|
{
|
|
"timestamp": "{{ update_report.timestamp }}",
|
|
"hostname": "{{ update_report.hostname }}",
|
|
"docker_available": {{ update_report.docker_available | lower }},
|
|
"pre_update_check": {{ update_report.pre_update_check | to_json }},
|
|
"container_discovery": {{ update_report.container_discovery | to_json }},
|
|
"backup_snapshots": {{ update_report.backup_snapshots | to_json }},
|
|
"orchestrated_updates": {{ update_report.orchestrated_updates | to_json }},
|
|
"specific_update": {{ update_report.specific_update | to_json }},
|
|
"post_update_verification": {{ update_report.post_update_verification | to_json }},
|
|
"rollback_result": {{ update_report.rollback_result | to_json }},
|
|
"cleanup_result": {{ update_report.cleanup_result | to_json }},
|
|
"recommendations": [
|
|
"Test updates in staging environment first",
|
|
"Monitor container health after updates",
|
|
"Maintain regular backup snapshots",
|
|
"Keep rollback procedures tested and ready",
|
|
"Schedule updates during maintenance windows"
|
|
]
|
|
}
|
|
dest: "{{ update_report_dir }}/{{ inventory_hostname }}_container_updates_{{ ansible_date_time.epoch }}.json"
|
|
delegate_to: localhost
|
|
|
|
- name: Summary message
|
|
debug:
|
|
msg: |
|
|
|
|
🔄 Container update orchestration complete for {{ inventory_hostname }}
|
|
📄 Report saved to: {{ update_report_dir }}/{{ inventory_hostname }}_container_updates_{{ ansible_date_time.epoch }}.json
|
|
|
|
{% if target_container is defined %}
|
|
🎯 Updated container: {{ target_container }}
|
|
{% endif %}
|
|
|
|
{% if update_mode is defined %}
|
|
🔄 Update mode: {{ update_mode }}
|
|
{% endif %}
|
|
|
|
💡 Use -e target_container=<name> to update specific containers
|
|
💡 Use -e update_mode=orchestrated for priority-based updates
|
|
💡 Use -e rollback_enabled=false to disable automatic rollback
|