Sanitized mirror from private repository - 2026-04-18 11:19:59 UTC

ansible/automation/playbooks/backup_verification.yml

@@ -0,0 +1,431 @@
+---
+- name: Backup Verification and Testing
+  hosts: all
+  gather_facts: yes
+  vars:
+    verification_timestamp: "{{ ansible_date_time.iso8601 }}"
+    verification_report_dir: "/tmp/backup_verification"
+    backup_base_dir: "/opt/backups"
+    test_restore_dir: "/tmp/restore_test"
+    max_backup_age_days: 7
+
+  tasks:
+    - name: Create verification directories
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: '0755'
+      loop:
+        - "{{ verification_report_dir }}"
+        - "{{ test_restore_dir }}"
+      delegate_to: localhost
+      run_once: true
+
+    - name: Discover backup locations
+      shell: |
+        echo "=== BACKUP LOCATION DISCOVERY ==="
+
+        # Common backup directories
+        backup_dirs="/opt/backups /home/backups /var/backups /volume1/backups /mnt/backups"
+
+        echo "Searching for backup directories:"
+        for dir in $backup_dirs; do
+          if [ -d "$dir" ]; then
+            echo "✅ Found: $dir"
+            ls -la "$dir" 2>/dev/null | head -5
+            echo ""
+          fi
+        done
+
+        # Look for backup files in common locations
+        echo "Searching for backup files:"
+        find /opt /home /var -name "*.sql" -o -name "*.dump" -o -name "*.tar.gz" -o -name "*.zip" -o -name "*backup*" 2>/dev/null | head -20 | while read backup_file; do
+          if [ -f "$backup_file" ]; then
+            size=$(du -h "$backup_file" 2>/dev/null | cut -f1)
+            date=$(stat -c %y "$backup_file" 2>/dev/null | cut -d' ' -f1)
+            echo "📁 $backup_file ($size, $date)"
+          fi
+        done
+      register: backup_discovery
+      changed_when: false
+
+    - name: Analyze backup integrity
+      shell: |
+        echo "=== BACKUP INTEGRITY ANALYSIS ==="
+
+        # Check for recent backups
+        echo "Recent backup files (last {{ max_backup_age_days }} days):"
+        find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | while read backup_file; do
+          if [ -f "$backup_file" ]; then
+            size=$(du -h "$backup_file" 2>/dev/null | cut -f1)
+            date=$(stat -c %y "$backup_file" 2>/dev/null | cut -d' ' -f1)
+
+            # Basic integrity checks
+            integrity_status="✅ OK"
+
+            # Check if file is empty
+            if [ ! -s "$backup_file" ]; then
+              integrity_status="❌ EMPTY"
+            fi
+
+            # Check file extension and try basic validation
+            case "$backup_file" in
+              *.sql)
+                if ! head -1 "$backup_file" 2>/dev/null | grep -q "SQL\|CREATE\|INSERT\|--"; then
+                  integrity_status="⚠️  SUSPICIOUS"
+                fi
+                ;;
+              *.tar.gz)
+                if ! tar -tzf "$backup_file" >/dev/null 2>&1; then
+                  integrity_status="❌ CORRUPT"
+                fi
+                ;;
+              *.zip)
+                if command -v unzip >/dev/null 2>&1; then
+                  if ! unzip -t "$backup_file" >/dev/null 2>&1; then
+                    integrity_status="❌ CORRUPT"
+                  fi
+                fi
+                ;;
+            esac
+
+            echo "$integrity_status $backup_file ($size, $date)"
+          fi
+        done
+        echo ""
+
+        # Check for old backups
+        echo "Old backup files (older than {{ max_backup_age_days }} days):"
+        old_backups=$(find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime +{{ max_backup_age_days }} 2>/dev/null | wc -l)
+        echo "Found $old_backups old backup files"
+
+        if [ "$old_backups" -gt "0" ]; then
+          echo "Oldest 5 backup files:"
+          find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime +{{ max_backup_age_days }} 2>/dev/null | head -5 | while read old_file; do
+            date=$(stat -c %y "$old_file" 2>/dev/null | cut -d' ' -f1)
+            size=$(du -h "$old_file" 2>/dev/null | cut -f1)
+            echo "  $old_file ($size, $date)"
+          done
+        fi
+      register: integrity_analysis
+      changed_when: false
+
+    - name: Test database backup restoration
+      shell: |
+        echo "=== DATABASE BACKUP RESTORATION TEST ==="
+
+        # Find recent database backups
+        db_backups=$(find /opt /home /var -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | head -5)
+
+        if [ -z "$db_backups" ]; then
+          echo "No recent database backups found for testing"
+          exit 0
+        fi
+
+        echo "Testing database backup restoration:"
+
+        for backup_file in $db_backups; do
+          echo "Testing: $backup_file"
+
+          # Determine database type from filename or content
+          db_type="unknown"
+          if echo "$backup_file" | grep -qi "postgres\|postgresql"; then
+            db_type="postgresql"
+          elif echo "$backup_file" | grep -qi "mysql\|mariadb"; then
+            db_type="mysql"
+          elif head -5 "$backup_file" 2>/dev/null | grep -qi "postgresql"; then
+            db_type="postgresql"
+          elif head -5 "$backup_file" 2>/dev/null | grep -qi "mysql"; then
+            db_type="mysql"
+          fi
+
+          echo "  Detected type: $db_type"
+
+          # Basic syntax validation
+          case "$db_type" in
+            "postgresql")
+              if command -v psql >/dev/null 2>&1; then
+                # Test PostgreSQL backup syntax
+                if psql --set ON_ERROR_STOP=1 -f "$backup_file" -d template1 --dry-run 2>/dev/null; then
+                  echo "  ✅ PostgreSQL syntax valid"
+                else
+                  echo "  ⚠️  PostgreSQL syntax check failed (may require specific database)"
+                fi
+              else
+                echo "  ⚠️  PostgreSQL client not available for testing"
+              fi
+              ;;
+            "mysql")
+              if command -v mysql >/dev/null 2>&1; then
+                # Test MySQL backup syntax
+                if mysql --execute="source $backup_file" --force --dry-run 2>/dev/null; then
+                  echo "  ✅ MySQL syntax valid"
+                else
+                  echo "  ⚠️  MySQL syntax check failed (may require specific database)"
+                fi
+              else
+                echo "  ⚠️  MySQL client not available for testing"
+              fi
+              ;;
+            *)
+              # Generic SQL validation
+              if grep -q "CREATE\|INSERT\|UPDATE" "$backup_file" 2>/dev/null; then
+                echo "  ✅ Contains SQL statements"
+              else
+                echo "  ❌ No SQL statements found"
+              fi
+              ;;
+          esac
+
+          echo ""
+        done
+      register: db_restore_test
+      changed_when: false
+      ignore_errors: yes
+
+    - name: Test file backup restoration
+      shell: |
+        echo "=== FILE BACKUP RESTORATION TEST ==="
+
+        # Find recent archive backups
+        archive_backups=$(find /opt /home /var -name "*.tar.gz" -o -name "*.zip" -mtime -{{ max_backup_age_days }} 2>/dev/null | head -3)
+
+        if [ -z "$archive_backups" ]; then
+          echo "No recent archive backups found for testing"
+          exit 0
+        fi
+
+        echo "Testing file backup restoration:"
+
+        for backup_file in $archive_backups; do
+          echo "Testing: $backup_file"
+
+          # Create test extraction directory
+          test_dir="{{ test_restore_dir }}/$(basename "$backup_file" | sed 's/\.[^.]*$//')_test"
+          mkdir -p "$test_dir"
+
+          case "$backup_file" in
+            *.tar.gz)
+              if tar -tzf "$backup_file" >/dev/null 2>&1; then
+                echo "  ✅ Archive is readable"
+
+                # Test partial extraction
+                if tar -xzf "$backup_file" -C "$test_dir" --strip-components=1 2>/dev/null | head -5; then
+                  extracted_files=$(find "$test_dir" -type f 2>/dev/null | wc -l)
+                  echo "  ✅ Extracted $extracted_files files successfully"
+                else
+                  echo "  ❌ Extraction failed"
+                fi
+              else
+                echo "  ❌ Archive is corrupted or unreadable"
+              fi
+              ;;
+            *.zip)
+              if command -v unzip >/dev/null 2>&1; then
+                if unzip -t "$backup_file" >/dev/null 2>&1; then
+                  echo "  ✅ ZIP archive is valid"
+
+                  # Test partial extraction
+                  if unzip -q "$backup_file" -d "$test_dir" 2>/dev/null; then
+                    extracted_files=$(find "$test_dir" -type f 2>/dev/null | wc -l)
+                    echo "  ✅ Extracted $extracted_files files successfully"
+                  else
+                    echo "  ❌ Extraction failed"
+                  fi
+                else
+                  echo "  ❌ ZIP archive is corrupted"
+                fi
+              else
+                echo "  ⚠️  unzip command not available"
+              fi
+              ;;
+          esac
+
+          # Cleanup test directory
+          rm -rf "$test_dir" 2>/dev/null
+          echo ""
+        done
+      register: file_restore_test
+      changed_when: false
+      ignore_errors: yes
+
+    - name: Check backup automation status
+      shell: |
+        echo "=== BACKUP AUTOMATION STATUS ==="
+
+        # Check for cron jobs related to backups
+        echo "Cron jobs (backup-related):"
+        if command -v crontab >/dev/null 2>&1; then
+          crontab -l 2>/dev/null | grep -i backup || echo "No backup cron jobs found"
+        else
+          echo "Crontab not available"
+        fi
+        echo ""
+
+        # Check systemd timers
+        if command -v systemctl >/dev/null 2>&1; then
+          echo "Systemd timers (backup-related):"
+          systemctl list-timers --no-pager 2>/dev/null | grep -i backup || echo "No backup timers found"
+          echo ""
+        fi
+
+        # Check for Docker containers that might be doing backups
+        if command -v docker >/dev/null 2>&1; then
+          echo "Docker containers (backup-related):"
+          docker ps --format "{{.Names}}\t{{.Image}}" 2>/dev/null | grep -i backup || echo "No backup containers found"
+          echo ""
+        fi
+
+        # Check for backup scripts
+        echo "Backup scripts:"
+        find /opt /home /usr/local -name "*backup*" -type f -executable 2>/dev/null | head -10 | while read script; do
+          echo "  $script"
+        done
+      register: automation_status
+      changed_when: false
+
+    - name: Generate backup health score
+      shell: |
+        echo "=== BACKUP HEALTH SCORE ==="
+
+        score=100
+        issues=0
+
+        # Check for recent backups
+        recent_backups=$(find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | wc -l)
+        if [ "$recent_backups" -eq "0" ]; then
+          echo "❌ No recent backups found (-30 points)"
+          score=$((score - 30))
+          issues=$((issues + 1))
+        elif [ "$recent_backups" -lt "3" ]; then
+          echo "⚠️  Few recent backups found (-10 points)"
+          score=$((score - 10))
+          issues=$((issues + 1))
+        else
+          echo "✅ Recent backups found (+0 points)"
+        fi
+
+        # Check for automation
+        cron_backups=$(crontab -l 2>/dev/null | grep -i backup | wc -l)
+        if [ "$cron_backups" -eq "0" ]; then
+          echo "⚠️  No automated backup jobs found (-20 points)"
+          score=$((score - 20))
+          issues=$((issues + 1))
+        else
+          echo "✅ Automated backup jobs found (+0 points)"
+        fi
+
+        # Check for old backups (retention policy)
+        old_backups=$(find /opt /home /var -name "*backup*" -mtime +30 2>/dev/null | wc -l)
+        if [ "$old_backups" -gt "10" ]; then
+          echo "⚠️  Many old backups found - consider cleanup (-5 points)"
+          score=$((score - 5))
+          issues=$((issues + 1))
+        else
+          echo "✅ Backup retention appears managed (+0 points)"
+        fi
+
+        # Determine health status
+        if [ "$score" -ge "90" ]; then
+          health_status="EXCELLENT"
+        elif [ "$score" -ge "70" ]; then
+          health_status="GOOD"
+        elif [ "$score" -ge "50" ]; then
+          health_status="FAIR"
+        else
+          health_status="POOR"
+        fi
+
+        echo ""
+        echo "BACKUP HEALTH SCORE: $score/100 ($health_status)"
+        echo "ISSUES FOUND: $issues"
+      register: health_score
+      changed_when: false
+
+    - name: Create verification report
+      set_fact:
+        verification_report:
+          timestamp: "{{ verification_timestamp }}"
+          hostname: "{{ inventory_hostname }}"
+          backup_discovery: "{{ backup_discovery.stdout }}"
+          integrity_analysis: "{{ integrity_analysis.stdout }}"
+          db_restore_test: "{{ db_restore_test.stdout }}"
+          file_restore_test: "{{ file_restore_test.stdout }}"
+          automation_status: "{{ automation_status.stdout }}"
+          health_score: "{{ health_score.stdout }}"
+
+    - name: Display verification report
+      debug:
+        msg: |
+
+          ==========================================
+          🔍 BACKUP VERIFICATION - {{ inventory_hostname }}
+          ==========================================
+
+          📁 BACKUP DISCOVERY:
+          {{ verification_report.backup_discovery }}
+
+          🔒 INTEGRITY ANALYSIS:
+          {{ verification_report.integrity_analysis }}
+
+          🗄️  DATABASE RESTORE TEST:
+          {{ verification_report.db_restore_test }}
+
+          📦 FILE RESTORE TEST:
+          {{ verification_report.file_restore_test }}
+
+          🤖 AUTOMATION STATUS:
+          {{ verification_report.automation_status }}
+
+          📊 HEALTH SCORE:
+          {{ verification_report.health_score }}
+
+          ==========================================
+
+    - name: Generate JSON verification report
+      copy:
+        content: |
+          {
+            "timestamp": "{{ verification_report.timestamp }}",
+            "hostname": "{{ verification_report.hostname }}",
+            "backup_discovery": {{ verification_report.backup_discovery | to_json }},
+            "integrity_analysis": {{ verification_report.integrity_analysis | to_json }},
+            "db_restore_test": {{ verification_report.db_restore_test | to_json }},
+            "file_restore_test": {{ verification_report.file_restore_test | to_json }},
+            "automation_status": {{ verification_report.automation_status | to_json }},
+            "health_score": {{ verification_report.health_score | to_json }},
+            "recommendations": [
+              {% if 'No recent backups found' in verification_report.integrity_analysis %}
+              "Implement regular backup procedures",
+              {% endif %}
+              {% if 'No backup cron jobs found' in verification_report.automation_status %}
+              "Set up automated backup scheduling",
+              {% endif %}
+              {% if 'CORRUPT' in verification_report.integrity_analysis %}
+              "Investigate and fix corrupted backup files",
+              {% endif %}
+              {% if 'old backup files' in verification_report.integrity_analysis %}
+              "Implement backup retention policy",
+              {% endif %}
+              "Regular backup verification testing recommended"
+            ]
+          }
+        dest: "{{ verification_report_dir }}/{{ inventory_hostname }}_backup_verification_{{ ansible_date_time.epoch }}.json"
+      delegate_to: localhost
+
+    - name: Cleanup test files
+      file:
+        path: "{{ test_restore_dir }}"
+        state: absent
+      ignore_errors: yes
+
+    - name: Summary message
+      debug:
+        msg: |
+
+          🔍 Backup verification complete for {{ inventory_hostname }}
+          📄 Report saved to: {{ verification_report_dir }}/{{ inventory_hostname }}_backup_verification_{{ ansible_date_time.epoch }}.json
+
+          💡 Regular backup verification ensures data recovery capability
+          💡 Test restore procedures periodically to validate backup integrity
+          💡 Monitor backup automation to ensure continuous protection