432 lines
16 KiB
YAML
432 lines
16 KiB
YAML
---
|
|
- name: Backup Verification and Testing
|
|
hosts: all
|
|
gather_facts: yes
|
|
vars:
|
|
verification_timestamp: "{{ ansible_date_time.iso8601 }}"
|
|
verification_report_dir: "/tmp/backup_verification"
|
|
backup_base_dir: "/opt/backups"
|
|
test_restore_dir: "/tmp/restore_test"
|
|
max_backup_age_days: 7
|
|
|
|
tasks:
|
|
- name: Create verification directories
|
|
file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
mode: '0755'
|
|
loop:
|
|
- "{{ verification_report_dir }}"
|
|
- "{{ test_restore_dir }}"
|
|
delegate_to: localhost
|
|
run_once: true
|
|
|
|
- name: Discover backup locations
|
|
shell: |
|
|
echo "=== BACKUP LOCATION DISCOVERY ==="
|
|
|
|
# Common backup directories
|
|
backup_dirs="/opt/backups /home/backups /var/backups /volume1/backups /mnt/backups"
|
|
|
|
echo "Searching for backup directories:"
|
|
for dir in $backup_dirs; do
|
|
if [ -d "$dir" ]; then
|
|
echo "✅ Found: $dir"
|
|
ls -la "$dir" 2>/dev/null | head -5
|
|
echo ""
|
|
fi
|
|
done
|
|
|
|
# Look for backup files in common locations
|
|
echo "Searching for backup files:"
|
|
find /opt /home /var -name "*.sql" -o -name "*.dump" -o -name "*.tar.gz" -o -name "*.zip" -o -name "*backup*" 2>/dev/null | head -20 | while read backup_file; do
|
|
if [ -f "$backup_file" ]; then
|
|
size=$(du -h "$backup_file" 2>/dev/null | cut -f1)
|
|
date=$(stat -c %y "$backup_file" 2>/dev/null | cut -d' ' -f1)
|
|
echo "📁 $backup_file ($size, $date)"
|
|
fi
|
|
done
|
|
register: backup_discovery
|
|
changed_when: false
|
|
|
|
- name: Analyze backup integrity
|
|
shell: |
|
|
echo "=== BACKUP INTEGRITY ANALYSIS ==="
|
|
|
|
# Check for recent backups
|
|
echo "Recent backup files (last {{ max_backup_age_days }} days):"
|
|
find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | while read backup_file; do
|
|
if [ -f "$backup_file" ]; then
|
|
size=$(du -h "$backup_file" 2>/dev/null | cut -f1)
|
|
date=$(stat -c %y "$backup_file" 2>/dev/null | cut -d' ' -f1)
|
|
|
|
# Basic integrity checks
|
|
integrity_status="✅ OK"
|
|
|
|
# Check if file is empty
|
|
if [ ! -s "$backup_file" ]; then
|
|
integrity_status="❌ EMPTY"
|
|
fi
|
|
|
|
# Check file extension and try basic validation
|
|
case "$backup_file" in
|
|
*.sql)
|
|
if ! head -1 "$backup_file" 2>/dev/null | grep -q "SQL\|CREATE\|INSERT\|--"; then
|
|
integrity_status="⚠️ SUSPICIOUS"
|
|
fi
|
|
;;
|
|
*.tar.gz)
|
|
if ! tar -tzf "$backup_file" >/dev/null 2>&1; then
|
|
integrity_status="❌ CORRUPT"
|
|
fi
|
|
;;
|
|
*.zip)
|
|
if command -v unzip >/dev/null 2>&1; then
|
|
if ! unzip -t "$backup_file" >/dev/null 2>&1; then
|
|
integrity_status="❌ CORRUPT"
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
echo "$integrity_status $backup_file ($size, $date)"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
# Check for old backups
|
|
echo "Old backup files (older than {{ max_backup_age_days }} days):"
|
|
old_backups=$(find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime +{{ max_backup_age_days }} 2>/dev/null | wc -l)
|
|
echo "Found $old_backups old backup files"
|
|
|
|
if [ "$old_backups" -gt "0" ]; then
|
|
echo "Oldest 5 backup files:"
|
|
find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime +{{ max_backup_age_days }} 2>/dev/null | head -5 | while read old_file; do
|
|
date=$(stat -c %y "$old_file" 2>/dev/null | cut -d' ' -f1)
|
|
size=$(du -h "$old_file" 2>/dev/null | cut -f1)
|
|
echo " $old_file ($size, $date)"
|
|
done
|
|
fi
|
|
register: integrity_analysis
|
|
changed_when: false
|
|
|
|
- name: Test database backup restoration
|
|
shell: |
|
|
echo "=== DATABASE BACKUP RESTORATION TEST ==="
|
|
|
|
# Find recent database backups
|
|
db_backups=$(find /opt /home /var -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | head -5)
|
|
|
|
if [ -z "$db_backups" ]; then
|
|
echo "No recent database backups found for testing"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Testing database backup restoration:"
|
|
|
|
for backup_file in $db_backups; do
|
|
echo "Testing: $backup_file"
|
|
|
|
# Determine database type from filename or content
|
|
db_type="unknown"
|
|
if echo "$backup_file" | grep -qi "postgres\|postgresql"; then
|
|
db_type="postgresql"
|
|
elif echo "$backup_file" | grep -qi "mysql\|mariadb"; then
|
|
db_type="mysql"
|
|
elif head -5 "$backup_file" 2>/dev/null | grep -qi "postgresql"; then
|
|
db_type="postgresql"
|
|
elif head -5 "$backup_file" 2>/dev/null | grep -qi "mysql"; then
|
|
db_type="mysql"
|
|
fi
|
|
|
|
echo " Detected type: $db_type"
|
|
|
|
# Basic syntax validation
|
|
case "$db_type" in
|
|
"postgresql")
|
|
if command -v psql >/dev/null 2>&1; then
|
|
# Test PostgreSQL backup syntax
|
|
if psql --set ON_ERROR_STOP=1 -f "$backup_file" -d template1 --dry-run 2>/dev/null; then
|
|
echo " ✅ PostgreSQL syntax valid"
|
|
else
|
|
echo " ⚠️ PostgreSQL syntax check failed (may require specific database)"
|
|
fi
|
|
else
|
|
echo " ⚠️ PostgreSQL client not available for testing"
|
|
fi
|
|
;;
|
|
"mysql")
|
|
if command -v mysql >/dev/null 2>&1; then
|
|
# Test MySQL backup syntax
|
|
if mysql --execute="source $backup_file" --force --dry-run 2>/dev/null; then
|
|
echo " ✅ MySQL syntax valid"
|
|
else
|
|
echo " ⚠️ MySQL syntax check failed (may require specific database)"
|
|
fi
|
|
else
|
|
echo " ⚠️ MySQL client not available for testing"
|
|
fi
|
|
;;
|
|
*)
|
|
# Generic SQL validation
|
|
if grep -q "CREATE\|INSERT\|UPDATE" "$backup_file" 2>/dev/null; then
|
|
echo " ✅ Contains SQL statements"
|
|
else
|
|
echo " ❌ No SQL statements found"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
done
|
|
register: db_restore_test
|
|
changed_when: false
|
|
ignore_errors: yes
|
|
|
|
- name: Test file backup restoration
|
|
shell: |
|
|
echo "=== FILE BACKUP RESTORATION TEST ==="
|
|
|
|
# Find recent archive backups
|
|
archive_backups=$(find /opt /home /var -name "*.tar.gz" -o -name "*.zip" -mtime -{{ max_backup_age_days }} 2>/dev/null | head -3)
|
|
|
|
if [ -z "$archive_backups" ]; then
|
|
echo "No recent archive backups found for testing"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Testing file backup restoration:"
|
|
|
|
for backup_file in $archive_backups; do
|
|
echo "Testing: $backup_file"
|
|
|
|
# Create test extraction directory
|
|
test_dir="{{ test_restore_dir }}/$(basename "$backup_file" | sed 's/\.[^.]*$//')_test"
|
|
mkdir -p "$test_dir"
|
|
|
|
case "$backup_file" in
|
|
*.tar.gz)
|
|
if tar -tzf "$backup_file" >/dev/null 2>&1; then
|
|
echo " ✅ Archive is readable"
|
|
|
|
# Test partial extraction
|
|
if tar -xzf "$backup_file" -C "$test_dir" --strip-components=1 2>/dev/null | head -5; then
|
|
extracted_files=$(find "$test_dir" -type f 2>/dev/null | wc -l)
|
|
echo " ✅ Extracted $extracted_files files successfully"
|
|
else
|
|
echo " ❌ Extraction failed"
|
|
fi
|
|
else
|
|
echo " ❌ Archive is corrupted or unreadable"
|
|
fi
|
|
;;
|
|
*.zip)
|
|
if command -v unzip >/dev/null 2>&1; then
|
|
if unzip -t "$backup_file" >/dev/null 2>&1; then
|
|
echo " ✅ ZIP archive is valid"
|
|
|
|
# Test partial extraction
|
|
if unzip -q "$backup_file" -d "$test_dir" 2>/dev/null; then
|
|
extracted_files=$(find "$test_dir" -type f 2>/dev/null | wc -l)
|
|
echo " ✅ Extracted $extracted_files files successfully"
|
|
else
|
|
echo " ❌ Extraction failed"
|
|
fi
|
|
else
|
|
echo " ❌ ZIP archive is corrupted"
|
|
fi
|
|
else
|
|
echo " ⚠️ unzip command not available"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# Cleanup test directory
|
|
rm -rf "$test_dir" 2>/dev/null
|
|
echo ""
|
|
done
|
|
register: file_restore_test
|
|
changed_when: false
|
|
ignore_errors: yes
|
|
|
|
- name: Check backup automation status
|
|
shell: |
|
|
echo "=== BACKUP AUTOMATION STATUS ==="
|
|
|
|
# Check for cron jobs related to backups
|
|
echo "Cron jobs (backup-related):"
|
|
if command -v crontab >/dev/null 2>&1; then
|
|
crontab -l 2>/dev/null | grep -i backup || echo "No backup cron jobs found"
|
|
else
|
|
echo "Crontab not available"
|
|
fi
|
|
echo ""
|
|
|
|
# Check systemd timers
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
echo "Systemd timers (backup-related):"
|
|
systemctl list-timers --no-pager 2>/dev/null | grep -i backup || echo "No backup timers found"
|
|
echo ""
|
|
fi
|
|
|
|
# Check for Docker containers that might be doing backups
|
|
if command -v docker >/dev/null 2>&1; then
|
|
echo "Docker containers (backup-related):"
|
|
docker ps --format "{{.Names}}\t{{.Image}}" 2>/dev/null | grep -i backup || echo "No backup containers found"
|
|
echo ""
|
|
fi
|
|
|
|
# Check for backup scripts
|
|
echo "Backup scripts:"
|
|
find /opt /home /usr/local -name "*backup*" -type f -executable 2>/dev/null | head -10 | while read script; do
|
|
echo " $script"
|
|
done
|
|
register: automation_status
|
|
changed_when: false
|
|
|
|
- name: Generate backup health score
|
|
shell: |
|
|
echo "=== BACKUP HEALTH SCORE ==="
|
|
|
|
score=100
|
|
issues=0
|
|
|
|
# Check for recent backups
|
|
recent_backups=$(find /opt /home /var -name "*backup*" -o -name "*.sql" -o -name "*.dump" -mtime -{{ max_backup_age_days }} 2>/dev/null | wc -l)
|
|
if [ "$recent_backups" -eq "0" ]; then
|
|
echo "❌ No recent backups found (-30 points)"
|
|
score=$((score - 30))
|
|
issues=$((issues + 1))
|
|
elif [ "$recent_backups" -lt "3" ]; then
|
|
echo "⚠️ Few recent backups found (-10 points)"
|
|
score=$((score - 10))
|
|
issues=$((issues + 1))
|
|
else
|
|
echo "✅ Recent backups found (+0 points)"
|
|
fi
|
|
|
|
# Check for automation
|
|
cron_backups=$(crontab -l 2>/dev/null | grep -i backup | wc -l)
|
|
if [ "$cron_backups" -eq "0" ]; then
|
|
echo "⚠️ No automated backup jobs found (-20 points)"
|
|
score=$((score - 20))
|
|
issues=$((issues + 1))
|
|
else
|
|
echo "✅ Automated backup jobs found (+0 points)"
|
|
fi
|
|
|
|
# Check for old backups (retention policy)
|
|
old_backups=$(find /opt /home /var -name "*backup*" -mtime +30 2>/dev/null | wc -l)
|
|
if [ "$old_backups" -gt "10" ]; then
|
|
echo "⚠️ Many old backups found - consider cleanup (-5 points)"
|
|
score=$((score - 5))
|
|
issues=$((issues + 1))
|
|
else
|
|
echo "✅ Backup retention appears managed (+0 points)"
|
|
fi
|
|
|
|
# Determine health status
|
|
if [ "$score" -ge "90" ]; then
|
|
health_status="EXCELLENT"
|
|
elif [ "$score" -ge "70" ]; then
|
|
health_status="GOOD"
|
|
elif [ "$score" -ge "50" ]; then
|
|
health_status="FAIR"
|
|
else
|
|
health_status="POOR"
|
|
fi
|
|
|
|
echo ""
|
|
echo "BACKUP HEALTH SCORE: $score/100 ($health_status)"
|
|
echo "ISSUES FOUND: $issues"
|
|
register: health_score
|
|
changed_when: false
|
|
|
|
- name: Create verification report
|
|
set_fact:
|
|
verification_report:
|
|
timestamp: "{{ verification_timestamp }}"
|
|
hostname: "{{ inventory_hostname }}"
|
|
backup_discovery: "{{ backup_discovery.stdout }}"
|
|
integrity_analysis: "{{ integrity_analysis.stdout }}"
|
|
db_restore_test: "{{ db_restore_test.stdout }}"
|
|
file_restore_test: "{{ file_restore_test.stdout }}"
|
|
automation_status: "{{ automation_status.stdout }}"
|
|
health_score: "{{ health_score.stdout }}"
|
|
|
|
- name: Display verification report
|
|
debug:
|
|
msg: |
|
|
|
|
==========================================
|
|
🔍 BACKUP VERIFICATION - {{ inventory_hostname }}
|
|
==========================================
|
|
|
|
📁 BACKUP DISCOVERY:
|
|
{{ verification_report.backup_discovery }}
|
|
|
|
🔒 INTEGRITY ANALYSIS:
|
|
{{ verification_report.integrity_analysis }}
|
|
|
|
🗄️ DATABASE RESTORE TEST:
|
|
{{ verification_report.db_restore_test }}
|
|
|
|
📦 FILE RESTORE TEST:
|
|
{{ verification_report.file_restore_test }}
|
|
|
|
🤖 AUTOMATION STATUS:
|
|
{{ verification_report.automation_status }}
|
|
|
|
📊 HEALTH SCORE:
|
|
{{ verification_report.health_score }}
|
|
|
|
==========================================
|
|
|
|
- name: Generate JSON verification report
|
|
copy:
|
|
content: |
|
|
{
|
|
"timestamp": "{{ verification_report.timestamp }}",
|
|
"hostname": "{{ verification_report.hostname }}",
|
|
"backup_discovery": {{ verification_report.backup_discovery | to_json }},
|
|
"integrity_analysis": {{ verification_report.integrity_analysis | to_json }},
|
|
"db_restore_test": {{ verification_report.db_restore_test | to_json }},
|
|
"file_restore_test": {{ verification_report.file_restore_test | to_json }},
|
|
"automation_status": {{ verification_report.automation_status | to_json }},
|
|
"health_score": {{ verification_report.health_score | to_json }},
|
|
"recommendations": [
|
|
{% if 'No recent backups found' in verification_report.integrity_analysis %}
|
|
"Implement regular backup procedures",
|
|
{% endif %}
|
|
{% if 'No backup cron jobs found' in verification_report.automation_status %}
|
|
"Set up automated backup scheduling",
|
|
{% endif %}
|
|
{% if 'CORRUPT' in verification_report.integrity_analysis %}
|
|
"Investigate and fix corrupted backup files",
|
|
{% endif %}
|
|
{% if 'old backup files' in verification_report.integrity_analysis %}
|
|
"Implement backup retention policy",
|
|
{% endif %}
|
|
"Regular backup verification testing recommended"
|
|
]
|
|
}
|
|
dest: "{{ verification_report_dir }}/{{ inventory_hostname }}_backup_verification_{{ ansible_date_time.epoch }}.json"
|
|
delegate_to: localhost
|
|
|
|
- name: Cleanup test files
|
|
file:
|
|
path: "{{ test_restore_dir }}"
|
|
state: absent
|
|
ignore_errors: yes
|
|
|
|
- name: Summary message
|
|
debug:
|
|
msg: |
|
|
|
|
🔍 Backup verification complete for {{ inventory_hostname }}
|
|
📄 Report saved to: {{ verification_report_dir }}/{{ inventory_hostname }}_backup_verification_{{ ansible_date_time.epoch }}.json
|
|
|
|
💡 Regular backup verification ensures data recovery capability
|
|
💡 Test restore procedures periodically to validate backup integrity
|
|
💡 Monitor backup automation to ensure continuous protection
|