426 lines
16 KiB
YAML
426 lines
16 KiB
YAML
---
|
||
# Log Rotation and Cleanup Playbook
|
||
# Manage log files across all services and system components
|
||
# Usage: ansible-playbook playbooks/log_rotation.yml
|
||
# Usage: ansible-playbook playbooks/log_rotation.yml -e "aggressive_cleanup=true"
|
||
# Usage: ansible-playbook playbooks/log_rotation.yml -e "dry_run=true"
|
||
|
||
- name: Log Rotation and Cleanup
|
||
hosts: "{{ host_target | default('all') }}"
|
||
gather_facts: yes
|
||
vars:
|
||
dry_run: "{{ dry_run | default(false) }}"
|
||
aggressive_cleanup: "{{ aggressive_cleanup | default(false) }}"
|
||
max_log_age_days: "{{ max_log_age_days | default(30) }}"
|
||
max_log_size: "{{ max_log_size | default('100M') }}"
|
||
keep_compressed_logs: "{{ keep_compressed_logs | default(true) }}"
|
||
compress_old_logs: "{{ compress_old_logs | default(true) }}"
|
||
|
||
tasks:
|
||
- name: Create log cleanup report directory
|
||
file:
|
||
path: "/tmp/log_cleanup/{{ ansible_date_time.date }}"
|
||
state: directory
|
||
mode: '0755'
|
||
|
||
- name: Display log cleanup plan
|
||
debug:
|
||
msg: |
|
||
📝 LOG ROTATION AND CLEANUP PLAN
|
||
================================
|
||
🖥️ Host: {{ inventory_hostname }}
|
||
📅 Date: {{ ansible_date_time.date }}
|
||
🧪 Dry Run: {{ dry_run }}
|
||
💪 Aggressive: {{ aggressive_cleanup }}
|
||
📅 Max Age: {{ max_log_age_days }} days
|
||
📦 Max Size: {{ max_log_size }}
|
||
🗜️ Compress: {{ compress_old_logs }}
|
||
|
||
- name: Analyze current log usage
|
||
shell: |
|
||
echo "📊 LOG USAGE ANALYSIS"
|
||
echo "===================="
|
||
|
||
total_log_size=0
|
||
log_file_count=0
|
||
|
||
echo "=== SYSTEM LOGS ==="
|
||
if [ -d "/var/log" ]; then
|
||
system_log_size=$(du -sh /var/log 2>/dev/null | cut -f1 || echo "0")
|
||
system_log_count=$(find /var/log -type f -name "*.log" 2>/dev/null | wc -l)
|
||
echo "System logs: $system_log_size ($system_log_count files)"
|
||
|
||
echo "Largest system logs:"
|
||
find /var/log -type f -name "*.log" -exec du -h {} \; 2>/dev/null | sort -hr | head -10 || echo "No system logs found"
|
||
fi
|
||
|
||
echo ""
|
||
echo "=== DOCKER CONTAINER LOGS ==="
|
||
if [ -d "/var/lib/docker/containers" ]; then
|
||
docker_log_size=$(du -sh /var/lib/docker/containers 2>/dev/null | cut -f1 || echo "0")
|
||
docker_log_count=$(find /var/lib/docker/containers -name "*-json.log" 2>/dev/null | wc -l)
|
||
echo "Docker logs: $docker_log_size ($docker_log_count files)"
|
||
|
||
echo "Largest container logs:"
|
||
find /var/lib/docker/containers -name "*-json.log" -exec du -h {} \; 2>/dev/null | sort -hr | head -10 || echo "No Docker logs found"
|
||
fi
|
||
|
||
echo ""
|
||
echo "=== APPLICATION LOGS ==="
|
||
for log_dir in /volume1/docker /opt/docker /home; do
|
||
if [ -d "$log_dir" ]; then
|
||
app_logs=$(find "$log_dir" -name "*.log" -type f 2>/dev/null | head -20)
|
||
if [ -n "$app_logs" ]; then
|
||
echo "Application logs in $log_dir:"
|
||
echo "$app_logs" | while read log_file; do
|
||
if [ -f "$log_file" ]; then
|
||
du -h "$log_file" 2>/dev/null || echo "Cannot access $log_file"
|
||
fi
|
||
done
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo "=== LARGE LOG FILES (>{{ max_log_size }}) ==="
|
||
find /var/log /var/lib/docker/containers /volume1 /opt -name "*.log" -size +{{ max_log_size }} -type f 2>/dev/null | head -20 | while read large_log; do
|
||
echo "$(du -h "$large_log" 2>/dev/null || echo "? $large_log")"
|
||
done || echo "No large log files found"
|
||
|
||
echo ""
|
||
echo "=== OLD LOG FILES (>{{ max_log_age_days }} days) ==="
|
||
old_logs=$(find /var/log /var/lib/docker/containers /volume1 /opt -name "*.log" -mtime +{{ max_log_age_days }} -type f 2>/dev/null | wc -l)
|
||
echo "Old log files found: $old_logs"
|
||
register: log_analysis
|
||
changed_when: false
|
||
|
||
- name: Rotate system logs
|
||
shell: |
|
||
echo "🔄 SYSTEM LOG ROTATION"
|
||
echo "====================="
|
||
|
||
rotated_logs=()
|
||
|
||
{% if dry_run %}
|
||
echo "DRY RUN: System log rotation simulation"
|
||
|
||
# Check what would be rotated
|
||
if command -v logrotate >/dev/null 2>&1; then
|
||
echo "Would run: logrotate -d /etc/logrotate.conf"
|
||
logrotate -d /etc/logrotate.conf 2>/dev/null | head -20 || echo "Logrotate config not found"
|
||
fi
|
||
{% else %}
|
||
# Force log rotation
|
||
if command -v logrotate >/dev/null 2>&1; then
|
||
echo "Running logrotate..."
|
||
logrotate -f /etc/logrotate.conf 2>/dev/null && echo "✅ System log rotation completed" || echo "⚠️ Logrotate had issues"
|
||
rotated_logs+=("system_logs")
|
||
else
|
||
echo "⚠️ Logrotate not available"
|
||
fi
|
||
|
||
# Manual rotation for specific large logs
|
||
for log_file in /var/log/syslog /var/log/auth.log /var/log/kern.log; do
|
||
if [ -f "$log_file" ] && [ $(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0) -gt 104857600 ]; then # 100MB
|
||
echo "Rotating large log: $log_file"
|
||
{% if compress_old_logs %}
|
||
gzip -c "$log_file" > "${log_file}.$(date +%Y%m%d).gz" && > "$log_file"
|
||
{% else %}
|
||
cp "$log_file" "${log_file}.$(date +%Y%m%d)" && > "$log_file"
|
||
{% endif %}
|
||
rotated_logs+=("$(basename $log_file)")
|
||
fi
|
||
done
|
||
{% endif %}
|
||
|
||
echo "📊 ROTATION SUMMARY:"
|
||
for log in "${rotated_logs[@]}"; do
|
||
echo " - $log"
|
||
done
|
||
register: system_log_rotation
|
||
|
||
- name: Manage Docker container logs
|
||
shell: |
|
||
echo "🐳 DOCKER LOG MANAGEMENT"
|
||
echo "========================"
|
||
|
||
managed_containers=()
|
||
total_space_saved=0
|
||
|
||
{% if dry_run %}
|
||
echo "DRY RUN: Docker log management simulation"
|
||
|
||
# Show what would be cleaned
|
||
large_logs=$(find /var/lib/docker/containers -name "*-json.log" -size +{{ max_log_size }} 2>/dev/null)
|
||
if [ -n "$large_logs" ]; then
|
||
echo "Would truncate large container logs:"
|
||
echo "$large_logs" | while read log_file; do
|
||
size=$(du -h "$log_file" 2>/dev/null | cut -f1)
|
||
container_id=$(basename $(dirname "$log_file"))
|
||
container_name=$(docker ps -a --filter "id=$container_id" --format "{{.Names}}" 2>/dev/null || echo "unknown")
|
||
echo " - $container_name: $size"
|
||
done
|
||
fi
|
||
{% else %}
|
||
# Truncate large container logs
|
||
find /var/lib/docker/containers -name "*-json.log" -size +{{ max_log_size }} 2>/dev/null | while read log_file; do
|
||
if [ -f "$log_file" ]; then
|
||
container_id=$(basename $(dirname "$log_file"))
|
||
container_name=$(docker ps -a --filter "id=$container_id" --format "{{.Names}}" 2>/dev/null || echo "unknown")
|
||
|
||
size_before=$(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0)
|
||
|
||
echo "Truncating log for container: $container_name"
|
||
|
||
# Keep last 1000 lines
|
||
tail -1000 "$log_file" > "${log_file}.tmp" && mv "${log_file}.tmp" "$log_file"
|
||
|
||
size_after=$(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0)
|
||
space_saved=$((size_before - size_after))
|
||
total_space_saved=$((total_space_saved + space_saved))
|
||
|
||
managed_containers+=("$container_name")
|
||
echo " ✅ Truncated: $(echo $space_saved | numfmt --to=iec) saved"
|
||
fi
|
||
done
|
||
|
||
# Clean up old rotated Docker logs
|
||
{% if aggressive_cleanup %}
|
||
echo "Cleaning old Docker log files..."
|
||
find /var/lib/docker/containers -name "*.log.*" -mtime +{{ max_log_age_days }} -delete 2>/dev/null
|
||
{% endif %}
|
||
{% endif %}
|
||
|
||
echo "📊 DOCKER LOG SUMMARY:"
|
||
echo "Containers managed: ${#managed_containers[@]}"
|
||
if [ $total_space_saved -gt 0 ]; then
|
||
echo "Total space saved: $(echo $total_space_saved | numfmt --to=iec)"
|
||
fi
|
||
|
||
for container in "${managed_containers[@]}"; do
|
||
echo " - $container"
|
||
done
|
||
register: docker_log_management
|
||
|
||
- name: Clean up application logs
|
||
shell: |
|
||
echo "📱 APPLICATION LOG CLEANUP"
|
||
echo "=========================="
|
||
|
||
cleaned_apps=()
|
||
|
||
{% if dry_run %}
|
||
echo "DRY RUN: Application log cleanup simulation"
|
||
|
||
# Show what would be cleaned
|
||
for log_dir in /volume1/docker /opt/docker; do
|
||
if [ -d "$log_dir" ]; then
|
||
old_app_logs=$(find "$log_dir" -name "*.log" -mtime +{{ max_log_age_days }} -type f 2>/dev/null)
|
||
if [ -n "$old_app_logs" ]; then
|
||
echo "Would clean logs in $log_dir:"
|
||
echo "$old_app_logs" | head -10
|
||
fi
|
||
fi
|
||
done
|
||
{% else %}
|
||
# Clean old application logs
|
||
for log_dir in /volume1/docker /opt/docker; do
|
||
if [ -d "$log_dir" ]; then
|
||
echo "Cleaning logs in $log_dir..."
|
||
|
||
# Compress old logs if requested
|
||
{% if compress_old_logs %}
|
||
find "$log_dir" -name "*.log" -mtime +7 -mtime -{{ max_log_age_days }} -type f 2>/dev/null | while read log_file; do
|
||
if [ -f "$log_file" ] && [[ "$log_file" != *.gz ]]; then
|
||
gzip "$log_file" 2>/dev/null && echo " Compressed: $(basename $log_file)"
|
||
fi
|
||
done
|
||
{% endif %}
|
||
|
||
# Remove very old logs
|
||
old_logs_removed=$(find "$log_dir" -name "*.log" -mtime +{{ max_log_age_days }} -type f -delete 2>/dev/null | wc -l)
|
||
{% if keep_compressed_logs %}
|
||
old_gz_removed=$(find "$log_dir" -name "*.log.gz" -mtime +$(({{ max_log_age_days }} * 2)) -type f -delete 2>/dev/null | wc -l)
|
||
{% else %}
|
||
old_gz_removed=$(find "$log_dir" -name "*.log.gz" -mtime +{{ max_log_age_days }} -type f -delete 2>/dev/null | wc -l)
|
||
{% endif %}
|
||
|
||
if [ $old_logs_removed -gt 0 ] || [ $old_gz_removed -gt 0 ]; then
|
||
echo " ✅ Cleaned $old_logs_removed logs, $old_gz_removed compressed logs"
|
||
cleaned_apps+=("$(basename $log_dir)")
|
||
fi
|
||
fi
|
||
done
|
||
{% endif %}
|
||
|
||
echo "📊 APPLICATION CLEANUP SUMMARY:"
|
||
for app in "${cleaned_apps[@]}"; do
|
||
echo " - $app"
|
||
done
|
||
register: app_log_cleanup
|
||
|
||
- name: Configure log rotation for services
|
||
shell: |
|
||
echo "⚙️ LOG ROTATION CONFIGURATION"
|
||
echo "============================="
|
||
|
||
config_updates=()
|
||
|
||
{% if dry_run %}
|
||
echo "DRY RUN: Would configure log rotation"
|
||
{% else %}
|
||
# Create custom logrotate config for Docker containers
|
||
logrotate_config="/etc/logrotate.d/docker-containers"
|
||
|
||
if [ ! -f "$logrotate_config" ]; then
|
||
echo "Creating Docker container log rotation config..."
|
||
cat > "$logrotate_config" << 'LOGROTATE_EOF'
|
||
/var/lib/docker/containers/*/*.log {
|
||
rotate 7
|
||
daily
|
||
compress
|
||
size 100M
|
||
missingok
|
||
delaycompress
|
||
copytruncate
|
||
}
|
||
LOGROTATE_EOF
|
||
config_updates+=("docker-containers")
|
||
echo " ✅ Docker container log rotation configured"
|
||
fi
|
||
|
||
# Update Docker daemon configuration for log limits
|
||
docker_config="/etc/docker/daemon.json"
|
||
if [ -f "$docker_config" ]; then
|
||
# Check if log driver is already configured
|
||
if ! grep -q "log-driver" "$docker_config" 2>/dev/null; then
|
||
echo "Updating Docker daemon log configuration..."
|
||
# Backup existing config
|
||
cp "$docker_config" "${docker_config}.backup.$(date +%Y%m%d)"
|
||
|
||
# Add log configuration (this would need careful JSON manipulation in practice)
|
||
echo " ℹ️ Manual Docker daemon config update recommended"
|
||
echo " Add: \"log-driver\": \"json-file\", \"log-opts\": {\"max-size\": \"{{ max_log_size }}\", \"max-file\": \"3\"}"
|
||
fi
|
||
fi
|
||
{% endif %}
|
||
|
||
echo "📊 CONFIGURATION SUMMARY:"
|
||
for config in "${config_updates[@]}"; do
|
||
echo " - $config"
|
||
done
|
||
register: log_rotation_config
|
||
|
||
- name: Generate log cleanup report
|
||
copy:
|
||
content: |
|
||
📝 LOG ROTATION AND CLEANUP REPORT - {{ inventory_hostname }}
|
||
==========================================================
|
||
|
||
📅 Cleanup Date: {{ ansible_date_time.iso8601 }}
|
||
🖥️ Host: {{ inventory_hostname }}
|
||
🧪 Dry Run: {{ dry_run }}
|
||
💪 Aggressive Mode: {{ aggressive_cleanup }}
|
||
📅 Max Age: {{ max_log_age_days }} days
|
||
📦 Max Size: {{ max_log_size }}
|
||
|
||
📊 LOG USAGE ANALYSIS:
|
||
{{ log_analysis.stdout }}
|
||
|
||
🔄 SYSTEM LOG ROTATION:
|
||
{{ system_log_rotation.stdout }}
|
||
|
||
🐳 DOCKER LOG MANAGEMENT:
|
||
{{ docker_log_management.stdout }}
|
||
|
||
📱 APPLICATION LOG CLEANUP:
|
||
{{ app_log_cleanup.stdout }}
|
||
|
||
⚙️ CONFIGURATION UPDATES:
|
||
{{ log_rotation_config.stdout }}
|
||
|
||
💡 RECOMMENDATIONS:
|
||
- Schedule regular log rotation via cron
|
||
- Monitor disk usage: ansible-playbook playbooks/disk_usage_report.yml
|
||
- Configure application-specific log rotation
|
||
- Set up log monitoring and alerting
|
||
{% if not dry_run %}
|
||
- Verify services are functioning after log cleanup
|
||
{% endif %}
|
||
{% if 'Large log files' in log_analysis.stdout %}
|
||
- Consider more aggressive log size limits
|
||
{% endif %}
|
||
|
||
📊 CLEANUP SUMMARY:
|
||
- System logs: {{ 'Rotated' if 'system_logs' in system_log_rotation.stdout else 'No action needed' }}
|
||
- Docker logs: {{ 'Managed' if 'managed' in docker_log_management.stdout else 'No action needed' }}
|
||
- Application logs: {{ 'Cleaned' if 'cleaned' in app_log_cleanup.stdout else 'No action needed' }}
|
||
- Configuration: {{ 'Updated' if 'config_updates' in log_rotation_config.stdout else 'No changes' }}
|
||
|
||
✅ LOG CLEANUP COMPLETE
|
||
|
||
dest: "/tmp/log_cleanup/{{ ansible_date_time.date }}/{{ inventory_hostname }}_log_cleanup_report.txt"
|
||
|
||
- name: Display log cleanup summary
|
||
debug:
|
||
msg: |
|
||
|
||
📝 LOG CLEANUP COMPLETE - {{ inventory_hostname }}
|
||
==========================================
|
||
|
||
📅 Date: {{ ansible_date_time.date }}
|
||
🧪 Mode: {{ 'Dry Run' if dry_run else 'Live Cleanup' }}
|
||
💪 Aggressive: {{ aggressive_cleanup }}
|
||
|
||
📊 ACTIONS TAKEN:
|
||
{{ system_log_rotation.stdout | regex_replace('\n.*', '') }}
|
||
{{ docker_log_management.stdout | regex_replace('\n.*', '') }}
|
||
{{ app_log_cleanup.stdout | regex_replace('\n.*', '') }}
|
||
|
||
📄 Full report: /tmp/log_cleanup/{{ ansible_date_time.date }}/{{ inventory_hostname }}_log_cleanup_report.txt
|
||
|
||
🔍 Next Steps:
|
||
{% if dry_run %}
|
||
- Run without dry_run to perform actual cleanup
|
||
{% endif %}
|
||
- Monitor disk usage improvements
|
||
- Schedule regular log rotation
|
||
- Verify service functionality
|
||
|
||
==========================================
|
||
|
||
- name: Restart services if needed
|
||
shell: |
|
||
echo "🔄 SERVICE RESTART CHECK"
|
||
echo "========================"
|
||
|
||
# Check if any critical services need restarting after log cleanup
|
||
services_to_restart=()
|
||
|
||
# Check if rsyslog needs restart (if log files were rotated)
|
||
if systemctl is-active --quiet rsyslog && [[ "{{ system_log_rotation.stdout }}" == *"system_logs"* ]]; then
|
||
services_to_restart+=("rsyslog")
|
||
fi
|
||
|
||
# Check if Docker daemon needs restart (if config was changed)
|
||
if [[ "{{ log_rotation_config.stdout }}" == *"docker"* ]]; then
|
||
echo "⚠️ Docker daemon config changed - manual restart may be needed"
|
||
echo " Run: sudo systemctl restart docker"
|
||
fi
|
||
|
||
{% if not dry_run %}
|
||
# Restart services that need it
|
||
for service in "${services_to_restart[@]}"; do
|
||
echo "Restarting $service..."
|
||
systemctl restart "$service" && echo " ✅ $service restarted" || echo " ❌ Failed to restart $service"
|
||
done
|
||
{% else %}
|
||
echo "DRY RUN: Would restart services: ${services_to_restart[*]}"
|
||
{% endif %}
|
||
|
||
if [ ${#services_to_restart[@]} -eq 0 ]; then
|
||
echo "ℹ️ No services need restarting"
|
||
fi
|
||
register: service_restart
|
||
when: restart_services | default(true) | bool
|