Sanitized mirror from private repository - 2026-03-26 10:25:55 UTC

2026-03-26 10:25:55 +00:00
commit 5b8d0afef7
1267 changed files with 311880 additions and 0 deletions
--- a/ansible/automation/playbooks/log_rotation.yml
+++ b/ansible/automation/playbooks/log_rotation.yml
@@ -0,0 +1,425 @@
+---
+# Log Rotation and Cleanup Playbook
+# Manage log files across all services and system components
+# Usage: ansible-playbook playbooks/log_rotation.yml
+# Usage: ansible-playbook playbooks/log_rotation.yml -e "aggressive_cleanup=true"
+# Usage: ansible-playbook playbooks/log_rotation.yml -e "dry_run=true"
+
+- name: Log Rotation and Cleanup
+  hosts: "{{ host_target | default('all') }}"
+  gather_facts: yes
+  vars:
+    dry_run: "{{ dry_run | default(false) }}"
+    aggressive_cleanup: "{{ aggressive_cleanup | default(false) }}"
+    max_log_age_days: "{{ max_log_age_days | default(30) }}"
+    max_log_size: "{{ max_log_size | default('100M') }}"
+    keep_compressed_logs: "{{ keep_compressed_logs | default(true) }}"
+    compress_old_logs: "{{ compress_old_logs | default(true) }}"
+
+  tasks:
+    - name: Create log cleanup report directory
+      file:
+        path: "/tmp/log_cleanup/{{ ansible_date_time.date }}"
+        state: directory
+        mode: '0755'
+
+    - name: Display log cleanup plan
+      debug:
+        msg: |
+          📝 LOG ROTATION AND CLEANUP PLAN
+          ================================
+          🖥️  Host: {{ inventory_hostname }}
+          📅 Date: {{ ansible_date_time.date }}
+          🧪 Dry Run: {{ dry_run }}
+          💪 Aggressive: {{ aggressive_cleanup }}
+          📅 Max Age: {{ max_log_age_days }} days
+          📦 Max Size: {{ max_log_size }}
+          🗜️  Compress: {{ compress_old_logs }}
+
+    - name: Analyze current log usage
+      shell: |
+        echo "📊 LOG USAGE ANALYSIS"
+        echo "===================="
+
+        total_log_size=0
+        log_file_count=0
+
+        echo "=== SYSTEM LOGS ==="
+        if [ -d "/var/log" ]; then
+          system_log_size=$(du -sh /var/log 2>/dev/null | cut -f1 || echo "0")
+          system_log_count=$(find /var/log -type f -name "*.log" 2>/dev/null | wc -l)
+          echo "System logs: $system_log_size ($system_log_count files)"
+
+          echo "Largest system logs:"
+          find /var/log -type f -name "*.log" -exec du -h {} \; 2>/dev/null | sort -hr | head -10 || echo "No system logs found"
+        fi
+
+        echo ""
+        echo "=== DOCKER CONTAINER LOGS ==="
+        if [ -d "/var/lib/docker/containers" ]; then
+          docker_log_size=$(du -sh /var/lib/docker/containers 2>/dev/null | cut -f1 || echo "0")
+          docker_log_count=$(find /var/lib/docker/containers -name "*-json.log" 2>/dev/null | wc -l)
+          echo "Docker logs: $docker_log_size ($docker_log_count files)"
+
+          echo "Largest container logs:"
+          find /var/lib/docker/containers -name "*-json.log" -exec du -h {} \; 2>/dev/null | sort -hr | head -10 || echo "No Docker logs found"
+        fi
+
+        echo ""
+        echo "=== APPLICATION LOGS ==="
+        for log_dir in /volume1/docker /opt/docker /home; do
+          if [ -d "$log_dir" ]; then
+            app_logs=$(find "$log_dir" -name "*.log" -type f 2>/dev/null | head -20)
+            if [ -n "$app_logs" ]; then
+              echo "Application logs in $log_dir:"
+              echo "$app_logs" | while read log_file; do
+                if [ -f "$log_file" ]; then
+                  du -h "$log_file" 2>/dev/null || echo "Cannot access $log_file"
+                fi
+              done
+            fi
+          fi
+        done
+
+        echo ""
+        echo "=== LARGE LOG FILES (>{{ max_log_size }}) ==="
+        find /var/log /var/lib/docker/containers /volume1 /opt -name "*.log" -size +{{ max_log_size }} -type f 2>/dev/null | head -20 | while read large_log; do
+          echo "$(du -h "$large_log" 2>/dev/null || echo "? $large_log")"
+        done || echo "No large log files found"
+
+        echo ""
+        echo "=== OLD LOG FILES (>{{ max_log_age_days }} days) ==="
+        old_logs=$(find /var/log /var/lib/docker/containers /volume1 /opt -name "*.log" -mtime +{{ max_log_age_days }} -type f 2>/dev/null | wc -l)
+        echo "Old log files found: $old_logs"
+      register: log_analysis
+      changed_when: false
+
+    - name: Rotate system logs
+      shell: |
+        echo "🔄 SYSTEM LOG ROTATION"
+        echo "====================="
+
+        rotated_logs=()
+
+        {% if dry_run %}
+        echo "DRY RUN: System log rotation simulation"
+
+        # Check what would be rotated
+        if command -v logrotate >/dev/null 2>&1; then
+          echo "Would run: logrotate -d /etc/logrotate.conf"
+          logrotate -d /etc/logrotate.conf 2>/dev/null | head -20 || echo "Logrotate config not found"
+        fi
+        {% else %}
+        # Force log rotation
+        if command -v logrotate >/dev/null 2>&1; then
+          echo "Running logrotate..."
+          logrotate -f /etc/logrotate.conf 2>/dev/null && echo "✅ System log rotation completed" || echo "⚠️  Logrotate had issues"
+          rotated_logs+=("system_logs")
+        else
+          echo "⚠️  Logrotate not available"
+        fi
+
+        # Manual rotation for specific large logs
+        for log_file in /var/log/syslog /var/log/auth.log /var/log/kern.log; do
+          if [ -f "$log_file" ] && [ $(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0) -gt 104857600 ]; then  # 100MB
+            echo "Rotating large log: $log_file"
+            {% if compress_old_logs %}
+            gzip -c "$log_file" > "${log_file}.$(date +%Y%m%d).gz" && > "$log_file"
+            {% else %}
+            cp "$log_file" "${log_file}.$(date +%Y%m%d)" && > "$log_file"
+            {% endif %}
+            rotated_logs+=("$(basename $log_file)")
+          fi
+        done
+        {% endif %}
+
+        echo "📊 ROTATION SUMMARY:"
+        for log in "${rotated_logs[@]}"; do
+          echo "  - $log"
+        done
+      register: system_log_rotation
+
+    - name: Manage Docker container logs
+      shell: |
+        echo "🐳 DOCKER LOG MANAGEMENT"
+        echo "========================"
+
+        managed_containers=()
+        total_space_saved=0
+
+        {% if dry_run %}
+        echo "DRY RUN: Docker log management simulation"
+
+        # Show what would be cleaned
+        large_logs=$(find /var/lib/docker/containers -name "*-json.log" -size +{{ max_log_size }} 2>/dev/null)
+        if [ -n "$large_logs" ]; then
+          echo "Would truncate large container logs:"
+          echo "$large_logs" | while read log_file; do
+            size=$(du -h "$log_file" 2>/dev/null | cut -f1)
+            container_id=$(basename $(dirname "$log_file"))
+            container_name=$(docker ps -a --filter "id=$container_id" --format "{{.Names}}" 2>/dev/null || echo "unknown")
+            echo "  - $container_name: $size"
+          done
+        fi
+        {% else %}
+        # Truncate large container logs
+        find /var/lib/docker/containers -name "*-json.log" -size +{{ max_log_size }} 2>/dev/null | while read log_file; do
+          if [ -f "$log_file" ]; then
+            container_id=$(basename $(dirname "$log_file"))
+            container_name=$(docker ps -a --filter "id=$container_id" --format "{{.Names}}" 2>/dev/null || echo "unknown")
+
+            size_before=$(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0)
+
+            echo "Truncating log for container: $container_name"
+
+            # Keep last 1000 lines
+            tail -1000 "$log_file" > "${log_file}.tmp" && mv "${log_file}.tmp" "$log_file"
+
+            size_after=$(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file" 2>/dev/null || echo 0)
+            space_saved=$((size_before - size_after))
+            total_space_saved=$((total_space_saved + space_saved))
+
+            managed_containers+=("$container_name")
+            echo "  ✅ Truncated: $(echo $space_saved | numfmt --to=iec) saved"
+          fi
+        done
+
+        # Clean up old rotated Docker logs
+        {% if aggressive_cleanup %}
+        echo "Cleaning old Docker log files..."
+        find /var/lib/docker/containers -name "*.log.*" -mtime +{{ max_log_age_days }} -delete 2>/dev/null
+        {% endif %}
+        {% endif %}
+
+        echo "📊 DOCKER LOG SUMMARY:"
+        echo "Containers managed: ${#managed_containers[@]}"
+        if [ $total_space_saved -gt 0 ]; then
+          echo "Total space saved: $(echo $total_space_saved | numfmt --to=iec)"
+        fi
+
+        for container in "${managed_containers[@]}"; do
+          echo "  - $container"
+        done
+      register: docker_log_management
+
+    - name: Clean up application logs
+      shell: |
+        echo "📱 APPLICATION LOG CLEANUP"
+        echo "=========================="
+
+        cleaned_apps=()
+
+        {% if dry_run %}
+        echo "DRY RUN: Application log cleanup simulation"
+
+        # Show what would be cleaned
+        for log_dir in /volume1/docker /opt/docker; do
+          if [ -d "$log_dir" ]; then
+            old_app_logs=$(find "$log_dir" -name "*.log" -mtime +{{ max_log_age_days }} -type f 2>/dev/null)
+            if [ -n "$old_app_logs" ]; then
+              echo "Would clean logs in $log_dir:"
+              echo "$old_app_logs" | head -10
+            fi
+          fi
+        done
+        {% else %}
+        # Clean old application logs
+        for log_dir in /volume1/docker /opt/docker; do
+          if [ -d "$log_dir" ]; then
+            echo "Cleaning logs in $log_dir..."
+
+            # Compress old logs if requested
+            {% if compress_old_logs %}
+            find "$log_dir" -name "*.log" -mtime +7 -mtime -{{ max_log_age_days }} -type f 2>/dev/null | while read log_file; do
+              if [ -f "$log_file" ] && [[ "$log_file" != *.gz ]]; then
+                gzip "$log_file" 2>/dev/null && echo "  Compressed: $(basename $log_file)"
+              fi
+            done
+            {% endif %}
+
+            # Remove very old logs
+            old_logs_removed=$(find "$log_dir" -name "*.log" -mtime +{{ max_log_age_days }} -type f -delete 2>/dev/null | wc -l)
+            {% if keep_compressed_logs %}
+            old_gz_removed=$(find "$log_dir" -name "*.log.gz" -mtime +$(({{ max_log_age_days }} * 2)) -type f -delete 2>/dev/null | wc -l)
+            {% else %}
+            old_gz_removed=$(find "$log_dir" -name "*.log.gz" -mtime +{{ max_log_age_days }} -type f -delete 2>/dev/null | wc -l)
+            {% endif %}
+
+            if [ $old_logs_removed -gt 0 ] || [ $old_gz_removed -gt 0 ]; then
+              echo "  ✅ Cleaned $old_logs_removed logs, $old_gz_removed compressed logs"
+              cleaned_apps+=("$(basename $log_dir)")
+            fi
+          fi
+        done
+        {% endif %}
+
+        echo "📊 APPLICATION CLEANUP SUMMARY:"
+        for app in "${cleaned_apps[@]}"; do
+          echo "  - $app"
+        done
+      register: app_log_cleanup
+
+    - name: Configure log rotation for services
+      shell: |
+        echo "⚙️  LOG ROTATION CONFIGURATION"
+        echo "============================="
+
+        config_updates=()
+
+        {% if dry_run %}
+        echo "DRY RUN: Would configure log rotation"
+        {% else %}
+        # Create custom logrotate config for Docker containers
+        logrotate_config="/etc/logrotate.d/docker-containers"
+
+        if [ ! -f "$logrotate_config" ]; then
+          echo "Creating Docker container log rotation config..."
+          cat > "$logrotate_config" << 'LOGROTATE_EOF'
+/var/lib/docker/containers/*/*.log {
+    rotate 7
+    daily
+    compress
+    size 100M
+    missingok
+    delaycompress
+    copytruncate
+}
+LOGROTATE_EOF
+          config_updates+=("docker-containers")
+          echo "  ✅ Docker container log rotation configured"
+        fi
+
+        # Update Docker daemon configuration for log limits
+        docker_config="/etc/docker/daemon.json"
+        if [ -f "$docker_config" ]; then
+          # Check if log driver is already configured
+          if ! grep -q "log-driver" "$docker_config" 2>/dev/null; then
+            echo "Updating Docker daemon log configuration..."
+            # Backup existing config
+            cp "$docker_config" "${docker_config}.backup.$(date +%Y%m%d)"
+
+            # Add log configuration (this would need careful JSON manipulation in practice)
+            echo "  ℹ️  Manual Docker daemon config update recommended"
+            echo "  Add: \"log-driver\": \"json-file\", \"log-opts\": {\"max-size\": \"{{ max_log_size }}\", \"max-file\": \"3\"}"
+          fi
+        fi
+        {% endif %}
+
+        echo "📊 CONFIGURATION SUMMARY:"
+        for config in "${config_updates[@]}"; do
+          echo "  - $config"
+        done
+      register: log_rotation_config
+
+    - name: Generate log cleanup report
+      copy:
+        content: |
+          📝 LOG ROTATION AND CLEANUP REPORT - {{ inventory_hostname }}
+          ==========================================================
+
+          📅 Cleanup Date: {{ ansible_date_time.iso8601 }}
+          🖥️  Host: {{ inventory_hostname }}
+          🧪 Dry Run: {{ dry_run }}
+          💪 Aggressive Mode: {{ aggressive_cleanup }}
+          📅 Max Age: {{ max_log_age_days }} days
+          📦 Max Size: {{ max_log_size }}
+
+          📊 LOG USAGE ANALYSIS:
+          {{ log_analysis.stdout }}
+
+          🔄 SYSTEM LOG ROTATION:
+          {{ system_log_rotation.stdout }}
+
+          🐳 DOCKER LOG MANAGEMENT:
+          {{ docker_log_management.stdout }}
+
+          📱 APPLICATION LOG CLEANUP:
+          {{ app_log_cleanup.stdout }}
+
+          ⚙️  CONFIGURATION UPDATES:
+          {{ log_rotation_config.stdout }}
+
+          💡 RECOMMENDATIONS:
+          - Schedule regular log rotation via cron
+          - Monitor disk usage: ansible-playbook playbooks/disk_usage_report.yml
+          - Configure application-specific log rotation
+          - Set up log monitoring and alerting
+          {% if not dry_run %}
+          - Verify services are functioning after log cleanup
+          {% endif %}
+          {% if 'Large log files' in log_analysis.stdout %}
+          - Consider more aggressive log size limits
+          {% endif %}
+
+          📊 CLEANUP SUMMARY:
+          - System logs: {{ 'Rotated' if 'system_logs' in system_log_rotation.stdout else 'No action needed' }}
+          - Docker logs: {{ 'Managed' if 'managed' in docker_log_management.stdout else 'No action needed' }}
+          - Application logs: {{ 'Cleaned' if 'cleaned' in app_log_cleanup.stdout else 'No action needed' }}
+          - Configuration: {{ 'Updated' if 'config_updates' in log_rotation_config.stdout else 'No changes' }}
+
+          ✅ LOG CLEANUP COMPLETE
+
+        dest: "/tmp/log_cleanup/{{ ansible_date_time.date }}/{{ inventory_hostname }}_log_cleanup_report.txt"
+
+    - name: Display log cleanup summary
+      debug:
+        msg: |
+
+          📝 LOG CLEANUP COMPLETE - {{ inventory_hostname }}
+          ==========================================
+
+          📅 Date: {{ ansible_date_time.date }}
+          🧪 Mode: {{ 'Dry Run' if dry_run else 'Live Cleanup' }}
+          💪 Aggressive: {{ aggressive_cleanup }}
+
+          📊 ACTIONS TAKEN:
+          {{ system_log_rotation.stdout | regex_replace('\n.*', '') }}
+          {{ docker_log_management.stdout | regex_replace('\n.*', '') }}
+          {{ app_log_cleanup.stdout | regex_replace('\n.*', '') }}
+
+          📄 Full report: /tmp/log_cleanup/{{ ansible_date_time.date }}/{{ inventory_hostname }}_log_cleanup_report.txt
+
+          🔍 Next Steps:
+          {% if dry_run %}
+          - Run without dry_run to perform actual cleanup
+          {% endif %}
+          - Monitor disk usage improvements
+          - Schedule regular log rotation
+          - Verify service functionality
+
+          ==========================================
+
+    - name: Restart services if needed
+      shell: |
+        echo "🔄 SERVICE RESTART CHECK"
+        echo "========================"
+
+        # Check if any critical services need restarting after log cleanup
+        services_to_restart=()
+
+        # Check if rsyslog needs restart (if log files were rotated)
+        if systemctl is-active --quiet rsyslog && [[ "{{ system_log_rotation.stdout }}" == *"system_logs"* ]]; then
+          services_to_restart+=("rsyslog")
+        fi
+
+        # Check if Docker daemon needs restart (if config was changed)
+        if [[ "{{ log_rotation_config.stdout }}" == *"docker"* ]]; then
+          echo "⚠️  Docker daemon config changed - manual restart may be needed"
+          echo "  Run: sudo systemctl restart docker"
+        fi
+
+        {% if not dry_run %}
+        # Restart services that need it
+        for service in "${services_to_restart[@]}"; do
+          echo "Restarting $service..."
+          systemctl restart "$service" && echo "  ✅ $service restarted" || echo "  ❌ Failed to restart $service"
+        done
+        {% else %}
+        echo "DRY RUN: Would restart services: ${services_to_restart[*]}"
+        {% endif %}
+
+        if [ ${#services_to_restart[@]} -eq 0 ]; then
+          echo "ℹ️  No services need restarting"
+        fi
+      register: service_restart
+      when: restart_services | default(true) | bool