228 lines
8.2 KiB
YAML
228 lines
8.2 KiB
YAML
---
|
|
# Container Dependency Orchestrator
|
|
# Smart restart ordering with dependency management across hosts
|
|
# Run with: ansible-playbook -i hosts.ini playbooks/container_dependency_orchestrator.yml
|
|
|
|
- name: Container Dependency Orchestration
|
|
hosts: all
|
|
gather_facts: yes
|
|
vars:
|
|
# Define service dependency tiers (restart order)
|
|
dependency_tiers:
|
|
tier_1_infrastructure:
|
|
- "postgres"
|
|
- "mariadb"
|
|
- "mysql"
|
|
- "redis"
|
|
- "memcached"
|
|
- "mongo"
|
|
tier_2_core_services:
|
|
- "authentik-server"
|
|
- "authentik-worker"
|
|
- "gitea"
|
|
- "portainer"
|
|
- "nginx-proxy-manager"
|
|
tier_3_applications:
|
|
- "plex"
|
|
- "sonarr"
|
|
- "radarr"
|
|
- "lidarr"
|
|
- "bazarr"
|
|
- "prowlarr"
|
|
- "jellyseerr"
|
|
- "immich-server"
|
|
- "paperlessngx"
|
|
tier_4_monitoring:
|
|
- "prometheus"
|
|
- "grafana"
|
|
- "alertmanager"
|
|
- "node_exporter"
|
|
- "snmp_exporter"
|
|
tier_5_utilities:
|
|
- "watchtower"
|
|
- "syncthing"
|
|
- "ntfy"
|
|
|
|
# Cross-host dependencies
|
|
cross_host_dependencies:
|
|
- service: "immich-server"
|
|
depends_on:
|
|
- host: "atlantis"
|
|
service: "postgres"
|
|
- service: "gitea"
|
|
depends_on:
|
|
- host: "calypso"
|
|
service: "postgres"
|
|
|
|
tasks:
|
|
- name: Gather container information
|
|
docker_host_info:
|
|
containers: yes
|
|
register: docker_info
|
|
when: ansible_facts['os_family'] != "Synology"
|
|
|
|
- name: Get Synology container info via docker command
|
|
shell: docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
|
|
register: synology_containers
|
|
when: ansible_facts['os_family'] == "Synology"
|
|
become: yes
|
|
|
|
- name: Parse container information
|
|
set_fact:
|
|
running_containers: "{{ docker_info.containers | selectattr('State', 'equalto', 'running') | map(attribute='Names') | map('first') | list if docker_info is defined else [] }}"
|
|
stopped_containers: "{{ docker_info.containers | rejectattr('State', 'equalto', 'running') | map(attribute='Names') | map('first') | list if docker_info is defined else [] }}"
|
|
|
|
- name: Categorize containers by dependency tier
|
|
set_fact:
|
|
tier_containers:
|
|
tier_1: "{{ running_containers | select('match', '.*(' + (dependency_tiers.tier_1_infrastructure | join('|')) + ').*') | list }}"
|
|
tier_2: "{{ running_containers | select('match', '.*(' + (dependency_tiers.tier_2_core_services | join('|')) + ').*') | list }}"
|
|
tier_3: "{{ running_containers | select('match', '.*(' + (dependency_tiers.tier_3_applications | join('|')) + ').*') | list }}"
|
|
tier_4: "{{ running_containers | select('match', '.*(' + (dependency_tiers.tier_4_monitoring | join('|')) + ').*') | list }}"
|
|
tier_5: "{{ running_containers | select('match', '.*(' + (dependency_tiers.tier_5_utilities | join('|')) + ').*') | list }}"
|
|
|
|
- name: Display container categorization
|
|
debug:
|
|
msg: |
|
|
Container Dependency Analysis for {{ inventory_hostname }}:
|
|
|
|
Tier 1 (Infrastructure): {{ tier_containers.tier_1 | length }} containers
|
|
{{ tier_containers.tier_1 | join(', ') }}
|
|
|
|
Tier 2 (Core Services): {{ tier_containers.tier_2 | length }} containers
|
|
{{ tier_containers.tier_2 | join(', ') }}
|
|
|
|
Tier 3 (Applications): {{ tier_containers.tier_3 | length }} containers
|
|
{{ tier_containers.tier_3 | join(', ') }}
|
|
|
|
Tier 4 (Monitoring): {{ tier_containers.tier_4 | length }} containers
|
|
{{ tier_containers.tier_4 | join(', ') }}
|
|
|
|
Tier 5 (Utilities): {{ tier_containers.tier_5 | length }} containers
|
|
{{ tier_containers.tier_5 | join(', ') }}
|
|
|
|
- name: Check container health status
|
|
shell: docker inspect {{ item }} --format='{{.State.Health.Status}}' 2>/dev/null || echo "no-healthcheck"
|
|
register: health_checks
|
|
loop: "{{ running_containers }}"
|
|
become: yes
|
|
failed_when: false
|
|
|
|
- name: Identify unhealthy containers
|
|
set_fact:
|
|
unhealthy_containers: "{{ health_checks.results | selectattr('stdout', 'equalto', 'unhealthy') | map(attribute='item') | list }}"
|
|
healthy_containers: "{{ health_checks.results | selectattr('stdout', 'in', ['healthy', 'no-healthcheck']) | map(attribute='item') | list }}"
|
|
|
|
- name: Display health status
|
|
debug:
|
|
msg: |
|
|
Container Health Status for {{ inventory_hostname }}:
|
|
- Healthy/No Check: {{ healthy_containers | length }}
|
|
- Unhealthy: {{ unhealthy_containers | length }}
|
|
{% if unhealthy_containers %}
|
|
|
|
Unhealthy Containers:
|
|
{% for container in unhealthy_containers %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
|
|
- name: Restart unhealthy containers (Tier 1 first)
|
|
docker_container:
|
|
name: "{{ item }}"
|
|
state: started
|
|
restart: yes
|
|
loop: "{{ tier_containers.tier_1 | intersect(unhealthy_containers) }}"
|
|
when:
|
|
- restart_unhealthy | default(false) | bool
|
|
- unhealthy_containers | length > 0
|
|
become: yes
|
|
|
|
- name: Wait for Tier 1 containers to be healthy
|
|
shell: |
|
|
for i in {1..30}; do
|
|
status=$(docker inspect {{ item }} --format='{{.State.Health.Status}}' 2>/dev/null || echo "no-healthcheck")
|
|
if [[ "$status" == "healthy" || "$status" == "no-healthcheck" ]]; then
|
|
echo "Container {{ item }} is ready"
|
|
exit 0
|
|
fi
|
|
sleep 10
|
|
done
|
|
echo "Container {{ item }} failed to become healthy"
|
|
exit 1
|
|
loop: "{{ tier_containers.tier_1 | intersect(unhealthy_containers) }}"
|
|
when:
|
|
- restart_unhealthy | default(false) | bool
|
|
- unhealthy_containers | length > 0
|
|
become: yes
|
|
|
|
- name: Restart unhealthy containers (Tier 2)
|
|
docker_container:
|
|
name: "{{ item }}"
|
|
state: started
|
|
restart: yes
|
|
loop: "{{ tier_containers.tier_2 | intersect(unhealthy_containers) }}"
|
|
when:
|
|
- restart_unhealthy | default(false) | bool
|
|
- unhealthy_containers | length > 0
|
|
become: yes
|
|
|
|
- name: Generate dependency report
|
|
copy:
|
|
content: |
|
|
# Container Dependency Report - {{ inventory_hostname }}
|
|
Generated: {{ ansible_date_time.iso8601 }}
|
|
|
|
## Container Summary
|
|
- Total Running: {{ running_containers | length }}
|
|
- Total Stopped: {{ stopped_containers | length }}
|
|
- Healthy: {{ healthy_containers | length }}
|
|
- Unhealthy: {{ unhealthy_containers | length }}
|
|
|
|
## Dependency Tiers
|
|
|
|
### Tier 1 - Infrastructure ({{ tier_containers.tier_1 | length }})
|
|
{% for container in tier_containers.tier_1 %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
|
|
### Tier 2 - Core Services ({{ tier_containers.tier_2 | length }})
|
|
{% for container in tier_containers.tier_2 %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
|
|
### Tier 3 - Applications ({{ tier_containers.tier_3 | length }})
|
|
{% for container in tier_containers.tier_3 %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
|
|
### Tier 4 - Monitoring ({{ tier_containers.tier_4 | length }})
|
|
{% for container in tier_containers.tier_4 %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
|
|
### Tier 5 - Utilities ({{ tier_containers.tier_5 | length }})
|
|
{% for container in tier_containers.tier_5 %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
|
|
{% if unhealthy_containers %}
|
|
## Unhealthy Containers
|
|
{% for container in unhealthy_containers %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
|
|
{% if stopped_containers %}
|
|
## Stopped Containers
|
|
{% for container in stopped_containers %}
|
|
- {{ container }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
dest: "/tmp/container_dependency_{{ inventory_hostname }}_{{ ansible_date_time.epoch }}.md"
|
|
delegate_to: localhost
|
|
|
|
- name: Display report location
|
|
debug:
|
|
msg: "Dependency report saved to: /tmp/container_dependency_{{ inventory_hostname }}_{{ ansible_date_time.epoch }}.md"
|