336 lines
17 KiB
Python
336 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Portainer Stack vs Git Repository Comparison Tool
|
|
Generates documentation comparing running stacks with repo configurations
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
# Endpoint ID to Server Name mapping
|
|
ENDPOINT_MAP = {
|
|
2: "Atlantis",
|
|
443395: "Concord NUC",
|
|
443397: "Calypso (vish-nuc)",
|
|
443398: "vish-nuc-edge",
|
|
443399: "Homelab VM"
|
|
}
|
|
|
|
# Server folder mapping in repo
|
|
REPO_FOLDER_MAP = {
|
|
"Atlantis": ["Atlantis"],
|
|
"Concord NUC": ["concord_nuc"],
|
|
"Calypso (vish-nuc)": ["Calypso"],
|
|
"vish-nuc-edge": [],
|
|
"Homelab VM": ["homelab_vm"]
|
|
}
|
|
|
|
# Running stacks data (collected from Portainer API)
|
|
RUNNING_STACKS = {
|
|
"Atlantis": {
|
|
"stacks": [
|
|
{"name": "arr-stack", "containers": ["deluge", "sonarr", "radarr", "lidarr", "gluetun", "jackett", "tautulli", "sabnzbd", "plex", "whisparr", "flaresolverr", "wizarr", "bazarr", "prowlarr", "jellyseerr"], "git_linked": True, "git_path": "Atlantis/arr-suite/"},
|
|
{"name": "nginx_repo-stack", "containers": ["nginx"], "git_linked": True, "git_path": "Atlantis/repo_nginx.yaml"},
|
|
{"name": "dyndns-updater-stack", "containers": ["ddns-vish-unproxied", "ddns-vish-proxied", "ddns-thevish-unproxied", "ddns-thevish-proxied"], "git_linked": True, "git_path": "Atlantis/dynamicdnsupdater.yaml"},
|
|
{"name": "baikal-stack", "containers": ["baikal"], "git_linked": True, "git_path": "Atlantis/baikal/"},
|
|
{"name": "jitsi", "containers": ["jitsi-web", "jitsi-jvb", "jitsi-jicofo", "coturn", "jitsi-prosody"], "git_linked": True, "git_path": "Atlantis/jitsi/"},
|
|
{"name": "youtubedl", "containers": ["youtube_downloader"], "git_linked": True, "git_path": "Atlantis/youtubedl.yaml"},
|
|
{"name": "matrix_synapse-stack", "containers": ["Synapse", "Synapse-DB"], "git_linked": True, "git_path": "Atlantis/synapse.yml", "issues": ["Synapse container exited"]},
|
|
{"name": "joplin-stack", "containers": ["joplin-app", "joplin-db"], "git_linked": True, "git_path": "Atlantis/joplin.yml"},
|
|
{"name": "immich-stack", "containers": ["Immich-SERVER", "Immich-LEARNING", "Immich-DB", "Immich-REDIS"], "git_linked": True, "git_path": "Atlantis/immich/"},
|
|
{"name": "vaultwarden-stack", "containers": ["Vaultwarden", "Vaultwarden-DB"], "git_linked": True, "git_path": "Atlantis/vaultwarden.yaml"},
|
|
{"name": "node-exporter-stack", "containers": ["snmp_exporter", "node_exporter"], "git_linked": False},
|
|
{"name": "fenrus-stack", "containers": ["Fenrus"], "git_linked": True, "git_path": "Atlantis/fenrus.yaml"},
|
|
{"name": "syncthing-stack", "containers": [], "git_linked": True, "git_path": "Atlantis/syncthing.yml", "status": "stopped"},
|
|
],
|
|
"standalone": ["portainer"]
|
|
},
|
|
"Concord NUC": {
|
|
"stacks": [
|
|
{"name": "invidious", "containers": ["invidious-companion", "invidious-db", "invidious"], "git_linked": True, "git_path": "concord_nuc/invidious/", "issues": ["invidious unhealthy"]},
|
|
{"name": "syncthing-stack", "containers": ["syncthing"], "git_linked": True, "git_path": "concord_nuc/syncthing.yaml"},
|
|
{"name": "homeassistant-stack", "containers": ["homeassistant", "matter-server"], "git_linked": True, "git_path": "concord_nuc/homeassistant.yaml"},
|
|
{"name": "adguard-stack", "containers": ["AdGuard"], "git_linked": True, "git_path": "concord_nuc/adguard.yaml"},
|
|
{"name": "yourspotify-stack", "containers": ["yourspotify-server", "mongo", "yourspotify-web"], "git_linked": True, "git_path": "concord_nuc/yourspotify.yaml"},
|
|
{"name": "dyndns-updater", "containers": ["ddns-vish-13340"], "git_linked": True, "git_path": "concord_nuc/dyndns_updater.yaml"},
|
|
{"name": "wireguard-stack", "containers": ["wg-easy"], "git_linked": True, "git_path": "concord_nuc/wireguard.yaml"},
|
|
{"name": "node-exporter", "containers": ["node_exporter"], "git_linked": False, "issues": ["restarting"]},
|
|
],
|
|
"standalone": ["portainer_edge_agent", "watchtower"],
|
|
"issues": ["watchtower restarting", "node_exporter restarting"]
|
|
},
|
|
"Calypso (vish-nuc)": {
|
|
"stacks": [
|
|
{"name": "arr-stack", "containers": ["jellyseerr", "bazarr", "sonarr", "lidarr", "prowlarr", "plex", "readarr", "radarr", "flaresolverr", "sabnzbd", "tautulli", "whisparr"], "git_linked": True, "git_path": "Calypso/arr_suite_with_dracula.yml"},
|
|
{"name": "rxv4-stack", "containers": ["Resume-ACCESS", "Resume-DB", "Resume-CHROME", "Resume-MINIO"], "git_linked": True, "git_path": "Calypso/reactive_resume_v4/"},
|
|
{"name": "seafile", "containers": ["Seafile-DB", "Seafile-CACHE", "Seafile-REDIS", "Seafile"], "git_linked": True, "git_path": "Calypso/seafile-server.yaml"},
|
|
{"name": "gitea", "containers": ["Gitea-DB", "Gitea"], "git_linked": True, "git_path": "Calypso/gitea-server.yaml"},
|
|
{"name": "paperless-testing", "containers": ["PaperlessNGX", "PaperlessNGX-REDIS", "PaperlessNGX-DB", "PaperlessNGX-GOTENBERG", "PaperlessNGX-TIKA"], "git_linked": False},
|
|
{"name": "paperless-ai", "containers": ["PaperlessNGX-AI"], "git_linked": False},
|
|
{"name": "rustdesk", "containers": ["Rustdesk-HBBS", "Rustdesk-HBBR"], "git_linked": False},
|
|
{"name": "immich-stack", "containers": ["Immich-SERVER", "Immich-LEARNING", "Immich-DB", "Immich-REDIS"], "git_linked": True, "git_path": "Calypso/immich/"},
|
|
{"name": "rackula-stack", "containers": ["Rackula"], "git_linked": True, "git_path": "Calypso/rackula.yml"},
|
|
{"name": "adguard-stack", "containers": ["AdGuard"], "git_linked": True, "git_path": "Calypso/adguard.yaml"},
|
|
{"name": "syncthing-stack", "containers": ["syncthing"], "git_linked": True, "git_path": "Calypso/syncthing.yaml"},
|
|
{"name": "node-exporter", "containers": ["snmp_exporter", "node_exporter"], "git_linked": False},
|
|
{"name": "actual-budget-stack", "containers": ["Actual"], "git_linked": True, "git_path": "Calypso/actualbudget.yml"},
|
|
{"name": "apt-cacher-ng", "containers": ["apt-cacher-ng"], "git_linked": True, "git_path": "Calypso/apt-cacher-ng/"},
|
|
{"name": "iperf3-stack", "containers": ["iperf3"], "git_linked": True, "git_path": "Calypso/iperf3.yml"},
|
|
{"name": "wireguard", "containers": ["wgeasy"], "git_linked": True, "git_path": "Calypso/wireguard-server.yaml"},
|
|
],
|
|
"standalone": ["portainer_edge_agent", "openspeedtest"]
|
|
},
|
|
"Homelab VM": {
|
|
"stacks": [
|
|
{"name": "openhands", "containers": ["openhands-app"], "git_linked": False},
|
|
{"name": "monitoring", "containers": ["prometheus", "grafana", "node_exporter"], "git_linked": True, "git_path": "homelab_vm/prometheus_grafana_hub/"},
|
|
{"name": "perplexica", "containers": ["perplexica"], "git_linked": False},
|
|
{"name": "syncthing-stack", "containers": ["syncthing"], "git_linked": True, "git_path": "homelab_vm/syncthing.yml"},
|
|
{"name": "hoarder-karakeep-stack", "containers": ["meilisearch", "web", "chrome"], "git_linked": True, "git_path": "homelab_vm/hoarder.yaml"},
|
|
{"name": "drawio-stack", "containers": ["Draw.io"], "git_linked": True, "git_path": "homelab_vm/drawio.yml"},
|
|
{"name": "redlib-stack", "containers": ["Libreddit"], "git_linked": True, "git_path": "homelab_vm/libreddit.yaml"},
|
|
{"name": "signal-api-stack", "containers": ["signal-api"], "git_linked": True, "git_path": "homelab_vm/signal_api.yaml"},
|
|
{"name": "binternet-stack", "containers": ["binternet"], "git_linked": True, "git_path": "homelab_vm/binternet.yaml"},
|
|
{"name": "archivebox-stack", "containers": ["archivebox_scheduler", "archivebox", "archivebox_sonic"], "git_linked": True, "git_path": "homelab_vm/archivebox.yaml"},
|
|
{"name": "watchyourlan-stack", "containers": ["WatchYourLAN"], "git_linked": True, "git_path": "homelab_vm/watchyourlan.yaml"},
|
|
{"name": "webcheck-stack", "containers": ["Web-Check"], "git_linked": True, "git_path": "homelab_vm/webcheck.yaml"},
|
|
],
|
|
"standalone": ["portainer_edge_agent", "openhands-runtime"]
|
|
},
|
|
"vish-nuc-edge": {
|
|
"stacks": [
|
|
{"name": "kuma", "containers": ["uptime-kuma"], "git_linked": False},
|
|
{"name": "glances", "containers": ["glances"], "git_linked": False},
|
|
],
|
|
"standalone": ["portainer_edge_agent"]
|
|
}
|
|
}
|
|
|
|
# Repo configs not running
|
|
def get_repo_configs():
|
|
"""List all compose files in the repo organized by server"""
|
|
repo_configs = {}
|
|
base_path = Path("/workspace/homelab")
|
|
|
|
server_folders = {
|
|
"Atlantis": base_path / "Atlantis",
|
|
"Calypso": base_path / "Calypso",
|
|
"concord_nuc": base_path / "concord_nuc",
|
|
"homelab_vm": base_path / "homelab_vm",
|
|
"Bulgaria_vm": base_path / "Bulgaria_vm",
|
|
"Chicago_vm": base_path / "Chicago_vm",
|
|
"anubis": base_path / "anubis",
|
|
"guava": base_path / "guava",
|
|
"setillo": base_path / "setillo",
|
|
}
|
|
|
|
for server, folder in server_folders.items():
|
|
if folder.exists():
|
|
configs = []
|
|
for ext in ["*.yml", "*.yaml"]:
|
|
configs.extend(folder.rglob(ext))
|
|
repo_configs[server] = [str(c.relative_to(base_path)) for c in configs]
|
|
|
|
return repo_configs
|
|
|
|
|
|
def generate_markdown_report():
|
|
"""Generate the comparison report in markdown"""
|
|
|
|
report = []
|
|
report.append("# Portainer Stack vs Repository Configuration Comparison")
|
|
report.append(f"\n*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}*")
|
|
report.append("\n---\n")
|
|
|
|
# Summary Section
|
|
report.append("## Executive Summary\n")
|
|
|
|
total_stacks = sum(len(data["stacks"]) for data in RUNNING_STACKS.values())
|
|
git_linked = sum(
|
|
sum(1 for s in data["stacks"] if s.get("git_linked", False))
|
|
for data in RUNNING_STACKS.values()
|
|
)
|
|
not_git_linked = total_stacks - git_linked
|
|
|
|
report.append(f"- **Total Running Stacks:** {total_stacks}")
|
|
report.append(f"- **Git-Linked Stacks:** {git_linked} ({git_linked/total_stacks*100:.1f}%)")
|
|
report.append(f"- **Not Git-Linked:** {not_git_linked}")
|
|
report.append(f"- **Servers Monitored:** {len(RUNNING_STACKS)}")
|
|
report.append("")
|
|
|
|
# Issues Summary
|
|
all_issues = []
|
|
for server, data in RUNNING_STACKS.items():
|
|
for stack in data["stacks"]:
|
|
if "issues" in stack:
|
|
for issue in stack["issues"]:
|
|
all_issues.append(f"{server}/{stack['name']}: {issue}")
|
|
if "issues" in data:
|
|
for issue in data["issues"]:
|
|
all_issues.append(f"{server}: {issue}")
|
|
|
|
if all_issues:
|
|
report.append("### ⚠️ Current Issues\n")
|
|
for issue in all_issues:
|
|
report.append(f"- {issue}")
|
|
report.append("")
|
|
|
|
# Per-Server Details
|
|
report.append("---\n")
|
|
report.append("## Server Details\n")
|
|
|
|
for server, data in RUNNING_STACKS.items():
|
|
report.append(f"### 🖥️ {server}\n")
|
|
|
|
# Running Stacks Table
|
|
report.append("#### Running Stacks\n")
|
|
report.append("| Stack Name | Containers | Git-Linked | Config Path | Status |")
|
|
report.append("|------------|------------|------------|-------------|--------|")
|
|
|
|
for stack in data["stacks"]:
|
|
name = stack["name"]
|
|
containers = len(stack["containers"])
|
|
git_linked = "✅" if stack.get("git_linked") else "❌"
|
|
config_path = stack.get("git_path", "-")
|
|
|
|
status = "🟢 Running"
|
|
if stack.get("status") == "stopped":
|
|
status = "🔴 Stopped"
|
|
elif "issues" in stack:
|
|
status = f"⚠️ {stack['issues'][0]}"
|
|
|
|
report.append(f"| {name} | {containers} | {git_linked} | `{config_path}` | {status} |")
|
|
|
|
report.append("")
|
|
|
|
# Standalone containers
|
|
if data.get("standalone"):
|
|
report.append("#### Standalone Containers (not in stacks)\n")
|
|
report.append(", ".join([f"`{c}`" for c in data["standalone"]]))
|
|
report.append("")
|
|
|
|
report.append("")
|
|
|
|
# Configs in Repo but Not Running
|
|
report.append("---\n")
|
|
report.append("## Repository Configs Not Currently Running\n")
|
|
report.append("These configurations exist in the repo but are not deployed:\n")
|
|
|
|
repo_configs = get_repo_configs()
|
|
|
|
# Known running config paths
|
|
running_paths = set()
|
|
for server, data in RUNNING_STACKS.items():
|
|
for stack in data["stacks"]:
|
|
if "git_path" in stack:
|
|
running_paths.add(stack["git_path"].rstrip("/"))
|
|
|
|
for server, configs in repo_configs.items():
|
|
not_running = []
|
|
for config in configs:
|
|
config_base = config.rsplit("/", 1)[0] if "/" in config else config
|
|
is_running = any(
|
|
config.startswith(p.rstrip("/")) or p.startswith(config.rsplit("/", 1)[0])
|
|
for p in running_paths
|
|
)
|
|
if not is_running:
|
|
not_running.append(config)
|
|
|
|
if not_running:
|
|
report.append(f"\n### {server}\n")
|
|
for config in not_running[:15]: # Limit to first 15
|
|
report.append(f"- `{config}`")
|
|
if len(not_running) > 15:
|
|
report.append(f"- ... and {len(not_running) - 15} more")
|
|
|
|
# Recommendations
|
|
report.append("\n---\n")
|
|
report.append("## Recommendations\n")
|
|
report.append("""
|
|
1. **Link Remaining Stacks to Git**: The following stacks should be linked to Git for version control:
|
|
- `paperless-testing` and `paperless-ai` on Calypso
|
|
- `rustdesk` on Calypso
|
|
- `node-exporter` stacks on multiple servers
|
|
- `openhands` and `perplexica` on Homelab VM
|
|
- `kuma` and `glances` on vish-nuc-edge
|
|
|
|
2. **Address Current Issues**:
|
|
- Fix `Synapse` container on Atlantis (currently exited)
|
|
- Investigate `invidious` unhealthy status on Concord NUC
|
|
- Fix `watchtower` and `node_exporter` restart loops on Concord NUC
|
|
|
|
3. **Cleanup Unused Configs**: Review configs in repo not currently deployed and either:
|
|
- Deploy if needed
|
|
- Archive if deprecated
|
|
- Document why they exist but aren't deployed
|
|
|
|
4. **Standardize Naming**: Some stacks use `-stack` suffix, others don't. Consider standardizing.
|
|
""")
|
|
|
|
return "\n".join(report)
|
|
|
|
|
|
def generate_infrastructure_overview():
|
|
"""Generate infrastructure overview document"""
|
|
|
|
report = []
|
|
report.append("# Homelab Infrastructure Overview")
|
|
report.append(f"\n*Last Updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}*")
|
|
report.append("\n---\n")
|
|
|
|
report.append("## Server Inventory\n")
|
|
report.append("| Server | Type | Endpoint ID | Status | Total Containers |")
|
|
report.append("|--------|------|-------------|--------|------------------|")
|
|
|
|
server_info = [
|
|
("Atlantis", "Local Docker", 2, "🟢 Online", "41"),
|
|
("Concord NUC", "Edge Agent", 443395, "🟢 Online", "15"),
|
|
("Calypso (vish-nuc)", "Edge Agent", 443397, "🟢 Online", "45"),
|
|
("vish-nuc-edge", "Edge Agent", 443398, "🟢 Online", "3"),
|
|
("Homelab VM", "Edge Agent", 443399, "🟢 Online", "20"),
|
|
]
|
|
|
|
for server, type_, eid, status, containers in server_info:
|
|
report.append(f"| {server} | {type_} | {eid} | {status} | {containers} |")
|
|
|
|
report.append("\n## Service Categories\n")
|
|
|
|
categories = {
|
|
"Media Management": ["arr-stack (Atlantis)", "arr-stack (Calypso)", "plex", "jellyseerr", "tautulli"],
|
|
"Photo Management": ["Immich (Atlantis)", "Immich (Calypso)"],
|
|
"Document Management": ["PaperlessNGX", "Joplin"],
|
|
"Network & DNS": ["AdGuard (Concord NUC)", "AdGuard (Calypso)", "WireGuard", "DynDNS"],
|
|
"Home Automation": ["Home Assistant", "Matter Server"],
|
|
"Development & DevOps": ["Gitea", "Portainer", "OpenHands"],
|
|
"Communication": ["Matrix/Synapse", "Jitsi", "Signal API"],
|
|
"Monitoring": ["Prometheus", "Grafana", "Uptime Kuma", "Glances", "WatchYourLAN"],
|
|
"Security": ["Vaultwarden/Bitwarden"],
|
|
"File Sync": ["Syncthing", "Seafile"],
|
|
"Privacy Tools": ["Invidious", "Libreddit/Redlib", "Binternet"],
|
|
"Productivity": ["Draw.io", "Reactive Resume", "ArchiveBox", "Hoarder/Karakeep"],
|
|
}
|
|
|
|
for category, services in categories.items():
|
|
report.append(f"### {category}\n")
|
|
for service in services:
|
|
report.append(f"- {service}")
|
|
report.append("")
|
|
|
|
return "\n".join(report)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Generate comparison report
|
|
comparison_report = generate_markdown_report()
|
|
with open("/workspace/homelab/docs/STACK_COMPARISON_REPORT.md", "w") as f:
|
|
f.write(comparison_report)
|
|
print("Generated: docs/STACK_COMPARISON_REPORT.md")
|
|
|
|
# Generate infrastructure overview
|
|
infra_report = generate_infrastructure_overview()
|
|
with open("/workspace/homelab/docs/INFRASTRUCTURE_OVERVIEW.md", "w") as f:
|
|
f.write(infra_report)
|
|
print("Generated: docs/INFRASTRUCTURE_OVERVIEW.md")
|