9.6 KiB
9.6 KiB
Stoatchat Troubleshooting Guide
Common issues and solutions for the Stoatchat deployment on Seattle VM.
Quick Diagnostics
Check All Services Status
# SSH to Seattle VM
ssh root@YOUR_WAN_IP
# Check Stoatchat processes
ps aux | grep revolt
# Check ports
ss -tlnp | grep -E "(14702|14703|14704|14705|14706|7880)"
# Check Docker services
cd /root/stoatchat
docker-compose ps
# Check nginx
systemctl status nginx
Test All Endpoints
# Test locally on server
curl -k https://api.st.vish.gg/ --resolve api.st.vish.gg:443:127.0.0.1
curl -k https://files.st.vish.gg/ --resolve files.st.vish.gg:443:127.0.0.1
curl -k https://proxy.st.vish.gg/ --resolve proxy.st.vish.gg:443:127.0.0.1
curl -k https://voice.st.vish.gg/ --resolve voice.st.vish.gg:443:127.0.0.1
# Test externally
curl https://api.st.vish.gg/
curl https://files.st.vish.gg/
curl https://proxy.st.vish.gg/
curl https://voice.st.vish.gg/
Common Issues
1. Services Not Starting
Symptoms
ps aux | grep revoltshows no processes- Ports not listening
- Connection refused errors
Diagnosis
cd /root/stoatchat
# Check if binaries exist
ls -la target/debug/revolt-*
# Try starting manually to see errors
./target/debug/revolt-delta
# Check logs
tail -f api.log events.log files.log proxy.log gifbox.log
Solutions
# Rebuild if binaries missing
cargo build
# Check configuration
cat Revolt.overrides.toml | grep -E "(mongodb|redis|s3_)"
# Restart infrastructure services
docker-compose down && docker-compose up -d
# Wait for services to be ready
sleep 30
# Start Stoatchat services
nohup ./target/debug/revolt-delta > api.log 2>&1 &
nohup ./target/debug/revolt-bonfire > events.log 2>&1 &
nohup ./target/debug/revolt-autumn > files.log 2>&1 &
nohup ./target/debug/revolt-january > proxy.log 2>&1 &
nohup ./target/debug/revolt-gifbox > gifbox.log 2>&1 &
2. Database Connection Issues
Symptoms
- Services start but crash immediately
- "Connection refused" in logs
- MongoDB/Redis errors
Diagnosis
# Check Docker services
docker-compose ps
# Test MongoDB connection
docker exec stoatchat-mongodb mongo --eval "db.adminCommand('ismaster')"
# Test Redis connection
docker exec stoatchat-redis redis-cli ping
# Check configuration
grep -E "(mongodb|redis)" /root/stoatchat/Revolt.overrides.toml
Solutions
# Restart infrastructure
docker-compose restart
# Check MongoDB logs
docker-compose logs database
# Check Redis logs
docker-compose logs redis
# Verify ports are accessible
telnet 127.0.0.1 27017
telnet 127.0.0.1 6380
3. SSL Certificate Issues
Symptoms
- SSL errors in browser
- Certificate expired warnings
- nginx fails to start
Diagnosis
# Check certificate validity
openssl x509 -in /etc/letsencrypt/live/api.st.vish.gg/fullchain.pem -text -noout | grep -A2 "Validity"
# Check nginx configuration
nginx -t
# Check certificate files exist
ls -la /etc/letsencrypt/live/*/
Solutions
# Renew certificates
certbot renew
# Or renew specific certificate
certbot renew --cert-name api.st.vish.gg
# Test renewal
certbot renew --dry-run
# Reload nginx after renewal
systemctl reload nginx
4. File Upload Issues
Symptoms
- File uploads fail
- 413 Request Entity Too Large
- MinIO connection errors
Diagnosis
# Check MinIO status
docker-compose logs minio
# Test MinIO connection
curl http://127.0.0.1:14009/minio/health/live
# Check nginx file size limits
grep client_max_body_size /etc/nginx/sites-available/stoatchat
# Check MinIO credentials
grep -A5 "\[files\]" /root/stoatchat/Revolt.overrides.toml
Solutions
# Restart MinIO
docker-compose restart minio
# Check MinIO bucket exists
docker exec stoatchat-minio mc ls local/
# Create bucket if missing
docker exec stoatchat-minio mc mb local/revolt-uploads
# Increase nginx file size limit if needed
sed -i 's/client_max_body_size 100M;/client_max_body_size 500M;/' /etc/nginx/sites-available/stoatchat
systemctl reload nginx
5. WebSocket Connection Issues
Symptoms
- Events service returns 502
- WebSocket connections fail
- Real-time features not working
Diagnosis
# Check events service
curl -k https://events.st.vish.gg/ --resolve events.st.vish.gg:443:127.0.0.1
# Check if service is listening
ss -tlnp | grep 14703
# Check nginx WebSocket configuration
grep -A10 "events.st.vish.gg" /etc/nginx/sites-available/stoatchat
Solutions
# Restart events service
pkill -f revolt-bonfire
nohup ./target/debug/revolt-bonfire > events.log 2>&1 &
# Check WebSocket headers in nginx
# Ensure these are present:
# proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection "upgrade";
# Test WebSocket connection (if wscat available)
wscat -c wss://events.st.vish.gg/
6. LiveKit Voice Issues
Symptoms
- Voice/video not working
- LiveKit returns errors
- Connection timeouts
Diagnosis
# Check LiveKit status
docker-compose logs livekit
# Test LiveKit endpoint
curl -k https://voice.st.vish.gg/ --resolve voice.st.vish.gg:443:127.0.0.1
# Check LiveKit configuration
cat /root/stoatchat/livekit.yml
# Check if using correct image
docker images | grep livekit
Solutions
# Restart LiveKit
docker-compose restart livekit
# Check Redis connection for LiveKit
docker exec stoatchat-redis redis-cli ping
# Verify LiveKit configuration
# Ensure Redis address matches: localhost:6380
# Check firewall for UDP ports
ufw status | grep 7882
7. Email/SMTP Issues
Symptoms
- Email verification not working
- SMTP connection errors
- Authentication failures
Diagnosis
# Check SMTP configuration
grep -A10 "\[email\]" /root/stoatchat/Revolt.overrides.toml
# Test SMTP connection
telnet smtp.gmail.com 587
# Check logs for SMTP errors
grep -i smtp /root/stoatchat/*.log
Solutions
# Verify Gmail App Password is correct
# Check if 2FA is enabled on Gmail account
# Ensure "Less secure app access" is not needed (use App Password instead)
# Test SMTP manually
openssl s_client -starttls smtp -connect smtp.gmail.com:587
Performance Issues
High CPU Usage
# Check which service is using CPU
top -p $(pgrep -d',' revolt)
# Check for memory leaks
ps aux --sort=-%mem | grep revolt
# Monitor resource usage
htop
High Memory Usage
# Check memory usage per service
ps aux --sort=-%mem | grep revolt
# Check Docker container usage
docker stats
# Check system memory
free -h
Slow Response Times
# Check nginx access logs
tail -f /var/log/nginx/access.log
# Check service logs for slow queries
grep -i "slow\|timeout" /root/stoatchat/*.log
# Test response times
time curl https://api.st.vish.gg/
Log Analysis
Service Logs Location
cd /root/stoatchat
# Main service logs
tail -f api.log # API server
tail -f events.log # WebSocket events
tail -f files.log # File server
tail -f proxy.log # Media proxy
tail -f gifbox.log # GIF service
# System logs
journalctl -u nginx -f
docker-compose logs -f
Common Log Patterns
# Database connection errors
grep -i "connection.*refused\|timeout" *.log
# Authentication errors
grep -i "auth\|login\|token" *.log
# File upload errors
grep -i "upload\|s3\|minio" *.log
# WebSocket errors
grep -i "websocket\|upgrade" *.log
Recovery Procedures
Complete Service Restart
cd /root/stoatchat
# Stop all Stoatchat services
pkill -f revolt-
# Restart infrastructure
docker-compose down
docker-compose up -d
# Wait for services to be ready
sleep 30
# Start Stoatchat services
nohup ./target/debug/revolt-delta > api.log 2>&1 &
nohup ./target/debug/revolt-bonfire > events.log 2>&1 &
nohup ./target/debug/revolt-autumn > files.log 2>&1 &
nohup ./target/debug/revolt-january > proxy.log 2>&1 &
nohup ./target/debug/revolt-gifbox > gifbox.log 2>&1 &
# Restart nginx
systemctl restart nginx
Emergency Rebuild
cd /root/stoatchat
# Stop services
pkill -f revolt-
# Clean build
cargo clean
cargo build
# Restart everything
docker-compose down && docker-compose up -d
sleep 30
# Start services with new binaries
./start-services.sh # If you created this script
Database Recovery
# If MongoDB is corrupted
docker-compose stop database
docker volume rm stoatchat_mongodb_data # WARNING: This deletes data
docker-compose up -d database
# Restore from backup if available
# mongorestore --uri="mongodb://127.0.0.1:27017/revolt" /path/to/backup
Monitoring Commands
Health Check Script
#!/bin/bash
# Save as /root/stoatchat/health-check.sh
echo "=== Stoatchat Health Check ==="
echo "Date: $(date)"
echo
echo "=== Process Status ==="
ps aux | grep revolt | grep -v grep
echo -e "\n=== Port Status ==="
ss -tlnp | grep -E "(14702|14703|14704|14705|14706|7880)"
echo -e "\n=== Docker Services ==="
cd /root/stoatchat && docker-compose ps
echo -e "\n=== Nginx Status ==="
systemctl is-active nginx
echo -e "\n=== Endpoint Tests ==="
for endpoint in api files proxy voice; do
echo -n "$endpoint.st.vish.gg: "
curl -s -o /dev/null -w "%{http_code}" https://$endpoint.st.vish.gg/ || echo "FAIL"
done
echo -e "\n=== Disk Usage ==="
df -h /root/stoatchat
echo -e "\n=== Memory Usage ==="
free -h
Automated Monitoring
# Add to crontab for regular health checks
# crontab -e
# */5 * * * * /root/stoatchat/health-check.sh >> /var/log/stoatchat-health.log 2>&1
Contact Information
For additional support:
- Repository: https://github.com/stoatchat/stoatchat
- Documentation: Check /root/stoatchat/docs/
- Logs: /root/stoatchat/*.log
- Configuration: /root/stoatchat/Revolt.overrides.toml