#!/bin/bash ############################################################################### # Layer 4: Core Monitoring & Notifications # # This script deploys ntfy and Uptime Kuma on watchtower. # Must be run after Layers 1A, 2, and 3 are complete. ############################################################################### set -e # Exit on error # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Project root directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" ANSIBLE_DIR="$PROJECT_ROOT/ansible" ############################################################################### # Helper Functions ############################################################################### print_header() { echo -e "\n${BLUE}========================================${NC}" echo -e "${BLUE}$1${NC}" echo -e "${BLUE}========================================${NC}\n" } print_success() { echo -e "${GREEN}✓${NC} $1" } print_error() { echo -e "${RED}✗${NC} $1" } print_warning() { echo -e "${YELLOW}⚠${NC} $1" } print_info() { echo -e "${BLUE}ℹ${NC} $1" } confirm_action() { local prompt="$1" local response read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response [[ "$response" =~ ^[Yy]$ ]] } ############################################################################### # Verification Functions ############################################################################### check_prerequisites() { print_header "Verifying Prerequisites" local errors=0 if [ -z "$VIRTUAL_ENV" ]; then print_error "Virtual environment not activated" echo "Run: source venv/bin/activate" ((errors++)) else print_success "Virtual environment activated" fi if ! command -v ansible &> /dev/null; then print_error "Ansible not found" ((errors++)) else print_success "Ansible found" fi if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then print_error "inventory.ini not found" ((errors++)) else print_success "inventory.ini exists" fi # Check if watchtower is configured if ! grep -q "^\[watchtower\]" "$ANSIBLE_DIR/inventory.ini"; then print_error "watchtower not configured in inventory.ini" print_info "Layer 4 requires watchtower VPS" ((errors++)) else print_success "watchtower configured in inventory" fi if [ $errors -gt 0 ]; then print_error "Prerequisites not met" exit 1 fi print_success "Prerequisites verified" } check_vars_files() { print_header "Checking Configuration Files" # Check services_config.yml if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then print_error "services_config.yml not found" print_info "This file should have been created in Layer 0" exit 1 fi print_success "services_config.yml exists" # Show configured subdomains local ntfy_sub=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") local uptime_sub=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") print_info "Configured subdomains:" echo " • ntfy: $ntfy_sub" echo " • uptime_kuma: $uptime_sub" echo "" } check_dns_configuration() { print_header "Validating DNS Configuration" cd "$ANSIBLE_DIR" # Get watchtower IP local watchtower_ip=$(ansible-inventory -i inventory.ini --list | python3 -c "import sys, json; data=json.load(sys.stdin); hosts=data.get('watchtower', {}).get('hosts', []); print(hosts[0] if hosts else '')" 2>/dev/null) if [ -z "$watchtower_ip" ]; then print_error "Could not determine watchtower IP from inventory" return 1 fi print_info "Watchtower IP: $watchtower_ip" echo "" # Get domain from infra_vars.yml local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) if [ -z "$root_domain" ]; then print_error "Could not determine root_domain from infra_vars.yml" return 1 fi # Get subdomains from centralized config local ntfy_subdomain="ntfy" local uptime_subdomain="uptime" if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then ntfy_subdomain=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") uptime_subdomain=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") fi local ntfy_fqdn="${ntfy_subdomain}.${root_domain}" local uptime_fqdn="${uptime_subdomain}.${root_domain}" print_info "Checking DNS records..." echo "" local dns_ok=true # Check ntfy DNS print_info "Checking $ntfy_fqdn..." if command -v dig &> /dev/null; then local ntfy_resolved=$(dig +short "$ntfy_fqdn" | head -n1) if [ "$ntfy_resolved" = "$watchtower_ip" ]; then print_success "$ntfy_fqdn → $ntfy_resolved ✓" elif [ -n "$ntfy_resolved" ]; then print_error "$ntfy_fqdn → $ntfy_resolved (expected $watchtower_ip)" dns_ok=false else print_error "$ntfy_fqdn does not resolve" dns_ok=false fi else print_warning "dig command not found, skipping DNS validation" print_info "Install dnsutils/bind-tools to enable DNS validation" return 1 fi # Check Uptime Kuma DNS print_info "Checking $uptime_fqdn..." if command -v dig &> /dev/null; then local uptime_resolved=$(dig +short "$uptime_fqdn" | head -n1) if [ "$uptime_resolved" = "$watchtower_ip" ]; then print_success "$uptime_fqdn → $uptime_resolved ✓" elif [ -n "$uptime_resolved" ]; then print_error "$uptime_fqdn → $uptime_resolved (expected $watchtower_ip)" dns_ok=false else print_error "$uptime_fqdn does not resolve" dns_ok=false fi fi echo "" if [ "$dns_ok" = false ]; then print_error "DNS validation failed" print_info "Please configure DNS records:" echo " • $ntfy_fqdn → $watchtower_ip" echo " • $uptime_fqdn → $watchtower_ip" echo "" print_warning "DNS changes can take time to propagate (up to 24-48 hours)" echo "" if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then exit 1 fi else print_success "DNS validation passed" fi } ############################################################################### # ntfy Deployment ############################################################################### deploy_ntfy() { print_header "Deploying ntfy (Notification Service)" cd "$ANSIBLE_DIR" print_info "ntfy requires admin credentials for authentication" echo "" # Check if env vars are set if [ -z "$NTFY_USER" ] || [ -z "$NTFY_PASSWORD" ]; then print_warning "NTFY_USER and NTFY_PASSWORD environment variables not set" echo "" print_info "Please enter credentials for ntfy admin user:" echo "" echo -e -n "${BLUE}ntfy admin username${NC} [admin]: " read ntfy_user ntfy_user="${ntfy_user:-admin}" echo -e -n "${BLUE}ntfy admin password${NC}: " read -s ntfy_password echo "" if [ -z "$ntfy_password" ]; then print_error "Password cannot be empty" return 1 fi export NTFY_USER="$ntfy_user" export NTFY_PASSWORD="$ntfy_password" else print_success "Using NTFY_USER and NTFY_PASSWORD from environment" fi echo "" print_info "This will:" echo " • Install ntfy from official repositories" echo " • Configure ntfy with authentication (deny-all by default)" echo " • Create admin user: $NTFY_USER" echo " • Set up Caddy reverse proxy" echo "" if ! confirm_action "Proceed with ntfy deployment?"; then print_warning "Skipped ntfy deployment" return 1 fi print_info "Running: ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml" echo "" if ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml; then print_success "ntfy deployment complete" echo "" print_info "ntfy is now available at your configured subdomain" print_info "Admin user: $NTFY_USER" return 0 else print_error "ntfy deployment failed" return 1 fi } ############################################################################### # Uptime Kuma Deployment ############################################################################### deploy_uptime_kuma() { print_header "Deploying Uptime Kuma (Monitoring Platform)" cd "$ANSIBLE_DIR" print_info "This will:" echo " • Deploy Uptime Kuma via Docker" echo " • Configure Caddy reverse proxy" echo " • Set up data persistence" echo "" if ! confirm_action "Proceed with Uptime Kuma deployment?"; then print_warning "Skipped Uptime Kuma deployment" return 1 fi print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml" echo "" if ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml; then print_success "Uptime Kuma deployment complete" echo "" print_warning "IMPORTANT: First-time setup required" echo " 1. Access Uptime Kuma at your configured subdomain" echo " 2. Create admin user on first visit" echo " 3. Update ansible/infra_secrets.yml with credentials" return 0 else print_error "Uptime Kuma deployment failed" return 1 fi } ############################################################################### # Backup Configuration ############################################################################### setup_uptime_kuma_backup() { print_header "Setting Up Uptime Kuma Backup (Optional)" cd "$ANSIBLE_DIR" print_info "This will set up automated backups to lapy" echo "" if ! confirm_action "Set up Uptime Kuma backup to lapy?"; then print_warning "Skipped backup setup" return 0 fi # Check if rsync is available print_info "Verifying rsync is installed on watchtower and lapy..." if ! ansible watchtower -i inventory.ini -m shell -a "command -v rsync" &>/dev/null; then print_error "rsync not found on watchtower" print_info "Run Layer 2 to install rsync" print_warning "Backup setup skipped - rsync not available" return 0 fi print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml" echo "" if ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml; then print_success "Uptime Kuma backup configured" print_info "Backups will run periodically via cron" return 0 else print_error "Backup setup failed" return 1 fi } ############################################################################### # Post-Deployment Configuration ############################################################################### setup_ntfy_notification() { print_header "Setting Up ntfy Notification in Uptime Kuma (Optional)" cd "$ANSIBLE_DIR" print_info "This will automatically configure ntfy as a notification method in Uptime Kuma" print_warning "Prerequisites:" echo " • Uptime Kuma admin account must be created first" echo " • infra_secrets.yml must have Uptime Kuma credentials" echo "" if ! confirm_action "Set up ntfy notification in Uptime Kuma?"; then print_warning "Skipped ntfy notification setup" print_info "You can set this up manually or run this script again later" return 0 fi # Check if infra_secrets.yml has Uptime Kuma credentials if ! grep -q "uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null || \ ! grep -q "uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then print_error "Uptime Kuma credentials not found in infra_secrets.yml" print_info "Please complete Step 1 and 2 of post-deployment steps first:" echo " 1. Create admin user in Uptime Kuma web UI" echo " 2. Add credentials to ansible/infra_secrets.yml" print_warning "Skipped - you can run this script again after completing those steps" return 0 fi # Check credentials are not empty local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") if [ -z "$uk_user" ] || [ -z "$uk_pass" ]; then print_error "Uptime Kuma credentials are empty in infra_secrets.yml" print_info "Please update ansible/infra_secrets.yml with your credentials" return 0 fi print_success "Found Uptime Kuma credentials in infra_secrets.yml" print_info "Running playbook to configure ntfy notification..." echo "" if ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml; then print_success "ntfy notification configured in Uptime Kuma" print_info "You can now use ntfy for all your monitors!" return 0 else print_error "Failed to configure ntfy notification" print_info "You can set this up manually or run the playbook again later:" echo " ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml" return 0 fi } ############################################################################### # Verification Functions ############################################################################### verify_deployments() { print_header "Verifying Deployments" cd "$ANSIBLE_DIR" local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') ssh_key="${ssh_key/#\~/$HOME}" local watchtower_host=$(ansible-inventory -i inventory.ini --list | python3 -c "import sys, json; data=json.load(sys.stdin); print(' '.join(data.get('watchtower', {}).get('hosts', [])))" 2>/dev/null) if [ -z "$watchtower_host" ]; then print_error "Could not determine watchtower host" return fi print_info "Checking services on watchtower ($watchtower_host)..." echo "" # Check ntfy if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "systemctl is-active ntfy" &>/dev/null; then print_success "ntfy service running" else print_warning "ntfy service not running or not installed" fi # Check Uptime Kuma docker container if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "docker ps | grep uptime-kuma" &>/dev/null; then print_success "Uptime Kuma container running" else print_warning "Uptime Kuma container not running" fi # Check Caddy configs if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/ntfy.conf" &>/dev/null; then print_success "ntfy Caddy config exists" else print_warning "ntfy Caddy config not found" fi if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/uptime-kuma.conf" &>/dev/null; then print_success "Uptime Kuma Caddy config exists" else print_warning "Uptime Kuma Caddy config not found" fi echo "" } verify_final_setup() { print_header "Final Verification - Post-Deployment Steps" cd "$ANSIBLE_DIR" print_info "Checking if all post-deployment steps were completed..." echo "" local all_ok=true # Check 1: infra_secrets.yml has Uptime Kuma credentials print_info "Checking infra_secrets.yml..." if grep -q "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null && \ grep -q "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") if [ -n "$uk_user" ] && [ -n "$uk_pass" ] && [ "$uk_user" != '""' ] && [ "$uk_pass" != '""' ]; then print_success "Uptime Kuma credentials configured in infra_secrets.yml" else print_error "Uptime Kuma credentials are empty in infra_secrets.yml" print_info "Please complete Step 2: Update infra_secrets.yml" all_ok=false fi else print_error "Uptime Kuma credentials not found in infra_secrets.yml" print_info "Please complete Step 2: Update infra_secrets.yml" all_ok=false fi echo "" # Check 2: Can connect to Uptime Kuma API print_info "Checking Uptime Kuma API access..." if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then # Create a test Python script to check API access local test_script=$(mktemp) cat > "$test_script" << 'EOFPYTHON' import sys import yaml from uptime_kuma_api import UptimeKumaApi try: # Load config with open('infra_vars.yml', 'r') as f: infra_vars = yaml.safe_load(f) with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: uk_vars = yaml.safe_load(f) with open('infra_secrets.yml', 'r') as f: secrets = yaml.safe_load(f) root_domain = infra_vars.get('root_domain') subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') url = f"https://{subdomain}.{root_domain}" username = secrets.get('uptime_kuma_username') password = secrets.get('uptime_kuma_password') # Try to connect api = UptimeKumaApi(url) api.login(username, password) # Check if we can get monitors monitors = api.get_monitors() print(f"SUCCESS:{len(monitors)}") api.disconnect() sys.exit(0) except Exception as e: print(f"ERROR:{str(e)}", file=sys.stderr) sys.exit(1) EOFPYTHON local result=$(cd "$ANSIBLE_DIR" && python3 "$test_script" 2>&1) rm -f "$test_script" if echo "$result" | grep -q "^SUCCESS:"; then local monitor_count=$(echo "$result" | grep "^SUCCESS:" | cut -d: -f2) print_success "Successfully connected to Uptime Kuma API" print_info "Current monitors: $monitor_count" else print_error "Cannot connect to Uptime Kuma API" print_warning "This usually means:" echo " • Admin account not created yet (Step 1)" echo " • Wrong credentials in infra_secrets.yml (Step 2)" echo " • Uptime Kuma not accessible" all_ok=false fi else print_warning "Skipping API check - credentials not configured" all_ok=false fi echo "" # Check 3: ntfy notification configured in Uptime Kuma print_info "Checking ntfy notification configuration..." if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then local test_notif=$(mktemp) cat > "$test_notif" << 'EOFPYTHON' import sys import yaml from uptime_kuma_api import UptimeKumaApi try: # Load config with open('infra_vars.yml', 'r') as f: infra_vars = yaml.safe_load(f) with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: uk_vars = yaml.safe_load(f) with open('infra_secrets.yml', 'r') as f: secrets = yaml.safe_load(f) root_domain = infra_vars.get('root_domain') subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') url = f"https://{subdomain}.{root_domain}" username = secrets.get('uptime_kuma_username') password = secrets.get('uptime_kuma_password') # Connect api = UptimeKumaApi(url) api.login(username, password) # Check for ntfy notification notifications = api.get_notifications() ntfy_found = any(n.get('type') == 'ntfy' for n in notifications) if ntfy_found: print("SUCCESS:ntfy notification configured") else: print("NOTFOUND:No ntfy notification found") api.disconnect() sys.exit(0) except Exception as e: print(f"ERROR:{str(e)}", file=sys.stderr) sys.exit(1) EOFPYTHON local notif_result=$(cd "$ANSIBLE_DIR" && python3 "$test_notif" 2>&1) rm -f "$test_notif" if echo "$notif_result" | grep -q "^SUCCESS:"; then print_success "ntfy notification is configured in Uptime Kuma" elif echo "$notif_result" | grep -q "^NOTFOUND:"; then print_warning "ntfy notification not yet configured" print_info "Run the script again and choose 'yes' for ntfy notification setup" print_info "Or complete Step 3 manually" all_ok=false else print_warning "Could not verify ntfy notification (API access issue)" fi else print_warning "Skipping ntfy check - credentials not configured" fi echo "" # Summary if [ "$all_ok" = true ]; then print_success "All post-deployment steps completed! ✓" echo "" print_info "Layer 4 is fully configured and ready to use" print_info "You can now proceed to Layer 6 (infrastructure monitoring)" return 0 else print_warning "Some post-deployment steps are incomplete" echo "" print_info "Complete these steps:" echo " 1. Access Uptime Kuma web UI and create admin account" echo " 2. Update ansible/infra_secrets.yml with credentials" echo " 3. Run this script again to configure ntfy notification" echo "" print_info "You can also complete manually and verify with:" echo " ./scripts/setup_layer_4_monitoring.sh" return 1 fi } ############################################################################### # Summary Functions ############################################################################### print_summary() { print_header "Layer 4 Setup Complete! 🎉" echo "Summary of what was configured:" echo "" print_success "ntfy notification service deployed" print_success "Uptime Kuma monitoring platform deployed" print_success "Caddy reverse proxy configured for both services" echo "" print_warning "REQUIRED POST-DEPLOYMENT STEPS:" echo "" echo "MANUAL (do these first):" echo " 1. Access Uptime Kuma Web UI and create admin account" echo " 2. Update ansible/infra_secrets.yml with credentials" echo "" echo "AUTOMATED (script can do these):" echo " 3. Configure ntfy notification - script will offer to set this up" echo " 4. Final verification - script will check everything" echo "" print_info "After completing steps 1 & 2, the script will:" echo " • Automatically configure ntfy in Uptime Kuma" echo " • Verify all post-deployment steps" echo " • Tell you if anything is missing" echo "" print_warning "You MUST complete steps 1 & 2 before proceeding to Layer 6!" echo "" print_info "What these services enable:" echo " • ntfy: Push notifications to your devices" echo " • Uptime Kuma: Monitor all services and infrastructure" echo " • Together: Complete monitoring and alerting solution" echo "" print_info "Next steps:" echo " 1. Complete the post-deployment steps above" echo " 2. Test ntfy: Send a test notification" echo " 3. Test Uptime Kuma: Create a test monitor" echo " 4. Proceed to Layer 5: ./scripts/setup_layer_5_headscale.sh (optional)" echo " OR Layer 6: ./scripts/setup_layer_6_infra_monitoring.sh" echo "" } ############################################################################### # Main Execution ############################################################################### main() { clear print_header "📊 Layer 4: Core Monitoring & Notifications" echo "This script will deploy ntfy and Uptime Kuma on watchtower." echo "" print_info "Services to deploy:" echo " • ntfy (notification service)" echo " • Uptime Kuma (monitoring platform)" echo "" if ! confirm_action "Continue with Layer 4 setup?"; then echo "Setup cancelled." exit 0 fi check_prerequisites check_vars_files check_dns_configuration # Deploy services (don't fail if skipped) deploy_ntfy || true echo "" deploy_uptime_kuma || true echo "" setup_uptime_kuma_backup || true echo "" verify_deployments # Always show summary and offer ntfy configuration print_summary echo "" # Always ask about ntfy notification setup (regardless of deployment status) print_header "Configure ntfy Notification in Uptime Kuma" print_info "After creating your Uptime Kuma admin account and updating infra_secrets.yml," print_info "the script can automatically configure ntfy as a notification method." echo "" print_warning "Prerequisites:" echo " 1. Access Uptime Kuma web UI and create admin account" echo " 2. Update ansible/infra_secrets.yml with your credentials" echo "" # Always offer to set up ntfy notification setup_ntfy_notification # Final verification echo "" verify_final_setup } # Run main function main "$@"