From c14d61d090afd686d484b9a3f2ad3beb6da99597 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 7 Dec 2025 19:02:50 +0100 Subject: [PATCH 01/13] stuff --- SCRIPT_PLAYBOOK_MAPPING.md | 59 -- ansible/backup.infra_vars.yml | 4 - ansible/infra/920_join_headscale_mesh.yml | 1 + ansible/infra_secrets.yml.example | 10 + ansible/infra_vars.yml | 3 - ...le.inventory.ini => inventory.ini.example} | 0 .../deploy_headscale_ui_playbook.yml | 142 +++ .../deploy_ntfy_emergency_app_playbook.yml | 111 +++ ansible/services_config.yml | 2 +- ansible/services_config.yml.example | 32 - backup.inventory.ini | 16 - scripts/setup_layer_0.sh | 488 ----------- scripts/setup_layer_1a_vps.sh | 393 --------- scripts/setup_layer_1b_nodito.sh | 411 --------- scripts/setup_layer_2.sh | 407 --------- scripts/setup_layer_3_caddy.sh | 355 -------- scripts/setup_layer_4_monitoring.sh | 806 ------------------ scripts/setup_layer_5_headscale.sh | 524 ------------ scripts/setup_layer_6_infra_monitoring.sh | 473 ---------- scripts/setup_layer_7_services.sh | 524 ------------ scripts/setup_layer_8_secondary_services.sh | 384 --------- 21 files changed, 265 insertions(+), 4880 deletions(-) delete mode 100644 SCRIPT_PLAYBOOK_MAPPING.md delete mode 100644 ansible/backup.infra_vars.yml rename ansible/{example.inventory.ini => inventory.ini.example} (100%) create mode 100644 ansible/services/headscale/deploy_headscale_ui_playbook.yml delete mode 100644 ansible/services_config.yml.example delete mode 100644 backup.inventory.ini delete mode 100755 scripts/setup_layer_0.sh delete mode 100755 scripts/setup_layer_1a_vps.sh delete mode 100755 scripts/setup_layer_1b_nodito.sh delete mode 100755 scripts/setup_layer_2.sh delete mode 100755 scripts/setup_layer_3_caddy.sh delete mode 100755 scripts/setup_layer_4_monitoring.sh delete mode 100755 scripts/setup_layer_5_headscale.sh delete mode 100755 scripts/setup_layer_6_infra_monitoring.sh delete mode 100755 scripts/setup_layer_7_services.sh delete mode 100755 scripts/setup_layer_8_secondary_services.sh diff --git a/SCRIPT_PLAYBOOK_MAPPING.md b/SCRIPT_PLAYBOOK_MAPPING.md deleted file mode 100644 index 38189ab..0000000 --- a/SCRIPT_PLAYBOOK_MAPPING.md +++ /dev/null @@ -1,59 +0,0 @@ -# Script to Playbook Mapping - -This document describes which playbooks each setup script applies to which machines. - -## Table - -| Script | Playbook | Target Machines/Groups | Notes | -|--------|----------|------------------------|-------| -| **setup_layer_0.sh** | None | N/A | Initial setup script - creates venv, config files | -| **setup_layer_1a_vps.sh** | `infra/01_user_and_access_setup_playbook.yml` | `vps` (vipy, watchtower, spacey) | Creates counterweight user, configures SSH | -| **setup_layer_1a_vps.sh** | `infra/02_firewall_and_fail2ban_playbook.yml` | `vps` (vipy, watchtower, spacey) | Configures UFW firewall and fail2ban | -| **setup_layer_1b_nodito.sh** | `infra/nodito/30_proxmox_bootstrap_playbook.yml` | `nodito_host` (nodito) | Initial Proxmox bootstrap | -| **setup_layer_1b_nodito.sh** | `infra/nodito/31_proxmox_community_repos_playbook.yml` | `nodito_host` (nodito) | Configures Proxmox community repositories | -| **setup_layer_1b_nodito.sh** | `infra/nodito/32_zfs_pool_setup_playbook.yml` | `nodito_host` (nodito) | Sets up ZFS pool on Proxmox | -| **setup_layer_1b_nodito.sh** | `infra/nodito/33_proxmox_debian_cloud_template.yml` | `nodito_host` (nodito) | Creates Debian cloud template for VMs | -| **setup_layer_2.sh** | `infra/900_install_rsync.yml` | `all` (vipy, watchtower, spacey, nodito) | Installs rsync on all machines | -| **setup_layer_2.sh** | `infra/910_docker_playbook.yml` | `all` (vipy, watchtower, spacey, nodito) | Installs Docker on all machines | -| **setup_layer_3_caddy.sh** | `services/caddy_playbook.yml` | `vps` (vipy, watchtower, spacey) | Installs and configures Caddy reverse proxy | -| **setup_layer_4_monitoring.sh** | `services/ntfy/deploy_ntfy_playbook.yml` | `watchtower` | Deploys ntfy notification service | -| **setup_layer_4_monitoring.sh** | `services/uptime_kuma/deploy_uptime_kuma_playbook.yml` | `watchtower` | Deploys Uptime Kuma monitoring | -| **setup_layer_4_monitoring.sh** | `services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml` | `lapy` (localhost) | Configures backup of Uptime Kuma to laptop | -| **setup_layer_4_monitoring.sh** | `services/ntfy/setup_ntfy_uptime_kuma_notification.yml` | `watchtower` | Configures ntfy notifications for Uptime Kuma | -| **setup_layer_5_headscale.sh** | `services/headscale/deploy_headscale_playbook.yml` | `spacey` | Deploys Headscale mesh VPN server | -| **setup_layer_5_headscale.sh** | `infra/920_join_headscale_mesh.yml` | `all` (vipy, watchtower, spacey, nodito) | Joins all machines to Headscale mesh (with --limit) | -| **setup_layer_5_headscale.sh** | `services/headscale/setup_backup_headscale_to_lapy.yml` | `lapy` (localhost) | Configures backup of Headscale to laptop | -| **setup_layer_6_infra_monitoring.sh** | `infra/410_disk_usage_alerts.yml` | `all` (vipy, watchtower, spacey, nodito, lapy) | Sets up disk usage monitoring alerts | -| **setup_layer_6_infra_monitoring.sh** | `infra/420_system_healthcheck.yml` | `all` (vipy, watchtower, spacey, nodito, lapy) | Sets up system health checks | -| **setup_layer_6_infra_monitoring.sh** | `infra/430_cpu_temp_alerts.yml` | `nodito_host` (nodito) | Sets up CPU temperature alerts for Proxmox | -| **setup_layer_7_services.sh** | `services/vaultwarden/deploy_vaultwarden_playbook.yml` | `vipy` | Deploys Vaultwarden password manager | -| **setup_layer_7_services.sh** | `services/forgejo/deploy_forgejo_playbook.yml` | `vipy` | Deploys Forgejo Git server | -| **setup_layer_7_services.sh** | `services/lnbits/deploy_lnbits_playbook.yml` | `vipy` | Deploys LNbits Lightning wallet | -| **setup_layer_7_services.sh** | `services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml` | `lapy` (localhost) | Configures backup of Vaultwarden to laptop | -| **setup_layer_7_services.sh** | `services/lnbits/setup_backup_lnbits_to_lapy.yml` | `lapy` (localhost) | Configures backup of LNbits to laptop | -| **setup_layer_8_secondary_services.sh** | `services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml` | `vipy` | Deploys emergency ntfy app | -| **setup_layer_8_secondary_services.sh** | `services/memos/deploy_memos_playbook.yml` | `memos-box` (VM on nodito) | Deploys Memos note-taking service | - -## Machine Groups Reference - -- **vps**: vipy, watchtower, spacey (VPS servers) -- **nodito_host**: nodito (Proxmox server) -- **nodito_vms**: memos-box and other VMs created on nodito -- **lapy**: localhost (your laptop) -- **all**: All machines in inventory -- **watchtower**: Single VPS for monitoring services -- **vipy**: Single VPS for main services -- **spacey**: Single VPS for Headscale -- **memos-box**: VM on nodito for Memos service - -## Notes - -- Scripts use `--limit` flag to restrict playbooks that target `all` to specific hosts -- Backup playbooks run on `lapy` (localhost) to configure backup jobs -- Some playbooks are optional and may be skipped if hosts aren't configured -- Layer 0 is a prerequisite for all other layers - - - - - diff --git a/ansible/backup.infra_vars.yml b/ansible/backup.infra_vars.yml deleted file mode 100644 index 952df93..0000000 --- a/ansible/backup.infra_vars.yml +++ /dev/null @@ -1,4 +0,0 @@ -new_user: counterweight -ssh_port: 22 -allow_ssh_from: "any" -root_domain: contrapeso.xyz diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index a0c3b5a..425036c 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -99,6 +99,7 @@ --login-server {{ headscale_domain }} --authkey {{ auth_key }} --accept-dns=true + --advertise-tags "tag:{{ inventory_hostname }}" register: tailscale_up_result changed_when: "'already authenticated' not in tailscale_up_result.stdout" failed_when: tailscale_up_result.rc != 0 and 'already authenticated' not in tailscale_up_result.stdout diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index 07ee552..80b740b 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -9,3 +9,13 @@ uptime_kuma_password: "your_password_here" ntfy_username: "your_ntfy_username" ntfy_password: "your_ntfy_password" + +# headscale-ui credentials +# Used for HTTP basic authentication via Caddy +# Provide either: +# - headscale_ui_password: plain text password (will be hashed automatically) +# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) + +headscale_ui_username: "admin" +headscale_ui_password: "your_secure_password_here" +# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml index a719e68..952df93 100644 --- a/ansible/infra_vars.yml +++ b/ansible/infra_vars.yml @@ -1,6 +1,3 @@ -# Infrastructure Variables -# Generated by setup_layer_0.sh - new_user: counterweight ssh_port: 22 allow_ssh_from: "any" diff --git a/ansible/example.inventory.ini b/ansible/inventory.ini.example similarity index 100% rename from ansible/example.inventory.ini rename to ansible/inventory.ini.example diff --git a/ansible/services/headscale/deploy_headscale_ui_playbook.yml b/ansible/services/headscale/deploy_headscale_ui_playbook.yml new file mode 100644 index 0000000..3be792c --- /dev/null +++ b/ansible/services/headscale/deploy_headscale_ui_playbook.yml @@ -0,0 +1,142 @@ +- name: Deploy headscale-ui with Docker and configure Caddy reverse proxy + hosts: spacey + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./headscale_vars.yml + vars: + headscale_subdomain: "{{ subdomains.headscale }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" + headscale_ui_version: "2025.08.23" + headscale_ui_dir: /opt/headscale-ui + headscale_ui_http_port: 18080 + headscale_ui_https_port: 18443 + + tasks: + - name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: false + + - name: Fail if Docker is not installed + fail: + msg: "Docker is not installed. Please run the docker_playbook.yml first." + when: docker_check.rc != 0 + + - name: Ensure Docker service is running + systemd: + name: docker + state: started + enabled: yes + + - name: Create headscale-ui directory + file: + path: "{{ headscale_ui_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create docker-compose.yml for headscale-ui + copy: + dest: "{{ headscale_ui_dir }}/docker-compose.yml" + content: | + version: "3" + services: + headscale-ui: + image: ghcr.io/gurucomputing/headscale-ui:{{ headscale_ui_version }} + container_name: headscale-ui + restart: unless-stopped + ports: + - "{{ headscale_ui_http_port }}:8080" + - "{{ headscale_ui_https_port }}:8443" + owner: root + group: root + mode: '0644' + + - name: Deploy headscale-ui container with docker compose + command: docker compose up -d + args: + chdir: "{{ headscale_ui_dir }}" + register: docker_compose_result + changed_when: "'Creating' in docker_compose_result.stdout or 'Starting' in docker_compose_result.stdout or docker_compose_result.rc != 0" + + - name: Wait for headscale-ui to be ready + uri: + url: "http://localhost:{{ headscale_ui_http_port }}" + status_code: [200, 404] + register: headscale_ui_ready + until: headscale_ui_ready.status in [200, 404] + retries: 30 + delay: 2 + ignore_errors: yes + + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + + - name: Fail if username is not provided + fail: + msg: "headscale_ui_username must be set in infra_secrets.yml" + when: headscale_ui_username is not defined + + - name: Fail if neither password nor password hash is provided + fail: + msg: "Either headscale_ui_password or headscale_ui_password_hash must be set in infra_secrets.yml" + when: headscale_ui_password is not defined and headscale_ui_password_hash is not defined + + - name: Generate bcrypt hash for headscale-ui password + become: yes + command: caddy hash-password --plaintext "{{ headscale_ui_password }}" + register: headscale_ui_password_hash_result + changed_when: false + no_log: true + when: headscale_ui_password is defined and headscale_ui_password_hash is not defined + + - name: Set headscale-ui password hash from generated value + set_fact: + headscale_ui_password_hash: "{{ headscale_ui_password_hash_result.stdout.strip() }}" + when: headscale_ui_password is defined and headscale_ui_password_hash is not defined + + - name: Update headscale Caddy config to include headscale-ui /web route with authentication + become: yes + copy: + dest: "{{ caddy_sites_dir }}/headscale.conf" + content: | + {{ headscale_domain }} { + @headscale_ui { + path /web* + } + handle @headscale_ui { + basicauth { + {{ headscale_ui_username }} {{ headscale_ui_password_hash }} + } + reverse_proxy http://localhost:{{ headscale_ui_http_port }} + } + # Headscale API is protected by its own API key authentication + # All API operations require a valid Bearer token in the Authorization header + reverse_proxy * http://localhost:{{ headscale_port }} + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + command: systemctl reload caddy + diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index 18a3b72..b8c0064 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -14,6 +14,7 @@ ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -77,3 +78,113 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy + + - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_ntfy_emergency_app_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ ntfy_emergency_app_domain }}" + monitor_name: "ntfy-emergency-app" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_ntfy_emergency_app_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_ntfy_emergency_app_monitor.py + - /tmp/ansible_config.yml diff --git a/ansible/services_config.yml b/ansible/services_config.yml index 5c0dcbd..b497c51 100644 --- a/ansible/services_config.yml +++ b/ansible/services_config.yml @@ -16,7 +16,7 @@ subdomains: lnbits: wallet # Secondary Services (on vipy) - ntfy_emergency_app: emergency + ntfy_emergency_app: avisame personal_blog: pablohere # Memos (on memos-box) diff --git a/ansible/services_config.yml.example b/ansible/services_config.yml.example deleted file mode 100644 index 972b685..0000000 --- a/ansible/services_config.yml.example +++ /dev/null @@ -1,32 +0,0 @@ -# Centralized Services Configuration -# Copy this to services_config.yml and customize - -# Edit these subdomains to match your preferences -subdomains: - # Monitoring Services (on watchtower) - ntfy: ntfy - uptime_kuma: uptime - - # VPN Infrastructure (on spacey) - headscale: headscale - - # Core Services (on vipy) - vaultwarden: vault - forgejo: git - lnbits: lnbits - - # Secondary Services (on vipy) - ntfy_emergency_app: emergency - - # Memos (on memos-box) - memos: memos - -# Caddy configuration -caddy_sites_dir: /etc/caddy/sites-enabled - -# Service-specific settings shared across playbooks -service_settings: - ntfy: - topic: alerts - headscale: - namespace: counter-net diff --git a/backup.inventory.ini b/backup.inventory.ini deleted file mode 100644 index dec2de3..0000000 --- a/backup.inventory.ini +++ /dev/null @@ -1,16 +0,0 @@ -[vps] -vipy ansible_host=207.154.226.192 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -watchtower ansible_host=206.189.63.167 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua -spacey ansible_host=165.232.73.4 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - -[nodito_host] -nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_pass=noesfacilvivirenunmundocentralizado ansible_ssh_private_key_file=~/.ssh/counterganzua - -[nodito_vms] -memos-box ansible_host=192.168.1.149 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua - - -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -# Make sure to adjust the username -[lapy] -localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C \ No newline at end of file diff --git a/scripts/setup_layer_0.sh b/scripts/setup_layer_0.sh deleted file mode 100755 index f994f98..0000000 --- a/scripts/setup_layer_0.sh +++ /dev/null @@ -1,488 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 0: Foundation Setup -# -# This script sets up your laptop (lapy) as the Ansible control node. -# It prepares all the prerequisites needed for the infrastructure deployment. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -prompt_user() { - local prompt="$1" - local default="$2" - local result - - if [ -n "$default" ]; then - read -p "$(echo -e ${BLUE}${prompt}${NC} [${default}]: )" result - result="${result:-$default}" - else - read -p "$(echo -e ${BLUE}${prompt}${NC}: )" result - fi - - echo "$result" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Main Setup Functions -############################################################################### - -check_prerequisites() { - print_header "Checking Prerequisites" - - # Check if we're in the right directory - if [ ! -f "$PROJECT_ROOT/README.md" ] || [ ! -d "$PROJECT_ROOT/ansible" ]; then - print_error "Not in the correct project directory" - echo "Expected: $PROJECT_ROOT" - exit 1 - fi - print_success "Running from correct directory: $PROJECT_ROOT" - - # Check if Python 3 is installed - if ! command -v python3 &> /dev/null; then - print_error "Python 3 is not installed. Please install Python 3 first." - exit 1 - fi - print_success "Python 3 found: $(python3 --version)" - - # Check if git is installed - if ! command -v git &> /dev/null; then - print_warning "Git is not installed. Some features may not work." - else - print_success "Git found: $(git --version | head -n1)" - fi -} - -setup_python_venv() { - print_header "Setting Up Python Virtual Environment" - - cd "$PROJECT_ROOT" - - if [ -d "venv" ]; then - print_info "Virtual environment already exists" - if confirm_action "Recreate virtual environment?"; then - rm -rf venv - python3 -m venv venv - print_success "Virtual environment recreated" - else - print_success "Using existing virtual environment" - fi - else - python3 -m venv venv - print_success "Virtual environment created" - fi - - # Activate venv - source venv/bin/activate - print_success "Virtual environment activated" - - # Upgrade pip - print_info "Upgrading pip..." - pip install --upgrade pip > /dev/null 2>&1 - print_success "pip upgraded" -} - -install_python_requirements() { - print_header "Installing Python Requirements" - - cd "$PROJECT_ROOT" - - if [ ! -f "requirements.txt" ]; then - print_error "requirements.txt not found" - exit 1 - fi - - print_info "Installing packages from requirements.txt..." - pip install -r requirements.txt - print_success "Python requirements installed" - - # Verify Ansible installation - if ! command -v ansible &> /dev/null; then - print_error "Ansible installation failed" - exit 1 - fi - print_success "Ansible installed: $(ansible --version | head -n1)" -} - -install_ansible_collections() { - print_header "Installing Ansible Galaxy Collections" - - cd "$PROJECT_ROOT/ansible" - - if [ ! -f "requirements.yml" ]; then - print_warning "requirements.yml not found, skipping Ansible collections" - return - fi - - print_info "Installing Ansible Galaxy collections..." - ansible-galaxy collection install -r requirements.yml - print_success "Ansible Galaxy collections installed" -} - -setup_inventory_file() { - print_header "Setting Up Inventory File" - - cd "$PROJECT_ROOT/ansible" - - if [ -f "inventory.ini" ]; then - print_info "inventory.ini already exists" - cat inventory.ini - echo "" - if ! confirm_action "Do you want to update it?"; then - print_success "Using existing inventory.ini" - return - fi - fi - - print_info "Let's configure your infrastructure hosts" - echo "" - - # Collect information - echo -e -n "${BLUE}SSH key path${NC} [~/.ssh/counterganzua]: " - read ssh_key - ssh_key="${ssh_key:-~/.ssh/counterganzua}" - - echo "" - echo "Enter the IP addresses for your infrastructure (VMs will be added later):" - echo "" - - echo -e -n "${BLUE}vipy${NC} (main VPS) IP: " - read vipy_ip - echo -e -n "${BLUE}watchtower${NC} (monitoring VPS) IP: " - read watchtower_ip - echo -e -n "${BLUE}spacey${NC} (headscale VPS) IP: " - read spacey_ip - echo -e -n "${BLUE}nodito${NC} (Proxmox server) IP [optional]: " - read nodito_ip - - echo "" - echo -e -n "${BLUE}Your username on lapy${NC} [$(whoami)]: " - read lapy_user - lapy_user="${lapy_user:-$(whoami)}" - - echo -e -n "${BLUE}GPG recipient email${NC} [optional, for encrypted backups]: " - read gpg_email - echo -e -n "${BLUE}GPG key ID${NC} [optional, for encrypted backups]: " - read gpg_key - - # Generate inventory.ini - cat > inventory.ini << EOF -# Ansible Inventory File -# Generated by setup_layer_0.sh - -EOF - - vps_entries="" - if [ -n "$vipy_ip" ]; then - vps_entries+="vipy ansible_host=$vipy_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" - fi - if [ -n "$watchtower_ip" ]; then - vps_entries+="watchtower ansible_host=$watchtower_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" - fi - if [ -n "$spacey_ip" ]; then - vps_entries+="spacey ansible_host=$spacey_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" - fi - - if [ -n "$vps_entries" ]; then - cat >> inventory.ini << EOF -[vps] -${vps_entries} -EOF - fi - - if [ -n "$nodito_ip" ]; then - cat >> inventory.ini << EOF -[nodito_host] -nodito ansible_host=$nodito_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key - -EOF - fi - - # Add nodito_vms placeholder for VMs that will be created later - cat >> inventory.ini << EOF -# Nodito VMs - These don't exist yet and will be created on the Proxmox server -# Add them here once you create VMs on nodito (e.g., memos-box, etc.) -[nodito_vms] -# Example: -# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key - -EOF - - # Add lapy - cat >> inventory.ini << EOF -# Local connection to laptop: this assumes you're running ansible commands from your personal laptop -[lapy] -localhost ansible_connection=local ansible_user=$lapy_user -EOF - - if [ -n "$gpg_email" ] && [ -n "$gpg_key" ]; then - echo " gpg_recipient=$gpg_email gpg_key_id=$gpg_key" >> inventory.ini - fi - - print_success "inventory.ini created" - echo "" - print_info "Review your inventory file:" - cat inventory.ini - echo "" -} - -setup_infra_vars() { - print_header "Setting Up Infrastructure Variables" - - cd "$PROJECT_ROOT/ansible" - - if [ -f "infra_vars.yml" ]; then - print_info "infra_vars.yml already exists" - cat infra_vars.yml - echo "" - if ! confirm_action "Do you want to update it?"; then - print_success "Using existing infra_vars.yml" - return - fi - fi - - echo "" - echo -e -n "${BLUE}Your root domain${NC} (e.g., contrapeso.xyz): " - read domain - - while [ -z "$domain" ]; do - print_warning "Domain cannot be empty" - echo -e -n "${BLUE}Your root domain${NC}: " - read domain - done - - cat > infra_vars.yml << EOF -# Infrastructure Variables -# Generated by setup_layer_0.sh - -new_user: counterweight -ssh_port: 22 -allow_ssh_from: "any" -root_domain: $domain -EOF - - print_success "infra_vars.yml created" - echo "" - print_info "Contents:" - cat infra_vars.yml - echo "" -} - -setup_services_config() { - print_header "Setting Up Services Configuration" - - cd "$PROJECT_ROOT/ansible" - - if [ -f "services_config.yml" ]; then - print_info "services_config.yml already exists" - if ! confirm_action "Do you want to recreate it from template?"; then - print_success "Using existing services_config.yml" - return - fi - fi - - if [ ! -f "services_config.yml.example" ]; then - print_error "services_config.yml.example not found" - return - fi - - cp services_config.yml.example services_config.yml - - print_success "services_config.yml created" - echo "" - print_info "This file centralizes all service subdomains and Caddy settings" - print_info "Customize subdomains in: ansible/services_config.yml" - echo "" -} - -setup_infra_secrets() { - print_header "Setting Up Infrastructure Secrets" - - cd "$PROJECT_ROOT/ansible" - - if [ -f "infra_secrets.yml" ]; then - print_warning "infra_secrets.yml already exists" - if ! confirm_action "Do you want to recreate the template?"; then - print_success "Using existing infra_secrets.yml" - return - fi - fi - - cat > infra_secrets.yml << EOF -# Infrastructure Secrets -# Generated by setup_layer_0.sh -# -# IMPORTANT: This file contains sensitive credentials -# It is already in .gitignore - DO NOT commit it to git -# -# You'll need to fill in the Uptime Kuma credentials after Layer 4 -# when you deploy Uptime Kuma - -# Uptime Kuma Credentials (fill these in after deploying Uptime Kuma in Layer 4) -uptime_kuma_username: "" -uptime_kuma_password: "" -EOF - - print_success "infra_secrets.yml template created" - print_warning "You'll need to fill in Uptime Kuma credentials after Layer 4" - echo "" -} - -validate_ssh_key() { - print_header "Validating SSH Key" - - cd "$PROJECT_ROOT/ansible" - - # Extract SSH key path from inventory - if [ -f "inventory.ini" ]; then - ssh_key=$(grep "ansible_ssh_private_key_file" inventory.ini | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - - # Expand tilde - ssh_key="${ssh_key/#\~/$HOME}" - - if [ -f "$ssh_key" ]; then - print_success "SSH key found: $ssh_key" - - # Check permissions - perms=$(stat -c "%a" "$ssh_key" 2>/dev/null || stat -f "%OLp" "$ssh_key" 2>/dev/null) - if [ "$perms" != "600" ]; then - print_warning "SSH key permissions are $perms (should be 600)" - if confirm_action "Fix permissions?"; then - chmod 600 "$ssh_key" - print_success "Permissions fixed" - fi - else - print_success "SSH key permissions are correct (600)" - fi - else - print_error "SSH key not found: $ssh_key" - print_warning "Make sure to create your SSH key before proceeding to Layer 1" - echo "" - echo "To generate a new SSH key:" - echo " ssh-keygen -t ed25519 -f $ssh_key -C \"your-email@example.com\"" - fi - else - print_warning "inventory.ini not found, skipping SSH key validation" - fi -} - -print_summary() { - print_header "Layer 0 Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "Python virtual environment created and activated" - print_success "Ansible and dependencies installed" - print_success "Ansible Galaxy collections installed" - print_success "inventory.ini configured with your hosts" - print_success "infra_vars.yml configured with your domain" - print_success "services_config.yml created with subdomain settings" - print_success "infra_secrets.yml template created" - echo "" - - print_info "Before proceeding to Layer 1:" - echo " 1. Ensure your SSH key is added to all VPS root users" - echo " 2. Verify you can SSH into each machine manually" - echo " 3. Configure DNS nameservers for your domain (if not done)" - echo "" - - print_info "Note about inventory groups:" - echo " • [nodito_vms] group created as placeholder" - echo " • These VMs will be created later on Proxmox" - echo " • Add their host entries to inventory.ini once created" - echo "" - - print_info "To test SSH access to a host:" - echo " ssh -i ~/.ssh/counterganzua root@" - echo "" - - print_info "Next steps:" - echo " 1. Review the files in ansible/" - echo " 2. Test SSH connections to your hosts" - echo " 3. Proceed to Layer 1: ./scripts/setup_layer_1.sh" - echo "" - - print_warning "Remember to activate the venv before running other commands:" - echo " source venv/bin/activate" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🚀 Layer 0: Foundation Setup" - - echo "This script will set up your laptop (lapy) as the Ansible control node." - echo "It will install all prerequisites and configure basic settings." - echo "" - - if ! confirm_action "Continue with Layer 0 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_prerequisites - setup_python_venv - install_python_requirements - install_ansible_collections - setup_inventory_file - setup_infra_vars - setup_services_config - setup_infra_secrets - validate_ssh_key - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_1a_vps.sh b/scripts/setup_layer_1a_vps.sh deleted file mode 100755 index f60452f..0000000 --- a/scripts/setup_layer_1a_vps.sh +++ /dev/null @@ -1,393 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 1A: VPS Basic Setup -# -# This script configures users, SSH, firewall, and fail2ban on VPS machines. -# Runs independently - can be executed without Nodito setup. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_layer_0_complete() { - print_header "Verifying Layer 0 Prerequisites" - - local errors=0 - - # Check if venv exists - if [ ! -d "$PROJECT_ROOT/venv" ]; then - print_error "Python venv not found. Run Layer 0 first." - ((errors++)) - else - print_success "Python venv exists" - fi - - # Check if we're in a venv - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - # Check if Ansible is installed - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found: $(ansible --version | head -n1)" - fi - - # Check if inventory.ini exists - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - # Check if infra_vars.yml exists - if [ ! -f "$ANSIBLE_DIR/infra_vars.yml" ]; then - print_error "infra_vars.yml not found" - ((errors++)) - else - print_success "infra_vars.yml exists" - fi - - if [ $errors -gt 0 ]; then - print_error "Layer 0 is not complete. Please run ./scripts/setup_layer_0.sh first" - exit 1 - fi - - print_success "Layer 0 prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - - # Parse inventory.ini directly - more reliable than ansible-inventory - if [ -f "$ANSIBLE_DIR/inventory.ini" ]; then - # Look for the group section [target] - local in_section=false - local hosts="" - while IFS= read -r line; do - # Remove comments and whitespace - line=$(echo "$line" | sed 's/#.*$//' | xargs) - [ -z "$line" ] && continue - - # Check if we're entering the target section - if [[ "$line" =~ ^\[$target\]$ ]]; then - in_section=true - continue - fi - - # Check if we're entering a different section - if [[ "$line" =~ ^\[.*\]$ ]]; then - in_section=false - continue - fi - - # If we're in the target section, extract hostname - if [ "$in_section" = true ]; then - local hostname=$(echo "$line" | awk '{print $1}') - if [ -n "$hostname" ]; then - hosts="$hosts $hostname" - fi - fi - done < "$ANSIBLE_DIR/inventory.ini" - echo "$hosts" | xargs - fi -} - -check_vps_configured() { - print_header "Checking VPS Configuration" - - # Get all hosts from the vps group - local vps_hosts=$(get_hosts_from_inventory "vps") - local has_vps=false - - # Check for expected VPS hostnames - for expected_host in vipy watchtower spacey; do - if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then - print_success "$expected_host configured" - has_vps=true - else - print_info "$expected_host not configured (skipping)" - fi - done - - if [ "$has_vps" = false ]; then - print_error "No VPSs configured in inventory.ini" - print_info "Add at least one VPS (vipy, watchtower, or spacey) to the [vps] group to proceed" - exit 1 - fi - - echo "" -} - -check_ssh_connectivity() { - print_header "Testing SSH Connectivity as Root" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - print_info "Using SSH key: $ssh_key" - echo "" - - local all_good=true - - # Get all hosts from the vps group - local vps_hosts=$(get_hosts_from_inventory "vps") - - # Test VPSs (vipy, watchtower, spacey) - for expected_host in vipy watchtower spacey; do - if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then - print_info "Testing SSH to $expected_host as root..." - if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes root@$expected_host "echo 'SSH OK'" &>/dev/null; then - print_success "SSH to $expected_host as root: OK" - else - print_error "Cannot SSH to $expected_host as root" - print_warning "Make sure your SSH key is added to root on $expected_host" - all_good=false - fi - fi - done - - if [ "$all_good" = false ]; then - echo "" - print_error "SSH connectivity test failed" - print_info "To fix this:" - echo " 1. Ensure your VPS provider has added your SSH key to root" - echo " 2. Test manually: ssh -i $ssh_key root@" - echo "" - if ! confirm_action "Continue anyway?"; then - exit 1 - fi - fi - - echo "" - print_success "SSH connectivity verified" -} - -############################################################################### -# VPS Setup Functions -############################################################################### - -setup_vps_users_and_access() { - print_header "Setting Up Users and SSH Access on VPSs" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Create the 'counterweight' user with sudo access" - echo " • Configure SSH key authentication" - echo " • Disable root login (optional, configured in playbook)" - echo "" - print_info "Running: ansible-playbook -i inventory.ini infra/01_user_and_access_setup_playbook.yml" - echo "" - - if ! confirm_action "Proceed with user and access setup?"; then - print_warning "Skipped user and access setup" - return 1 - fi - - # Run the playbook with -e 'ansible_user="root"' to use root for this first run - if ansible-playbook -i inventory.ini infra/01_user_and_access_setup_playbook.yml -e 'ansible_user="root"'; then - print_success "User and access setup complete" - return 0 - else - print_error "User and access setup failed" - return 1 - fi -} - -setup_vps_firewall_and_fail2ban() { - print_header "Setting Up Firewall and Fail2ban on VPSs" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Configure UFW firewall with SSH access" - echo " • Install and configure fail2ban for brute force protection" - echo " • Install and configure auditd for security logging" - echo "" - print_info "Running: ansible-playbook -i inventory.ini infra/02_firewall_and_fail2ban_playbook.yml" - echo "" - - if ! confirm_action "Proceed with firewall and fail2ban setup?"; then - print_warning "Skipped firewall setup" - return 1 - fi - - # Now use the default counterweight user - if ansible-playbook -i inventory.ini infra/02_firewall_and_fail2ban_playbook.yml; then - print_success "Firewall and fail2ban setup complete" - return 0 - else - print_error "Firewall setup failed" - return 1 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_layer_1a() { - print_header "Verifying Layer 1A Completion" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - # Test SSH as counterweight user - print_info "Testing SSH as counterweight user..." - echo "" - - local all_good=true - - # Get all hosts from the vps group - local vps_hosts=$(get_hosts_from_inventory "vps") - - for expected_host in vipy watchtower spacey; do - if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then - if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$expected_host "echo 'SSH OK'" &>/dev/null; then - print_success "SSH to $expected_host as counterweight: OK" - else - print_error "Cannot SSH to $expected_host as counterweight" - all_good=false - fi - fi - done - - echo "" - if [ "$all_good" = true ]; then - print_success "All SSH connectivity verified" - else - print_warning "Some SSH tests failed - manual verification recommended" - print_info "Test manually: ssh -i $ssh_key counterweight@" - fi -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 1A: VPS Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "counterweight user created on all VPSs" - print_success "SSH key authentication configured" - print_success "UFW firewall active and configured" - print_success "fail2ban protecting against brute force attacks" - print_success "auditd logging security events" - echo "" - - print_warning "Important Security Changes:" - echo " • Root SSH login is now disabled (by design)" - echo " • Always use 'counterweight' user for SSH access" - echo " • Firewall is active - only SSH allowed by default" - echo "" - - print_info "Next steps:" - echo " 1. Test SSH access: ssh -i ~/.ssh/counterganzua counterweight@" - echo " 2. (Optional) Set up Nodito: ./scripts/setup_layer_1b_nodito.sh" - echo " 3. Proceed to Layer 2: ./scripts/setup_layer_2.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🔧 Layer 1A: VPS Basic Setup" - - echo "This script will configure users, SSH, firewall, and fail2ban on VPS machines." - echo "" - print_info "Targets: vipy, watchtower, spacey" - echo "" - - if ! confirm_action "Continue with Layer 1A setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_layer_0_complete - check_vps_configured - check_ssh_connectivity - - # VPS Setup - local setup_failed=false - setup_vps_users_and_access || setup_failed=true - setup_vps_firewall_and_fail2ban || setup_failed=true - - verify_layer_1a - - if [ "$setup_failed" = true ]; then - print_warning "Some steps failed - please review errors above" - fi - - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_1b_nodito.sh b/scripts/setup_layer_1b_nodito.sh deleted file mode 100755 index 5ebb243..0000000 --- a/scripts/setup_layer_1b_nodito.sh +++ /dev/null @@ -1,411 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 1B: Nodito (Proxmox) Setup -# -# This script configures the Nodito Proxmox server. -# Runs independently - can be executed without VPS setup. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_layer_0_complete() { - print_header "Verifying Layer 0 Prerequisites" - - local errors=0 - - # Check if venv exists - if [ ! -d "$PROJECT_ROOT/venv" ]; then - print_error "Python venv not found. Run Layer 0 first." - ((errors++)) - else - print_success "Python venv exists" - fi - - # Check if we're in a venv - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - # Check if Ansible is installed - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found: $(ansible --version | head -n1)" - fi - - # Check if inventory.ini exists - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - if [ $errors -gt 0 ]; then - print_error "Layer 0 is not complete. Please run ./scripts/setup_layer_0.sh first" - exit 1 - fi - - print_success "Layer 0 prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -check_nodito_configured() { - print_header "Checking Nodito Configuration" - - local nodito_hosts=$(get_hosts_from_inventory "nodito_host") - - if [ -z "$nodito_hosts" ]; then - print_error "No nodito host configured in inventory.ini" - print_info "Add the nodito host to the [nodito_host] group in inventory.ini to proceed" - exit 1 - fi - - print_success "Nodito configured: $nodito_hosts" - echo "" -} - -############################################################################### -# Nodito Setup Functions -############################################################################### - -setup_nodito_bootstrap() { - print_header "Bootstrapping Nodito (Proxmox Server)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Set up SSH key access for root" - echo " • Create the counterweight user with SSH keys" - echo " • Update and secure the system" - echo " • Disable root login and password authentication" - echo "" - print_info "Running: ansible-playbook -i inventory.ini infra/nodito/30_proxmox_bootstrap_playbook.yml" - print_warning "You will be prompted for the root password" - echo "" - - if ! confirm_action "Proceed with nodito bootstrap?"; then - print_warning "Skipped nodito bootstrap" - return 1 - fi - - # Run with root user and ask for password - if ansible-playbook -i inventory.ini infra/nodito/30_proxmox_bootstrap_playbook.yml -e 'ansible_user=root' --ask-pass; then - print_success "Nodito bootstrap complete" - return 0 - else - print_error "Nodito bootstrap failed" - return 1 - fi -} - -setup_nodito_community_repos() { - print_header "Switching Nodito to Community Repositories" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Remove enterprise repository files" - echo " • Add community repository files" - echo " • Disable subscription nag messages" - echo " • Update Proxmox packages" - echo "" - print_info "Running: ansible-playbook -i inventory.ini infra/nodito/31_proxmox_community_repos_playbook.yml" - echo "" - - if ! confirm_action "Proceed with community repos setup?"; then - print_warning "Skipped community repos setup" - return 1 - fi - - if ansible-playbook -i inventory.ini infra/nodito/31_proxmox_community_repos_playbook.yml; then - print_success "Community repositories configured" - print_warning "Clear browser cache before using Proxmox web UI (Ctrl+Shift+R)" - return 0 - else - print_error "Community repos setup failed" - return 1 - fi -} - -setup_nodito_zfs() { - print_header "Setting Up ZFS Storage Pool on Nodito (Optional)" - - cd "$ANSIBLE_DIR" - - print_warning "⚠️ ZFS setup will DESTROY ALL DATA on the specified disks!" - echo "" - print_info "Before proceeding, you must:" - echo " 1. SSH into nodito: ssh root@" - echo " 2. List disks: ls -la /dev/disk/by-id/ | grep -E '(ata-|scsi-|nvme-)'" - echo " 3. Identify the two disk IDs you want to use for RAID 1" - echo " 4. Edit ansible/infra/nodito/nodito_vars.yml" - echo " 5. Set zfs_disk_1 and zfs_disk_2 to your disk IDs" - echo "" - print_info "Example nodito_vars.yml content:" - echo ' zfs_disk_1: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K1234567"' - echo ' zfs_disk_2: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K7654321"' - echo "" - - if [ ! -f "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" ]; then - print_warning "nodito_vars.yml not found" - if confirm_action "Create nodito_vars.yml template?"; then - cat > "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" << 'EOF' -# Nodito Variables -# Configure these before running ZFS setup - -# ZFS Storage Pool Configuration -# Uncomment and configure these lines after identifying your disk IDs: -# zfs_disk_1: "/dev/disk/by-id/ata-YOUR-DISK-1-ID-HERE" -# zfs_disk_2: "/dev/disk/by-id/ata-YOUR-DISK-2-ID-HERE" -# zfs_pool_name: "proxmox-storage" - -# CPU Temperature Monitoring -monitoring_script_dir: /opt/cpu-temp-monitor -monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" -log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" -temp_threshold_celsius: 80 -EOF - print_success "Created nodito_vars.yml template" - print_info "Edit this file and configure ZFS disks, then re-run this script" - fi - return 1 - fi - - # Check if ZFS disks are configured - if ! grep -q "^zfs_disk_1:" "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" 2>/dev/null; then - print_info "ZFS disks not configured in nodito_vars.yml" - print_info "Edit ansible/infra/nodito/nodito_vars.yml to configure disk IDs" - if ! confirm_action "Skip ZFS setup for now?"; then - print_info "Please configure ZFS disks first" - return 1 - fi - print_warning "Skipped ZFS setup" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/nodito/32_zfs_pool_setup_playbook.yml" - echo "" - - if ! confirm_action "⚠️ Proceed with ZFS setup? (THIS WILL DESTROY DATA ON CONFIGURED DISKS)"; then - print_warning "Skipped ZFS setup" - return 1 - fi - - if ansible-playbook -i inventory.ini infra/nodito/32_zfs_pool_setup_playbook.yml; then - print_success "ZFS storage pool configured" - return 0 - else - print_error "ZFS setup failed" - return 1 - fi -} - -setup_nodito_cloud_template() { - print_header "Creating Debian Cloud Template on Nodito (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Download Debian cloud image" - echo " • Create a VM template (ID 9000)" - echo " • Configure cloud-init for easy VM creation" - echo "" - print_info "Running: ansible-playbook -i inventory.ini infra/nodito/33_proxmox_debian_cloud_template.yml" - echo "" - - if ! confirm_action "Proceed with cloud template creation?"; then - print_warning "Skipped cloud template creation" - return 1 - fi - - if ansible-playbook -i inventory.ini infra/nodito/33_proxmox_debian_cloud_template.yml; then - print_success "Debian cloud template created (VM ID 9000)" - return 0 - else - print_error "Cloud template creation failed" - return 1 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_layer_1b() { - print_header "Verifying Layer 1B Completion" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local nodito_hosts=$(get_hosts_from_inventory "nodito") - - print_info "Testing SSH as counterweight user..." - echo "" - - for host in $nodito_hosts; do - if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then - print_success "SSH to $host as counterweight: OK" - else - print_error "Cannot SSH to $host as counterweight" - print_info "Test manually: ssh -i $ssh_key counterweight@$host" - fi - done - - echo "" -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 1B: Nodito Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "Nodito bootstrapped with SSH keys" - print_success "counterweight user created" - print_success "Community repositories configured" - print_success "Root login and password auth disabled" - - if grep -q "^zfs_disk_1:" "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" 2>/dev/null; then - print_success "ZFS storage pool configured (if you ran it)" - fi - echo "" - - print_warning "Important Security Changes:" - echo " • Root SSH login is now disabled" - echo " • Always use 'counterweight' user for SSH access" - echo " • Password authentication is disabled" - echo "" - - print_info "Proxmox Web UI:" - local nodito_hosts=$(get_hosts_from_inventory "nodito") - echo " • Access at: https://$nodito_hosts:8006" - echo " • Clear browser cache (Ctrl+Shift+R) to avoid UI issues" - echo "" - - print_info "Next steps:" - echo " 1. Test SSH: ssh -i ~/.ssh/counterganzua counterweight@" - echo " 2. Access Proxmox web UI and verify community repos" - echo " 3. Create VMs on Proxmox (if needed)" - echo " 4. Proceed to Layer 2: ./scripts/setup_layer_2.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🖥️ Layer 1B: Nodito (Proxmox) Setup" - - echo "This script will configure your Nodito Proxmox server." - echo "" - print_info "Target: nodito (Proxmox server)" - echo "" - - if ! confirm_action "Continue with Layer 1B setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_layer_0_complete - check_nodito_configured - - # Nodito Setup - local setup_failed=false - setup_nodito_bootstrap || setup_failed=true - setup_nodito_community_repos || setup_failed=true - setup_nodito_zfs || setup_failed=true - setup_nodito_cloud_template || setup_failed=true - - verify_layer_1b - - if [ "$setup_failed" = true ]; then - print_warning "Some optional steps were skipped - this is normal" - fi - - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_2.sh b/scripts/setup_layer_2.sh deleted file mode 100755 index 1f35431..0000000 --- a/scripts/setup_layer_2.sh +++ /dev/null @@ -1,407 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 2: General Infrastructure Tools -# -# This script installs rsync and docker on the machines that need them. -# Must be run after Layer 1A (VPS) or Layer 1B (Nodito) is complete. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_layer_0_complete() { - print_header "Verifying Layer 0 Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - if [ $errors -gt 0 ]; then - print_error "Layer 0 is not complete" - exit 1 - fi - - print_success "Layer 0 prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -check_ssh_connectivity() { - print_header "Testing SSH Connectivity" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local all_good=true - - for group in vipy watchtower spacey nodito; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - for host in $hosts; do - print_info "Testing SSH to $host as counterweight..." - if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then - print_success "SSH to $host: OK" - else - print_error "Cannot SSH to $host as counterweight" - print_warning "Make sure Layer 1A or 1B is complete for this host" - all_good=false - fi - done - fi - done - - if [ "$all_good" = false ]; then - echo "" - print_error "SSH connectivity test failed" - print_info "Ensure Layer 1A (VPS) or Layer 1B (Nodito) is complete" - echo "" - if ! confirm_action "Continue anyway?"; then - exit 1 - fi - fi - - echo "" - print_success "SSH connectivity verified" -} - -############################################################################### -# rsync Installation -############################################################################### - -install_rsync() { - print_header "Installing rsync" - - cd "$ANSIBLE_DIR" - - print_info "rsync is needed for backup operations" - print_info "Recommended hosts: vipy, watchtower, lapy" - echo "" - - # Show available hosts - echo "Available hosts in inventory:" - for group in vipy watchtower spacey nodito lapy; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - fi - done - echo "" - - print_info "Installation options:" - echo " 1. Install on recommended hosts (vipy, watchtower, lapy)" - echo " 2. Install on all hosts" - echo " 3. Custom selection (specify groups)" - echo " 4. Skip rsync installation" - echo "" - - echo -e -n "${BLUE}Choose option${NC} [1-4]: " - read option - - local limit_hosts="" - case "$option" in - 1) - limit_hosts="vipy,watchtower,lapy" - print_info "Installing rsync on: vipy, watchtower, lapy" - ;; - 2) - limit_hosts="all" - print_info "Installing rsync on: all hosts" - ;; - 3) - echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower,nodito)${NC}: " - read limit_hosts - print_info "Installing rsync on: $limit_hosts" - ;; - 4) - print_warning "Skipping rsync installation" - return 1 - ;; - *) - print_error "Invalid option" - return 1 - ;; - esac - - echo "" - if ! confirm_action "Proceed with rsync installation?"; then - print_warning "Skipped rsync installation" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/900_install_rsync.yml --limit $limit_hosts" - echo "" - - if ansible-playbook -i inventory.ini infra/900_install_rsync.yml --limit "$limit_hosts"; then - print_success "rsync installation complete" - return 0 - else - print_error "rsync installation failed" - return 1 - fi -} - -############################################################################### -# Docker Installation -############################################################################### - -install_docker() { - print_header "Installing Docker and Docker Compose" - - cd "$ANSIBLE_DIR" - - print_info "Docker is needed for containerized services" - print_info "Recommended hosts: vipy, watchtower" - echo "" - - # Show available hosts (exclude lapy - docker on laptop is optional) - echo "Available hosts in inventory:" - for group in vipy watchtower spacey nodito; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - fi - done - echo "" - - print_info "Installation options:" - echo " 1. Install on recommended hosts (vipy, watchtower)" - echo " 2. Install on all hosts" - echo " 3. Custom selection (specify groups)" - echo " 4. Skip docker installation" - echo "" - - echo -e -n "${BLUE}Choose option${NC} [1-4]: " - read option - - local limit_hosts="" - case "$option" in - 1) - limit_hosts="vipy,watchtower" - print_info "Installing Docker on: vipy, watchtower" - ;; - 2) - limit_hosts="all" - print_info "Installing Docker on: all hosts" - ;; - 3) - echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower,nodito)${NC}: " - read limit_hosts - print_info "Installing Docker on: $limit_hosts" - ;; - 4) - print_warning "Skipping Docker installation" - return 1 - ;; - *) - print_error "Invalid option" - return 1 - ;; - esac - - echo "" - if ! confirm_action "Proceed with Docker installation?"; then - print_warning "Skipped Docker installation" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/910_docker_playbook.yml --limit $limit_hosts" - echo "" - - if ansible-playbook -i inventory.ini infra/910_docker_playbook.yml --limit "$limit_hosts"; then - print_success "Docker installation complete" - print_warning "You may need to log out and back in for docker group to take effect" - return 0 - else - print_error "Docker installation failed" - return 1 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_installations() { - print_header "Verifying Installations" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - echo "Checking installed tools on hosts..." - echo "" - - # Check all remote hosts - for group in vipy watchtower spacey nodito; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - for host in $hosts; do - print_info "Checking $host..." - - # Check rsync - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v rsync" &>/dev/null; then - print_success "$host: rsync installed" - else - print_warning "$host: rsync not found (may not be needed)" - fi - - # Check docker - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v docker" &>/dev/null; then - print_success "$host: docker installed" - - # Check docker service - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo systemctl is-active docker" &>/dev/null; then - print_success "$host: docker service running" - else - print_warning "$host: docker service not running" - fi - else - print_warning "$host: docker not found (may not be needed)" - fi - - echo "" - done - fi - done -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 2 Setup Complete! 🎉" - - echo "Summary:" - echo "" - print_success "Infrastructure tools installed on specified hosts" - echo "" - - print_info "What was installed:" - echo " • rsync - for backup operations" - echo " • docker + docker compose - for containerized services" - echo "" - - print_info "Next steps:" - echo " 1. Proceed to Layer 3: ./scripts/setup_layer_3_caddy.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🔧 Layer 2: General Infrastructure Tools" - - echo "This script will install rsync and docker on your infrastructure." - echo "" - - if ! confirm_action "Continue with Layer 2 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_layer_0_complete - check_ssh_connectivity - - # Install tools - install_rsync - echo "" - install_docker - - verify_installations - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_3_caddy.sh b/scripts/setup_layer_3_caddy.sh deleted file mode 100755 index 2ce0f6d..0000000 --- a/scripts/setup_layer_3_caddy.sh +++ /dev/null @@ -1,355 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 3: Reverse Proxy (Caddy) -# -# This script deploys Caddy reverse proxy on VPS machines. -# Must be run after Layer 1A (VPS setup) is complete. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_layer_0_complete() { - print_header "Verifying Layer 0 Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - if [ $errors -gt 0 ]; then - print_error "Layer 0 is not complete" - exit 1 - fi - - print_success "Layer 0 prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -check_target_hosts() { - print_header "Checking Target Hosts" - - local has_hosts=false - - print_info "Caddy will be deployed to these hosts:" - echo "" - - for group in vipy watchtower spacey; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - has_hosts=true - else - print_warning "[$group]: not configured (skipping)" - fi - done - - echo "" - - if [ "$has_hosts" = false ]; then - print_error "No target hosts configured for Caddy" - print_info "Caddy needs vipy, watchtower, or spacey in inventory.ini" - exit 1 - fi - - print_success "Target hosts verified" -} - -check_ssh_connectivity() { - print_header "Testing SSH Connectivity" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local all_good=true - - for group in vipy watchtower spacey; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - for host in $hosts; do - print_info "Testing SSH to $host as counterweight..." - if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then - print_success "SSH to $host: OK" - else - print_error "Cannot SSH to $host as counterweight" - print_warning "Make sure Layer 1A is complete for this host" - all_good=false - fi - done - fi - done - - if [ "$all_good" = false ]; then - echo "" - print_error "SSH connectivity test failed" - print_info "Ensure Layer 1A (VPS setup) is complete" - echo "" - if ! confirm_action "Continue anyway?"; then - exit 1 - fi - fi - - echo "" - print_success "SSH connectivity verified" -} - -############################################################################### -# Caddy Deployment -############################################################################### - -deploy_caddy() { - print_header "Deploying Caddy" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Install Caddy from official repositories" - echo " • Configure Caddy service" - echo " • Open firewall ports 80/443" - echo " • Create sites-enabled directory structure" - echo " • Enable automatic HTTPS with Let's Encrypt" - echo "" - - print_info "Target hosts: vipy, watchtower, spacey (if configured)" - echo "" - - print_warning "Important:" - echo " • Caddy will start with empty configuration" - echo " • Services will add their own config files in later layers" - echo " • Ports 80/443 must be available on the VPSs" - echo "" - - if ! confirm_action "Proceed with Caddy deployment?"; then - print_warning "Skipped Caddy deployment" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/caddy_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/caddy_playbook.yml; then - print_success "Caddy deployment complete" - return 0 - else - print_error "Caddy deployment failed" - return 1 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_caddy() { - print_header "Verifying Caddy Installation" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - echo "Checking Caddy on each host..." - echo "" - - for group in vipy watchtower spacey; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - for host in $hosts; do - print_info "Checking $host..." - - # Check if caddy is installed - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v caddy" &>/dev/null; then - print_success "$host: Caddy installed" - else - print_error "$host: Caddy not found" - continue - fi - - # Check if caddy service is running - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo systemctl is-active caddy" &>/dev/null; then - print_success "$host: Caddy service running" - else - print_error "$host: Caddy service not running" - fi - - # Check if sites-enabled directory exists - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "test -d /etc/caddy/sites-enabled" &>/dev/null; then - print_success "$host: sites-enabled directory exists" - else - print_warning "$host: sites-enabled directory not found" - fi - - # Check if ports 80/443 are open - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo ufw status | grep -E '80|443'" &>/dev/null; then - print_success "$host: Firewall ports 80/443 open" - else - print_warning "$host: Could not verify firewall ports" - fi - - echo "" - done - fi - done -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 3 Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "Caddy installed on VPS hosts" - print_success "Caddy service running" - print_success "Firewall ports 80/443 opened" - print_success "Sites-enabled directory structure created" - echo "" - - print_info "What Caddy provides:" - echo " • Automatic HTTPS with Let's Encrypt" - echo " • Reverse proxy for all web services" - echo " • HTTP/2 support" - echo " • Simple per-service configuration" - echo "" - - print_info "How services use Caddy:" - echo " • Each service adds a config file to /etc/caddy/sites-enabled/" - echo " • Main Caddyfile imports all configs" - echo " • Caddy automatically manages SSL certificates" - echo "" - - print_warning "Important Notes:" - echo " • Caddy is currently running with default/empty config" - echo " • Services deployed in later layers will add their configs" - echo " • DNS must point to your VPS IPs for SSL to work" - echo "" - - print_info "Next steps:" - echo " 1. Verify Caddy is accessible (optional): curl http://" - echo " 2. Proceed to Layer 4: ./scripts/setup_layer_4_monitoring.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🌐 Layer 3: Reverse Proxy (Caddy)" - - echo "This script will deploy Caddy reverse proxy on your VPS machines." - echo "" - print_info "Targets: vipy, watchtower, spacey" - echo "" - - if ! confirm_action "Continue with Layer 3 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_layer_0_complete - check_target_hosts - check_ssh_connectivity - - # Deploy Caddy - if deploy_caddy; then - verify_caddy - print_summary - else - print_error "Caddy deployment failed" - exit 1 - fi -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_4_monitoring.sh b/scripts/setup_layer_4_monitoring.sh deleted file mode 100755 index d82ad41..0000000 --- a/scripts/setup_layer_4_monitoring.sh +++ /dev/null @@ -1,806 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 4: Core Monitoring & Notifications -# -# This script deploys ntfy and Uptime Kuma on watchtower. -# Must be run after Layers 1A, 2, and 3 are complete. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -get_host_ip() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -hostvars = data.get('_meta', {}).get('hostvars', {}) -if target in hostvars: - print(hostvars[target].get('ansible_host', target)) -else: - hosts = data.get(target, {}).get('hosts', []) - if hosts: - first = hosts[0] - hv = hostvars.get(first, {}) - print(hv.get('ansible_host', first)) -PY -} - -############################################################################### -# Verification Functions -############################################################################### - -check_prerequisites() { - print_header "Verifying Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - # Check if watchtower is configured - if [ -z "$(get_hosts_from_inventory "watchtower")" ]; then - print_error "watchtower not configured in inventory.ini" - print_info "Layer 4 requires watchtower VPS" - ((errors++)) - else - print_success "watchtower configured in inventory" - fi - - if [ $errors -gt 0 ]; then - print_error "Prerequisites not met" - exit 1 - fi - - print_success "Prerequisites verified" -} - -check_vars_files() { - print_header "Checking Configuration Files" - - # Check services_config.yml - if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then - print_error "services_config.yml not found" - print_info "This file should have been created in Layer 0" - exit 1 - fi - - print_success "services_config.yml exists" - - # Show configured subdomains - local ntfy_sub=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") - local uptime_sub=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") - - print_info "Configured subdomains:" - echo " • ntfy: $ntfy_sub" - echo " • uptime_kuma: $uptime_sub" - echo "" -} - -check_dns_configuration() { - print_header "Validating DNS Configuration" - - cd "$ANSIBLE_DIR" - - # Get watchtower IP - local watchtower_ip=$(get_host_ip "watchtower") - - if [ -z "$watchtower_ip" ]; then - print_error "Could not determine watchtower IP from inventory" - return 1 - fi - - print_info "Watchtower IP: $watchtower_ip" - echo "" - - # Get domain from infra_vars.yml - local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) - - if [ -z "$root_domain" ]; then - print_error "Could not determine root_domain from infra_vars.yml" - return 1 - fi - - # Get subdomains from centralized config - local ntfy_subdomain="ntfy" - local uptime_subdomain="uptime" - - if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then - ntfy_subdomain=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") - uptime_subdomain=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") - fi - - local ntfy_fqdn="${ntfy_subdomain}.${root_domain}" - local uptime_fqdn="${uptime_subdomain}.${root_domain}" - - print_info "Checking DNS records..." - echo "" - - local dns_ok=true - - # Check ntfy DNS - print_info "Checking $ntfy_fqdn..." - if command -v dig &> /dev/null; then - local ntfy_resolved=$(dig +short "$ntfy_fqdn" | head -n1) - if [ "$ntfy_resolved" = "$watchtower_ip" ]; then - print_success "$ntfy_fqdn → $ntfy_resolved ✓" - elif [ -n "$ntfy_resolved" ]; then - print_error "$ntfy_fqdn → $ntfy_resolved (expected $watchtower_ip)" - dns_ok=false - else - print_error "$ntfy_fqdn does not resolve" - dns_ok=false - fi - else - print_warning "dig command not found, skipping DNS validation" - print_info "Install dnsutils/bind-tools to enable DNS validation" - return 1 - fi - - # Check Uptime Kuma DNS - print_info "Checking $uptime_fqdn..." - if command -v dig &> /dev/null; then - local uptime_resolved=$(dig +short "$uptime_fqdn" | head -n1) - if [ "$uptime_resolved" = "$watchtower_ip" ]; then - print_success "$uptime_fqdn → $uptime_resolved ✓" - elif [ -n "$uptime_resolved" ]; then - print_error "$uptime_fqdn → $uptime_resolved (expected $watchtower_ip)" - dns_ok=false - else - print_error "$uptime_fqdn does not resolve" - dns_ok=false - fi - fi - - echo "" - - if [ "$dns_ok" = false ]; then - print_error "DNS validation failed" - print_info "Please configure DNS records:" - echo " • $ntfy_fqdn → $watchtower_ip" - echo " • $uptime_fqdn → $watchtower_ip" - echo "" - print_warning "DNS changes can take time to propagate (up to 24-48 hours)" - echo "" - if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then - exit 1 - fi - else - print_success "DNS validation passed" - fi -} - -############################################################################### -# ntfy Deployment -############################################################################### - -deploy_ntfy() { - print_header "Deploying ntfy (Notification Service)" - - cd "$ANSIBLE_DIR" - - print_info "ntfy requires admin credentials for authentication" - echo "" - - # Check if env vars are set - if [ -z "$NTFY_USER" ] || [ -z "$NTFY_PASSWORD" ]; then - print_warning "NTFY_USER and NTFY_PASSWORD environment variables not set" - echo "" - print_info "Please enter credentials for ntfy admin user:" - echo "" - - echo -e -n "${BLUE}ntfy admin username${NC} [admin]: " - read ntfy_user - ntfy_user="${ntfy_user:-admin}" - - echo -e -n "${BLUE}ntfy admin password${NC}: " - read -s ntfy_password - echo "" - - if [ -z "$ntfy_password" ]; then - print_error "Password cannot be empty" - return 1 - fi - - export NTFY_USER="$ntfy_user" - export NTFY_PASSWORD="$ntfy_password" - else - print_success "Using NTFY_USER and NTFY_PASSWORD from environment" - fi - - echo "" - print_info "This will:" - echo " • Install ntfy from official repositories" - echo " • Configure ntfy with authentication (deny-all by default)" - echo " • Create admin user: $NTFY_USER" - echo " • Set up Caddy reverse proxy" - echo "" - - if ! confirm_action "Proceed with ntfy deployment?"; then - print_warning "Skipped ntfy deployment" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml; then - print_success "ntfy deployment complete" - echo "" - print_info "ntfy is now available at your configured subdomain" - print_info "Admin user: $NTFY_USER" - return 0 - else - print_error "ntfy deployment failed" - return 1 - fi -} - -############################################################################### -# Uptime Kuma Deployment -############################################################################### - -deploy_uptime_kuma() { - print_header "Deploying Uptime Kuma (Monitoring Platform)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Deploy Uptime Kuma via Docker" - echo " • Configure Caddy reverse proxy" - echo " • Set up data persistence" - echo "" - - if ! confirm_action "Proceed with Uptime Kuma deployment?"; then - print_warning "Skipped Uptime Kuma deployment" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml; then - print_success "Uptime Kuma deployment complete" - echo "" - print_warning "IMPORTANT: First-time setup required" - echo " 1. Access Uptime Kuma at your configured subdomain" - echo " 2. Create admin user on first visit" - echo " 3. Update ansible/infra_secrets.yml with credentials" - return 0 - else - print_error "Uptime Kuma deployment failed" - return 1 - fi -} - -############################################################################### -# Backup Configuration -############################################################################### - -setup_uptime_kuma_backup() { - print_header "Setting Up Uptime Kuma Backup (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "This will set up automated backups to lapy" - echo "" - - if ! confirm_action "Set up Uptime Kuma backup to lapy?"; then - print_warning "Skipped backup setup" - return 0 - fi - - # Check if rsync is available - print_info "Verifying rsync is installed on watchtower and lapy..." - if ! ansible watchtower -i inventory.ini -m shell -a "command -v rsync" &>/dev/null; then - print_error "rsync not found on watchtower" - print_info "Run Layer 2 to install rsync" - print_warning "Backup setup skipped - rsync not available" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml" - echo "" - - if ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml; then - print_success "Uptime Kuma backup configured" - print_info "Backups will run periodically via cron" - return 0 - else - print_error "Backup setup failed" - return 1 - fi -} - -############################################################################### -# Post-Deployment Configuration -############################################################################### - -setup_ntfy_notification() { - print_header "Setting Up ntfy Notification in Uptime Kuma (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "This will automatically configure ntfy as a notification method in Uptime Kuma" - print_warning "Prerequisites:" - echo " • Uptime Kuma admin account must be created first" - echo " • infra_secrets.yml must have Uptime Kuma credentials" - echo "" - - if ! confirm_action "Set up ntfy notification in Uptime Kuma?"; then - print_warning "Skipped ntfy notification setup" - print_info "You can set this up manually or run this script again later" - return 0 - fi - - # Check if infra_secrets.yml has Uptime Kuma credentials - if ! grep -q "uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null || \ - ! grep -q "uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then - print_error "Uptime Kuma credentials not found in infra_secrets.yml" - print_info "Please complete Step 1 and 2 of post-deployment steps first:" - echo " 1. Create admin user in Uptime Kuma web UI" - echo " 2. Add credentials to ansible/infra_secrets.yml" - print_warning "Skipped - you can run this script again after completing those steps" - return 0 - fi - - # Check credentials are not empty - local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - - if [ -z "$uk_user" ] || [ -z "$uk_pass" ]; then - print_error "Uptime Kuma credentials are empty in infra_secrets.yml" - print_info "Please update ansible/infra_secrets.yml with your credentials" - return 0 - fi - - print_success "Found Uptime Kuma credentials in infra_secrets.yml" - - print_info "Running playbook to configure ntfy notification..." - echo "" - - if ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml; then - print_success "ntfy notification configured in Uptime Kuma" - print_info "You can now use ntfy for all your monitors!" - return 0 - else - print_error "Failed to configure ntfy notification" - print_info "You can set this up manually or run the playbook again later:" - echo " ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml" - return 0 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_deployments() { - print_header "Verifying Deployments" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local watchtower_host - watchtower_host=$(get_hosts_from_inventory "watchtower") - - if [ -z "$watchtower_host" ]; then - print_error "Could not determine watchtower host" - return - fi - - print_info "Checking services on watchtower ($watchtower_host)..." - echo "" - - # Check ntfy - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "systemctl is-active ntfy" &>/dev/null; then - print_success "ntfy service running" - else - print_warning "ntfy service not running or not installed" - fi - - # Check Uptime Kuma docker container - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "docker ps | grep uptime-kuma" &>/dev/null; then - print_success "Uptime Kuma container running" - else - print_warning "Uptime Kuma container not running" - fi - - # Check Caddy configs - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/ntfy.conf" &>/dev/null; then - print_success "ntfy Caddy config exists" - else - print_warning "ntfy Caddy config not found" - fi - - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/uptime-kuma.conf" &>/dev/null; then - print_success "Uptime Kuma Caddy config exists" - else - print_warning "Uptime Kuma Caddy config not found" - fi - - echo "" -} - -verify_final_setup() { - print_header "Final Verification - Post-Deployment Steps" - - cd "$ANSIBLE_DIR" - - print_info "Checking if all post-deployment steps were completed..." - echo "" - - local all_ok=true - - # Check 1: infra_secrets.yml has Uptime Kuma credentials - print_info "Checking infra_secrets.yml..." - if grep -q "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null && \ - grep -q "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then - local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - - if [ -n "$uk_user" ] && [ -n "$uk_pass" ] && [ "$uk_user" != '""' ] && [ "$uk_pass" != '""' ]; then - print_success "Uptime Kuma credentials configured in infra_secrets.yml" - else - print_error "Uptime Kuma credentials are empty in infra_secrets.yml" - print_info "Please complete Step 2: Update infra_secrets.yml" - all_ok=false - fi - else - print_error "Uptime Kuma credentials not found in infra_secrets.yml" - print_info "Please complete Step 2: Update infra_secrets.yml" - all_ok=false - fi - - echo "" - - # Check 2: Can connect to Uptime Kuma API - print_info "Checking Uptime Kuma API access..." - - if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then - # Create a test Python script to check API access - local test_script=$(mktemp) - cat > "$test_script" << 'EOFPYTHON' -import sys -import yaml -from uptime_kuma_api import UptimeKumaApi - -try: - # Load config - with open('infra_vars.yml', 'r') as f: - infra_vars = yaml.safe_load(f) - - with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: - uk_vars = yaml.safe_load(f) - - with open('infra_secrets.yml', 'r') as f: - secrets = yaml.safe_load(f) - - root_domain = infra_vars.get('root_domain') - subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') - url = f"https://{subdomain}.{root_domain}" - - username = secrets.get('uptime_kuma_username') - password = secrets.get('uptime_kuma_password') - - # Try to connect - api = UptimeKumaApi(url) - api.login(username, password) - - # Check if we can get monitors - monitors = api.get_monitors() - - print(f"SUCCESS:{len(monitors)}") - api.disconnect() - sys.exit(0) - -except Exception as e: - print(f"ERROR:{str(e)}", file=sys.stderr) - sys.exit(1) -EOFPYTHON - - local result=$(cd "$ANSIBLE_DIR" && python3 "$test_script" 2>&1) - rm -f "$test_script" - - if echo "$result" | grep -q "^SUCCESS:"; then - local monitor_count=$(echo "$result" | grep "^SUCCESS:" | cut -d: -f2) - print_success "Successfully connected to Uptime Kuma API" - print_info "Current monitors: $monitor_count" - else - print_error "Cannot connect to Uptime Kuma API" - print_warning "This usually means:" - echo " • Admin account not created yet (Step 1)" - echo " • Wrong credentials in infra_secrets.yml (Step 2)" - echo " • Uptime Kuma not accessible" - all_ok=false - fi - else - print_warning "Skipping API check - credentials not configured" - all_ok=false - fi - - echo "" - - # Check 3: ntfy notification configured in Uptime Kuma - print_info "Checking ntfy notification configuration..." - - if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then - local test_notif=$(mktemp) - cat > "$test_notif" << 'EOFPYTHON' -import sys -import yaml -from uptime_kuma_api import UptimeKumaApi - -try: - # Load config - with open('infra_vars.yml', 'r') as f: - infra_vars = yaml.safe_load(f) - - with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: - uk_vars = yaml.safe_load(f) - - with open('infra_secrets.yml', 'r') as f: - secrets = yaml.safe_load(f) - - root_domain = infra_vars.get('root_domain') - subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') - url = f"https://{subdomain}.{root_domain}" - - username = secrets.get('uptime_kuma_username') - password = secrets.get('uptime_kuma_password') - - # Connect - api = UptimeKumaApi(url) - api.login(username, password) - - # Check for ntfy notification - notifications = api.get_notifications() - ntfy_found = any(n.get('type') == 'ntfy' for n in notifications) - - if ntfy_found: - print("SUCCESS:ntfy notification configured") - else: - print("NOTFOUND:No ntfy notification found") - - api.disconnect() - sys.exit(0) - -except Exception as e: - print(f"ERROR:{str(e)}", file=sys.stderr) - sys.exit(1) -EOFPYTHON - - local notif_result=$(cd "$ANSIBLE_DIR" && python3 "$test_notif" 2>&1) - rm -f "$test_notif" - - if echo "$notif_result" | grep -q "^SUCCESS:"; then - print_success "ntfy notification is configured in Uptime Kuma" - elif echo "$notif_result" | grep -q "^NOTFOUND:"; then - print_warning "ntfy notification not yet configured" - print_info "Run the script again and choose 'yes' for ntfy notification setup" - print_info "Or complete Step 3 manually" - all_ok=false - else - print_warning "Could not verify ntfy notification (API access issue)" - fi - else - print_warning "Skipping ntfy check - credentials not configured" - fi - - echo "" - - # Summary - if [ "$all_ok" = true ]; then - print_success "All post-deployment steps completed! ✓" - echo "" - print_info "Layer 4 is fully configured and ready to use" - print_info "You can now proceed to Layer 6 (infrastructure monitoring)" - return 0 - else - print_warning "Some post-deployment steps are incomplete" - echo "" - print_info "Complete these steps:" - echo " 1. Access Uptime Kuma web UI and create admin account" - echo " 2. Update ansible/infra_secrets.yml with credentials" - echo " 3. Run this script again to configure ntfy notification" - echo "" - print_info "You can also complete manually and verify with:" - echo " ./scripts/setup_layer_4_monitoring.sh" - return 1 - fi -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 4 Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "ntfy notification service deployed" - print_success "Uptime Kuma monitoring platform deployed" - print_success "Caddy reverse proxy configured for both services" - echo "" - - print_warning "REQUIRED POST-DEPLOYMENT STEPS:" - echo "" - echo "MANUAL (do these first):" - echo " 1. Access Uptime Kuma Web UI and create admin account" - echo " 2. Update ansible/infra_secrets.yml with credentials" - echo "" - echo "AUTOMATED (script can do these):" - echo " 3. Configure ntfy notification - script will offer to set this up" - echo " 4. Final verification - script will check everything" - echo "" - print_info "After completing steps 1 & 2, the script will:" - echo " • Automatically configure ntfy in Uptime Kuma" - echo " • Verify all post-deployment steps" - echo " • Tell you if anything is missing" - echo "" - print_warning "You MUST complete steps 1 & 2 before proceeding to Layer 6!" - echo "" - - print_info "What these services enable:" - echo " • ntfy: Push notifications to your devices" - echo " • Uptime Kuma: Monitor all services and infrastructure" - echo " • Together: Complete monitoring and alerting solution" - echo "" - - print_info "Next steps:" - echo " 1. Complete the post-deployment steps above" - echo " 2. Test ntfy: Send a test notification" - echo " 3. Test Uptime Kuma: Create a test monitor" - echo " 4. Proceed to Layer 5: ./scripts/setup_layer_5_headscale.sh (optional)" - echo " OR Layer 6: ./scripts/setup_layer_6_infra_monitoring.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "📊 Layer 4: Core Monitoring & Notifications" - - echo "This script will deploy ntfy and Uptime Kuma on watchtower." - echo "" - print_info "Services to deploy:" - echo " • ntfy (notification service)" - echo " • Uptime Kuma (monitoring platform)" - echo "" - - if ! confirm_action "Continue with Layer 4 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_prerequisites - check_vars_files - check_dns_configuration - - # Deploy services (don't fail if skipped) - deploy_ntfy || true - echo "" - deploy_uptime_kuma || true - echo "" - setup_uptime_kuma_backup || true - - echo "" - verify_deployments - - # Always show summary and offer ntfy configuration - print_summary - echo "" - - # Always ask about ntfy notification setup (regardless of deployment status) - print_header "Configure ntfy Notification in Uptime Kuma" - print_info "After creating your Uptime Kuma admin account and updating infra_secrets.yml," - print_info "the script can automatically configure ntfy as a notification method." - echo "" - print_warning "Prerequisites:" - echo " 1. Access Uptime Kuma web UI and create admin account" - echo " 2. Update ansible/infra_secrets.yml with your credentials" - echo "" - - # Always offer to set up ntfy notification - setup_ntfy_notification - - # Final verification - echo "" - verify_final_setup -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_5_headscale.sh b/scripts/setup_layer_5_headscale.sh deleted file mode 100755 index 0c89745..0000000 --- a/scripts/setup_layer_5_headscale.sh +++ /dev/null @@ -1,524 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 5: VPN Infrastructure (Headscale) -# -# This script deploys Headscale and optionally joins machines to the mesh. -# Must be run after Layers 0, 1A, and 3 are complete. -# THIS LAYER IS OPTIONAL - skip to Layer 6 if you don't need VPN. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_prerequisites() { - print_header "Verifying Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - # Check if spacey is configured - if [ -z "$(get_hosts_from_inventory "spacey")" ]; then - print_error "spacey not configured in inventory.ini" - print_info "Layer 5 requires spacey VPS for Headscale server" - ((errors++)) - else - print_success "spacey configured in inventory" - fi - - if [ $errors -gt 0 ]; then - print_error "Prerequisites not met" - exit 1 - fi - - print_success "Prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -get_host_ip() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -hostvars = data.get('_meta', {}).get('hostvars', {}) -if target in hostvars: - print(hostvars[target].get('ansible_host', target)) -else: - hosts = data.get(target, {}).get('hosts', []) - if hosts: - first = hosts[0] - hv = hostvars.get(first, {}) - print(hv.get('ansible_host', first)) -PY -} - -check_vars_files() { - print_header "Checking Configuration Files" - - # Check services_config.yml - if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then - print_error "services_config.yml not found" - print_info "This file should have been created in Layer 0" - exit 1 - fi - - print_success "services_config.yml exists" - - # Show configured subdomain - local hs_sub=$(grep "^ headscale:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "headscale") - print_info "Configured subdomain: headscale: $hs_sub" - echo "" -} - -check_dns_configuration() { - print_header "Validating DNS Configuration" - - cd "$ANSIBLE_DIR" - - # Get spacey IP - local spacey_ip=$(get_host_ip "spacey") - - if [ -z "$spacey_ip" ]; then - print_error "Could not determine spacey IP from inventory" - return 1 - fi - - print_info "Spacey IP: $spacey_ip" - echo "" - - # Get domain from infra_vars.yml - local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) - - if [ -z "$root_domain" ]; then - print_error "Could not determine root_domain from infra_vars.yml" - return 1 - fi - - # Get subdomain from centralized config - local headscale_subdomain="headscale" - - if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then - headscale_subdomain=$(grep "^ headscale:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "headscale") - fi - - local headscale_fqdn="${headscale_subdomain}.${root_domain}" - - print_info "Checking DNS record..." - echo "" - - # Check Headscale DNS - print_info "Checking $headscale_fqdn..." - if command -v dig &> /dev/null; then - local resolved=$(dig +short "$headscale_fqdn" | head -n1) - if [ "$resolved" = "$spacey_ip" ]; then - print_success "$headscale_fqdn → $resolved ✓" - elif [ -n "$resolved" ]; then - print_error "$headscale_fqdn → $resolved (expected $spacey_ip)" - print_warning "DNS changes can take time to propagate (up to 24-48 hours)" - echo "" - if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then - exit 1 - fi - else - print_error "$headscale_fqdn does not resolve" - print_warning "DNS changes can take time to propagate" - echo "" - if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then - exit 1 - fi - fi - else - print_warning "dig command not found, skipping DNS validation" - print_info "Install dnsutils/bind-tools to enable DNS validation" - fi - - echo "" - print_success "DNS validation complete" -} - -############################################################################### -# Headscale Deployment -############################################################################### - -deploy_headscale() { - print_header "Deploying Headscale Server" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Install Headscale on spacey" - echo " • Configure with deny-all ACL policy (you customize later)" - echo " • Create namespace for your network" - echo " • Set up Caddy reverse proxy" - echo " • Configure embedded DERP server" - echo "" - - print_warning "After deployment, you MUST configure ACL policies for machines to communicate" - echo "" - - if ! confirm_action "Proceed with Headscale deployment?"; then - print_warning "Skipped Headscale deployment" - return 1 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/headscale/deploy_headscale_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/headscale/deploy_headscale_playbook.yml; then - print_success "Headscale deployment complete" - return 0 - else - print_error "Headscale deployment failed" - return 1 - fi -} - -############################################################################### -# Join Machines to Mesh -############################################################################### - -join_machines_to_mesh() { - print_header "Join Machines to Mesh (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "This will install Tailscale client and join machines to your Headscale mesh" - echo "" - - # Show available hosts - echo "Available hosts to join:" - for group in vipy watchtower nodito lapy; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - fi - done - echo "" - - print_info "Join options:" - echo " 1. Join recommended machines (vipy, watchtower, nodito)" - echo " 2. Join all machines" - echo " 3. Custom selection (specify groups)" - echo " 4. Skip - join machines later manually" - echo "" - - echo -e -n "${BLUE}Choose option${NC} [1-4]: " - read option - - local limit_hosts="" - case "$option" in - 1) - limit_hosts="vipy,watchtower,nodito" - print_info "Joining: vipy, watchtower, nodito" - ;; - 2) - limit_hosts="all" - print_info "Joining: all hosts" - ;; - 3) - echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower)${NC}: " - read limit_hosts - print_info "Joining: $limit_hosts" - ;; - 4) - print_warning "Skipping machine join - you can join manually later" - print_info "To join manually:" - echo " ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit " - return 0 - ;; - *) - print_error "Invalid option" - return 0 - ;; - esac - - echo "" - if ! confirm_action "Proceed with joining machines?"; then - print_warning "Skipped joining machines" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit $limit_hosts" - echo "" - - if ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit "$limit_hosts"; then - print_success "Machines joined to mesh" - return 0 - else - print_error "Failed to join some machines" - print_info "You can retry or join manually later" - return 0 - fi -} - -############################################################################### -# Backup Configuration -############################################################################### - -setup_headscale_backup() { - print_header "Setting Up Headscale Backup (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "This will set up automated backups to lapy" - echo "" - - if ! confirm_action "Set up Headscale backup to lapy?"; then - print_warning "Skipped backup setup" - return 0 - fi - - # Check if rsync is available - print_info "Verifying rsync is installed on spacey and lapy..." - if ! ansible spacey -i inventory.ini -m shell -a "command -v rsync" &>/dev/null; then - print_error "rsync not found on spacey" - print_info "Run Layer 2 to install rsync" - print_warning "Backup setup skipped - rsync not available" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml" - echo "" - - if ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml; then - print_success "Headscale backup configured" - print_info "Backups will run periodically via cron" - return 0 - else - print_error "Backup setup failed" - return 0 - fi -} - -############################################################################### -# Verification Functions -############################################################################### - -verify_deployment() { - print_header "Verifying Headscale Deployment" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local spacey_host=$(get_hosts_from_inventory "spacey") - - if [ -z "$spacey_host" ]; then - print_error "Could not determine spacey host" - return - fi - - print_info "Checking Headscale on spacey ($spacey_host)..." - echo "" - - # Check Headscale service - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "systemctl is-active headscale" &>/dev/null; then - print_success "Headscale service running" - else - print_warning "Headscale service not running" - fi - - # Check Caddy config - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "test -f /etc/caddy/sites-enabled/headscale.conf" &>/dev/null; then - print_success "Headscale Caddy config exists" - else - print_warning "Headscale Caddy config not found" - fi - - # Check ACL file - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "test -f /etc/headscale/acl.json" &>/dev/null; then - print_success "ACL policy file exists" - else - print_warning "ACL policy file not found" - fi - - # List nodes - print_info "Attempting to list connected nodes..." - local nodes_output=$(timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "sudo headscale nodes list" 2>/dev/null || echo "") - - if [ -n "$nodes_output" ]; then - echo "$nodes_output" - else - print_warning "Could not list nodes (this is normal if no machines joined yet)" - fi - - echo "" -} - -############################################################################### -# Summary Functions -############################################################################### - -print_summary() { - print_header "Layer 5 Setup Complete! 🎉" - - echo "Summary of what was configured:" - echo "" - print_success "Headscale VPN server deployed on spacey" - print_success "Caddy reverse proxy configured" - print_success "Namespace created for your network" - echo "" - - print_warning "CRITICAL POST-DEPLOYMENT STEPS:" - echo "" - echo "1. Configure ACL Policies (REQUIRED for machines to communicate):" - echo " • SSH to spacey: ssh counterweight@" - echo " • Edit ACL: sudo nano /etc/headscale/acl.json" - echo " • Add rules to allow communication" - echo " • Restart: sudo systemctl restart headscale" - echo "" - echo "2. Verify machines joined (if you selected that option):" - echo " • SSH to spacey: ssh counterweight@" - echo " • List nodes: sudo headscale nodes list" - echo "" - echo "3. Join additional machines (mobile, desktop):" - echo " • Generate key: sudo headscale preauthkeys create --user --reusable" - echo " • On device: tailscale up --login-server https:// --authkey " - echo "" - - print_info "What Headscale enables:" - echo " • Secure mesh networking between all machines" - echo " • Magic DNS - access machines by hostname" - echo " • NAT traversal - works behind firewalls" - echo " • Self-hosted Tailscale alternative" - echo "" - - print_info "Next steps:" - echo " 1. Configure ACL policies on spacey" - echo " 2. Verify nodes are connected" - echo " 3. Proceed to Layer 6: ./scripts/setup_layer_6_infra_monitoring.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🔐 Layer 5: VPN Infrastructure (Headscale)" - - echo "This script will deploy Headscale for secure mesh networking." - echo "" - print_warning "THIS LAYER IS OPTIONAL" - print_info "Skip to Layer 6 if you don't need VPN mesh networking" - echo "" - - if ! confirm_action "Continue with Layer 5 setup?"; then - echo "Setup skipped - proceeding to Layer 6 is fine!" - exit 0 - fi - - check_prerequisites - check_vars_files - check_dns_configuration - - # Deploy Headscale - if deploy_headscale; then - echo "" - join_machines_to_mesh - echo "" - setup_headscale_backup - echo "" - verify_deployment - print_summary - else - print_error "Headscale deployment failed" - exit 1 - fi -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_6_infra_monitoring.sh b/scripts/setup_layer_6_infra_monitoring.sh deleted file mode 100755 index 7c12780..0000000 --- a/scripts/setup_layer_6_infra_monitoring.sh +++ /dev/null @@ -1,473 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 6: Infrastructure Monitoring -# -# This script deploys disk usage, healthcheck, and CPU temp monitoring. -# Must be run after Layer 4 (Uptime Kuma) is complete with credentials set. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_prerequisites() { - print_header "Verifying Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - # Check Python uptime-kuma-api - if ! python3 -c "import uptime_kuma_api" 2>/dev/null; then - print_error "uptime-kuma-api Python package not found" - print_info "Install with: pip install -r requirements.txt" - ((errors++)) - else - print_success "uptime-kuma-api package found" - fi - - if [ $errors -gt 0 ]; then - print_error "Prerequisites not met" - exit 1 - fi - - print_success "Prerequisites verified" -} - -check_uptime_kuma_credentials() { - print_header "Verifying Uptime Kuma Configuration" - - cd "$ANSIBLE_DIR" - - # Check if infra_secrets.yml has credentials - if ! grep -q "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null || \ - ! grep -q "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then - print_error "Uptime Kuma credentials not found in infra_secrets.yml" - print_info "You must complete Layer 4 post-deployment steps first:" - echo " 1. Create admin user in Uptime Kuma web UI" - echo " 2. Add credentials to ansible/infra_secrets.yml" - exit 1 - fi - - local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") - - if [ -z "$uk_user" ] || [ -z "$uk_pass" ]; then - print_error "Uptime Kuma credentials are empty in infra_secrets.yml" - exit 1 - fi - - print_success "Uptime Kuma credentials found" - - # Test API connection - print_info "Testing Uptime Kuma API connection..." - - local test_script=$(mktemp) - cat > "$test_script" << 'EOFPYTHON' -import sys -import yaml -from uptime_kuma_api import UptimeKumaApi - -try: - with open('infra_vars.yml', 'r') as f: - infra_vars = yaml.safe_load(f) - - with open('services_config.yml', 'r') as f: - services_config = yaml.safe_load(f) - - with open('infra_secrets.yml', 'r') as f: - secrets = yaml.safe_load(f) - - root_domain = infra_vars.get('root_domain') - subdomain = services_config.get('subdomains', {}).get('uptime_kuma', 'uptime') - url = f"https://{subdomain}.{root_domain}" - - username = secrets.get('uptime_kuma_username') - password = secrets.get('uptime_kuma_password') - - api = UptimeKumaApi(url) - api.login(username, password) - - monitors = api.get_monitors() - print(f"SUCCESS:{len(monitors)}") - api.disconnect() - -except Exception as e: - print(f"ERROR:{str(e)}", file=sys.stderr) - sys.exit(1) -EOFPYTHON - - local result=$(cd "$ANSIBLE_DIR" && python3 "$test_script" 2>&1) - rm -f "$test_script" - - if echo "$result" | grep -q "^SUCCESS:"; then - local monitor_count=$(echo "$result" | grep "^SUCCESS:" | cut -d: -f2) - print_success "Successfully connected to Uptime Kuma API" - print_info "Current monitors: $monitor_count" - else - print_error "Cannot connect to Uptime Kuma API" - print_info "Error: $result" - echo "" - print_info "Make sure:" - echo " • Uptime Kuma is running (Layer 4)" - echo " • Credentials are correct in infra_secrets.yml" - echo " • Uptime Kuma is accessible" - exit 1 - fi - - echo "" - print_success "Uptime Kuma configuration verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -############################################################################### -# Disk Usage Monitoring -############################################################################### - -deploy_disk_usage_monitoring() { - print_header "Deploying Disk Usage Monitoring" - - cd "$ANSIBLE_DIR" - - print_info "This will deploy disk usage monitoring on selected hosts" - print_info "Default settings:" - echo " • Threshold: 80%" - echo " • Check interval: 15 minutes" - echo " • Mount point: /" - echo "" - - # Show available hosts - echo "Available hosts:" - for group in vipy watchtower spacey nodito lapy; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - fi - done - echo "" - - print_info "Deployment options:" - echo " 1. Deploy on all remote hosts (vipy, watchtower, spacey, nodito)" - echo " 2. Deploy on all hosts (including lapy)" - echo " 3. Custom selection (specify groups)" - echo " 4. Skip disk monitoring" - echo "" - - echo -e -n "${BLUE}Choose option${NC} [1-4]: " - read option - - local limit_hosts="" - case "$option" in - 1) - limit_hosts="vipy,watchtower,spacey,nodito" - print_info "Deploying to remote hosts" - ;; - 2) - limit_hosts="all" - print_info "Deploying to all hosts" - ;; - 3) - echo -e -n "${BLUE}Enter groups (comma-separated)${NC}: " - read limit_hosts - print_info "Deploying to: $limit_hosts" - ;; - 4) - print_warning "Skipping disk usage monitoring" - return 0 - ;; - *) - print_error "Invalid option" - return 0 - ;; - esac - - echo "" - if ! confirm_action "Proceed with disk usage monitoring deployment?"; then - print_warning "Skipped" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml --limit $limit_hosts" - echo "" - - if ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml --limit "$limit_hosts"; then - print_success "Disk usage monitoring deployed" - return 0 - else - print_error "Deployment failed" - return 0 - fi -} - -############################################################################### -# System Healthcheck Monitoring -############################################################################### - -deploy_system_healthcheck() { - print_header "Deploying System Healthcheck Monitoring" - - cd "$ANSIBLE_DIR" - - print_info "This will deploy system healthcheck monitoring on selected hosts" - print_info "Default settings:" - echo " • Heartbeat interval: 60 seconds" - echo " • Upside-down mode (no news is good news)" - echo "" - - # Show available hosts - echo "Available hosts:" - for group in vipy watchtower spacey nodito lapy; do - local hosts=$(get_hosts_from_inventory "$group") - if [ -n "$hosts" ]; then - echo " [$group]: $hosts" - fi - done - echo "" - - print_info "Deployment options:" - echo " 1. Deploy on all remote hosts (vipy, watchtower, spacey, nodito)" - echo " 2. Deploy on all hosts (including lapy)" - echo " 3. Custom selection (specify groups)" - echo " 4. Skip healthcheck monitoring" - echo "" - - echo -e -n "${BLUE}Choose option${NC} [1-4]: " - read option - - local limit_hosts="" - case "$option" in - 1) - limit_hosts="vipy,watchtower,spacey,nodito" - print_info "Deploying to remote hosts" - ;; - 2) - limit_hosts="all" - print_info "Deploying to all hosts" - ;; - 3) - echo -e -n "${BLUE}Enter groups (comma-separated)${NC}: " - read limit_hosts - print_info "Deploying to: $limit_hosts" - ;; - 4) - print_warning "Skipping healthcheck monitoring" - return 0 - ;; - *) - print_error "Invalid option" - return 0 - ;; - esac - - echo "" - if ! confirm_action "Proceed with healthcheck monitoring deployment?"; then - print_warning "Skipped" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml --limit $limit_hosts" - echo "" - - if ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml --limit "$limit_hosts"; then - print_success "System healthcheck monitoring deployed" - return 0 - else - print_error "Deployment failed" - return 0 - fi -} - -############################################################################### -# CPU Temperature Monitoring (Nodito) -############################################################################### - -deploy_cpu_temp_monitoring() { - print_header "Deploying CPU Temperature Monitoring (Nodito)" - - cd "$ANSIBLE_DIR" - - # Check if nodito is configured - local nodito_hosts=$(get_hosts_from_inventory "nodito") - if [ -z "$nodito_hosts" ]; then - print_info "Nodito not configured in inventory, skipping CPU temp monitoring" - return 0 - fi - - print_info "This will deploy CPU temperature monitoring on nodito (Proxmox)" - print_info "Default settings:" - echo " • Threshold: 80°C" - echo " • Check interval: 60 seconds" - echo "" - - echo "" - if ! confirm_action "Proceed with CPU temp monitoring deployment?"; then - print_warning "Skipped" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml" - echo "" - - if ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml; then - print_success "CPU temperature monitoring deployed" - return 0 - else - print_error "Deployment failed" - return 0 - fi -} - -############################################################################### -# Summary -############################################################################### - -print_summary() { - print_header "Layer 6 Setup Complete! 🎉" - - echo "Summary of what was deployed:" - echo "" - print_success "Infrastructure monitoring configured" - print_success "Monitors created in Uptime Kuma" - print_success "Systemd services and timers running" - echo "" - - print_info "What you have now:" - echo " • Disk usage monitoring on selected hosts" - echo " • System healthcheck monitoring" - echo " • CPU temperature monitoring (if nodito configured)" - echo " • All organized in host-specific groups" - echo "" - - print_info "Verify your monitoring:" - echo " 1. Open Uptime Kuma web UI" - echo " 2. Check monitors organized by host groups" - echo " 3. Verify monitors are receiving data" - echo " 4. Configure notification rules" - echo " 5. Watch for alerts via ntfy" - echo "" - - print_info "Next steps:" - echo " 1. Customize thresholds if needed" - echo " 2. Proceed to Layer 7: Core Services deployment" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "📊 Layer 6: Infrastructure Monitoring" - - echo "This script will deploy automated monitoring for your infrastructure." - echo "" - - if ! confirm_action "Continue with Layer 6 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_prerequisites - check_uptime_kuma_credentials - - # Deploy monitoring - deploy_disk_usage_monitoring - echo "" - deploy_system_healthcheck - echo "" - deploy_cpu_temp_monitoring - - echo "" - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_7_services.sh b/scripts/setup_layer_7_services.sh deleted file mode 100755 index 27c3c8d..0000000 --- a/scripts/setup_layer_7_services.sh +++ /dev/null @@ -1,524 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 7: Core Services -# -# This script deploys Vaultwarden, Forgejo, and LNBits on vipy. -# Must be run after Layers 0, 1A, 2, and 3 are complete. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project root directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -############################################################################### -# Helper Functions -############################################################################### - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -############################################################################### -# Verification Functions -############################################################################### - -check_prerequisites() { - print_header "Verifying Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - # Check if vipy is configured - if [ -z "$(get_hosts_from_inventory "vipy")" ]; then - print_error "vipy not configured in inventory.ini" - print_info "Layer 7 requires vipy VPS" - ((errors++)) - else - print_success "vipy configured in inventory" - fi - - if [ $errors -gt 0 ]; then - print_error "Prerequisites not met" - exit 1 - fi - - print_success "Prerequisites verified" -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -get_host_ip() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -hostvars = data.get('_meta', {}).get('hostvars', {}) -if target in hostvars: - print(hostvars[target].get('ansible_host', target)) -else: - hosts = data.get(target, {}).get('hosts', []) - if hosts: - first = hosts[0] - hv = hostvars.get(first, {}) - print(hv.get('ansible_host', first)) -PY -} - -check_dns_configuration() { - print_header "Validating DNS Configuration" - - cd "$ANSIBLE_DIR" - - # Get vipy IP - local vipy_ip=$(get_host_ip "vipy") - - if [ -z "$vipy_ip" ]; then - print_error "Could not determine vipy IP from inventory" - return 1 - fi - - print_info "Vipy IP: $vipy_ip" - echo "" - - # Get domain from infra_vars.yml - local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) - - if [ -z "$root_domain" ]; then - print_error "Could not determine root_domain from infra_vars.yml" - return 1 - fi - - # Get subdomains from centralized config - local vw_subdomain="vault" - local fg_subdomain="git" - local ln_subdomain="lnbits" - - if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then - vw_subdomain=$(grep "^ vaultwarden:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "vault") - fg_subdomain=$(grep "^ forgejo:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "git") - ln_subdomain=$(grep "^ lnbits:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "lnbits") - fi - - print_info "Checking DNS records..." - echo "" - - local dns_ok=true - - if command -v dig &> /dev/null; then - # Check each subdomain - for service in "vaultwarden:$vw_subdomain" "forgejo:$fg_subdomain" "lnbits:$ln_subdomain"; do - local name=$(echo "$service" | cut -d: -f1) - local subdomain=$(echo "$service" | cut -d: -f2) - local fqdn="${subdomain}.${root_domain}" - - print_info "Checking $fqdn..." - local resolved=$(dig +short "$fqdn" | head -n1) - - if [ "$resolved" = "$vipy_ip" ]; then - print_success "$fqdn → $resolved ✓" - elif [ -n "$resolved" ]; then - print_error "$fqdn → $resolved (expected $vipy_ip)" - dns_ok=false - else - print_error "$fqdn does not resolve" - dns_ok=false - fi - done - else - print_warning "dig command not found, skipping DNS validation" - print_info "Install dnsutils/bind-tools to enable DNS validation" - return 1 - fi - - echo "" - - if [ "$dns_ok" = false ]; then - print_error "DNS validation failed" - print_info "Please configure DNS records for all services" - echo "" - print_warning "DNS changes can take time to propagate" - echo "" - if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then - exit 1 - fi - else - print_success "DNS validation passed" - fi -} - -############################################################################### -# Service Deployment -############################################################################### - -deploy_vaultwarden() { - print_header "Deploying Vaultwarden (Password Manager)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Deploy Vaultwarden via Docker" - echo " • Configure Caddy reverse proxy" - echo " • Set up fail2ban protection" - echo " • Enable sign-ups (disable after first user)" - echo "" - - if ! confirm_action "Proceed with Vaultwarden deployment?"; then - print_warning "Skipped Vaultwarden deployment" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/vaultwarden/deploy_vaultwarden_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/vaultwarden/deploy_vaultwarden_playbook.yml; then - print_success "Vaultwarden deployed" - echo "" - print_warning "POST-DEPLOYMENT:" - echo " 1. Visit your Vaultwarden subdomain" - echo " 2. Create your first user account" - echo " 3. Run: ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml" - return 0 - else - print_error "Vaultwarden deployment failed" - return 0 - fi -} - -deploy_forgejo() { - print_header "Deploying Forgejo (Git Server)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Install Forgejo binary" - echo " • Create git user and directories" - echo " • Configure Caddy reverse proxy" - echo " • Enable SSH cloning on port 22" - echo "" - - if ! confirm_action "Proceed with Forgejo deployment?"; then - print_warning "Skipped Forgejo deployment" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/forgejo/deploy_forgejo_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/forgejo/deploy_forgejo_playbook.yml; then - print_success "Forgejo deployed" - echo "" - print_warning "POST-DEPLOYMENT:" - echo " 1. Visit your Forgejo subdomain" - echo " 2. Create admin account on first visit" - echo " 3. Add your SSH key for git cloning" - return 0 - else - print_error "Forgejo deployment failed" - return 0 - fi -} - -deploy_lnbits() { - print_header "Deploying LNBits (Lightning Wallet)" - - cd "$ANSIBLE_DIR" - - print_info "This will:" - echo " • Install system dependencies and uv (Python 3.12 tooling)" - echo " • Clone LNBits repository (version v1.3.1)" - echo " • Sync dependencies with uv targeting Python 3.12" - echo " • Configure with FakeWallet (testing)" - echo " • Create systemd service" - echo " • Configure Caddy reverse proxy" - echo "" - - if ! confirm_action "Proceed with LNBits deployment?"; then - print_warning "Skipped LNBits deployment" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/lnbits/deploy_lnbits_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/lnbits/deploy_lnbits_playbook.yml; then - print_success "LNBits deployed" - echo "" - print_warning "POST-DEPLOYMENT:" - echo " 1. Visit your LNBits subdomain" - echo " 2. Create superuser on first visit" - echo " 3. Configure real Lightning backend (FakeWallet is for testing only)" - echo " 4. Disable new user registration" - return 0 - else - print_error "LNBits deployment failed" - return 0 - fi -} - -############################################################################### -# Backup Configuration -############################################################################### - -setup_backups() { - print_header "Setting Up Backups (Optional)" - - cd "$ANSIBLE_DIR" - - print_info "Configure automated backups to lapy" - echo "" - - # Vaultwarden backup - if confirm_action "Set up Vaultwarden backup to lapy?"; then - print_info "Running: ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml" - if ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml; then - print_success "Vaultwarden backup configured" - else - print_error "Vaultwarden backup setup failed" - fi - echo "" - fi - - # LNBits backup - if confirm_action "Set up LNBits backup to lapy (GPG encrypted)?"; then - print_info "Running: ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml" - if ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml; then - print_success "LNBits backup configured" - else - print_error "LNBits backup setup failed" - fi - echo "" - fi - - print_warning "Forgejo backups are not automated - set up manually if needed" -} - -############################################################################### -# Verification -############################################################################### - -verify_services() { - print_header "Verifying Service Deployments" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local vipy_host=$(get_hosts_from_inventory "vipy") - - if [ -z "$vipy_host" ]; then - print_error "Could not determine vipy host" - return - fi - - print_info "Checking services on vipy ($vipy_host)..." - echo "" - - # Check Vaultwarden - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "docker ps | grep vaultwarden" &>/dev/null; then - print_success "Vaultwarden container running" - else - print_warning "Vaultwarden container not running (may not be deployed)" - fi - - # Check Forgejo - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "systemctl is-active forgejo" &>/dev/null; then - print_success "Forgejo service running" - else - print_warning "Forgejo service not running (may not be deployed)" - fi - - # Check LNBits - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "systemctl is-active lnbits" &>/dev/null; then - print_success "LNBits service running" - else - print_warning "LNBits service not running (may not be deployed)" - fi - - # Check Caddy configs - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "ls /etc/caddy/sites-enabled/*.conf 2>/dev/null" &>/dev/null; then - print_success "Caddy configs exist" - local configs=$(timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "ls /etc/caddy/sites-enabled/*.conf 2>/dev/null" | xargs -n1 basename) - print_info "Configured services:" - echo "$configs" | sed 's/^/ /' - else - print_warning "No Caddy configs found" - fi - - echo "" -} - -############################################################################### -# Summary -############################################################################### - -print_summary() { - print_header "Layer 7 Setup Complete! 🎉" - - echo "Summary of what was deployed:" - echo "" - print_success "Core services deployed on vipy" - echo "" - - print_warning "CRITICAL POST-DEPLOYMENT STEPS:" - echo "" - echo "For each service you deployed, you MUST:" - echo "" - - echo "1. Vaultwarden (if deployed):" - echo " • Visit web UI and create first user" - echo " • Disable sign-ups: ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml" - echo " • Optional: Set up backup" - echo "" - - echo "2. Forgejo (if deployed):" - echo " • Visit web UI and create admin account" - echo " • Add your SSH public key for git operations" - echo " • Test cloning: git clone git@.:username/repo.git" - echo "" - - echo "3. LNBits (if deployed):" - echo " • Visit web UI and create superuser" - echo " • Configure real Lightning backend (currently FakeWallet)" - echo " • Disable new user registration" - echo " • Optional: Set up encrypted backup" - echo "" - - print_info "Services are now accessible:" - echo " • Vaultwarden: https://." - echo " • Forgejo: https://." - echo " • LNBits: https://." - echo "" - - print_success "Uptime Kuma monitors automatically created:" - echo " • Check Uptime Kuma web UI" - echo " • Look in 'services' monitor group" - echo " • Monitors for Vaultwarden, Forgejo, LNBits should appear" - echo "" - - print_info "Next steps:" - echo " 1. Complete post-deployment steps above" - echo " 2. Test each service" - echo " 3. Check Uptime Kuma monitors are working" - echo " 4. Proceed to Layer 8: ./scripts/setup_layer_8_secondary_services.sh" - echo "" -} - -############################################################################### -# Main Execution -############################################################################### - -main() { - clear - - print_header "🚀 Layer 7: Core Services" - - echo "This script will deploy core services on vipy:" - echo " • Vaultwarden (password manager)" - echo " • Forgejo (git server)" - echo " • LNBits (Lightning wallet)" - echo "" - - if ! confirm_action "Continue with Layer 7 setup?"; then - echo "Setup cancelled." - exit 0 - fi - - check_prerequisites - check_dns_configuration - - # Deploy services - deploy_vaultwarden - echo "" - deploy_forgejo - echo "" - deploy_lnbits - - echo "" - verify_services - - echo "" - setup_backups - - print_summary -} - -# Run main function -main "$@" - diff --git a/scripts/setup_layer_8_secondary_services.sh b/scripts/setup_layer_8_secondary_services.sh deleted file mode 100755 index fccaad8..0000000 --- a/scripts/setup_layer_8_secondary_services.sh +++ /dev/null @@ -1,384 +0,0 @@ -#!/bin/bash - -############################################################################### -# Layer 8: Secondary Services -# -# This script deploys the ntfy-emergency-app and memos services. -# Must be run after Layers 0-7 are complete. -############################################################################### - -set -e # Exit on error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Project directories -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ANSIBLE_DIR="$PROJECT_ROOT/ansible" - -declare -a LAYER_SUMMARY=() - -print_header() { - echo -e "\n${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}\n" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -confirm_action() { - local prompt="$1" - local response - - read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response - [[ "$response" =~ ^[Yy]$ ]] -} - -record_summary() { - LAYER_SUMMARY+=("$1") -} - -get_hosts_from_inventory() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -if target in data: - print(' '.join(data[target].get('hosts', []))) -else: - hostvars = data.get('_meta', {}).get('hostvars', {}) - if target in hostvars: - print(target) -PY -} - -get_primary_host_ip() { - local target="$1" - cd "$ANSIBLE_DIR" - ansible-inventory -i inventory.ini --list | \ - python3 - "$target" <<'PY' 2>/dev/null || echo "" -import json, sys -data = json.load(sys.stdin) -target = sys.argv[1] -hostvars = data.get('_meta', {}).get('hostvars', {}) -if target in hostvars: - print(hostvars[target].get('ansible_host', target)) -else: - hosts = data.get(target, {}).get('hosts', []) - if hosts: - first = hosts[0] - hv = hostvars.get(first, {}) - print(hv.get('ansible_host', first)) -PY -} - -check_prerequisites() { - print_header "Verifying Prerequisites" - - local errors=0 - - if [ -z "$VIRTUAL_ENV" ]; then - print_error "Virtual environment not activated" - echo "Run: source venv/bin/activate" - ((errors++)) - else - print_success "Virtual environment activated" - fi - - if ! command -v ansible &> /dev/null; then - print_error "Ansible not found" - ((errors++)) - else - print_success "Ansible found" - fi - - if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then - print_error "inventory.ini not found" - ((errors++)) - else - print_success "inventory.ini exists" - fi - - if [ ! -f "$ANSIBLE_DIR/infra_vars.yml" ]; then - print_error "infra_vars.yml not found" - ((errors++)) - else - print_success "infra_vars.yml exists" - fi - - if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then - print_error "services_config.yml not found" - ((errors++)) - else - print_success "services_config.yml exists" - fi - - if [ -z "$(get_hosts_from_inventory "vipy")" ]; then - print_error "vipy not configured in inventory.ini" - ((errors++)) - else - print_success "vipy configured in inventory" - fi - - if [ -z "$(get_hosts_from_inventory "memos-box")" ]; then - print_warning "memos-box not configured in inventory.ini (memos deployment will be skipped)" - else - print_success "memos-box configured in inventory" - fi - - if [ $errors -gt 0 ]; then - print_error "Prerequisites not met. Resolve the issues above and re-run the script." - exit 1 - fi - - print_success "Prerequisites verified" - - # Display configured subdomains - local emergency_subdomain=$(grep "^ ntfy_emergency_app:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "emergency") - local memos_subdomain=$(grep "^ memos:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "memos") - - print_info "Configured subdomains:" - echo " • ntfy_emergency_app: $emergency_subdomain" - echo " • memos: $memos_subdomain" - echo "" -} - -check_dns_configuration() { - print_header "Validating DNS Configuration" - - if ! command -v dig &> /dev/null; then - print_warning "dig command not found. Skipping DNS validation." - print_info "Install dnsutils/bind-tools to enable DNS validation." - return 0 - fi - - cd "$ANSIBLE_DIR" - - local root_domain - root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) - - if [ -z "$root_domain" ]; then - print_error "Could not determine root_domain from infra_vars.yml" - return 1 - fi - - local emergency_subdomain=$(grep "^ ntfy_emergency_app:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "emergency") - local memos_subdomain=$(grep "^ memos:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "memos") - - local vipy_ip - vipy_ip=$(get_primary_host_ip "vipy") - - if [ -z "$vipy_ip" ]; then - print_error "Unable to determine vipy IP from inventory" - return 1 - fi - - local memos_ip="" - local memos_host=$(get_hosts_from_inventory "memos-box") - if [ -n "$memos_host" ]; then - memos_ip=$(get_primary_host_ip "$memos_host") - fi - - local dns_ok=true - - local emergency_fqdn="${emergency_subdomain}.${root_domain}" - local memos_fqdn="${memos_subdomain}.${root_domain}" - - print_info "Expected DNS:" - echo " • $emergency_fqdn → $vipy_ip" - if [ -n "$memos_ip" ]; then - echo " • $memos_fqdn → $memos_ip" - else - echo " • $memos_fqdn → (skipped - memos-box not in inventory)" - fi - echo "" - - local resolved - - print_info "Checking $emergency_fqdn..." - resolved=$(dig +short "$emergency_fqdn" | head -n1) - if [ "$resolved" = "$vipy_ip" ]; then - print_success "$emergency_fqdn resolves to $resolved" - elif [ -n "$resolved" ]; then - print_error "$emergency_fqdn resolves to $resolved (expected $vipy_ip)" - dns_ok=false - else - print_error "$emergency_fqdn does not resolve" - dns_ok=false - fi - - if [ -n "$memos_ip" ]; then - print_info "Checking $memos_fqdn..." - resolved=$(dig +short "$memos_fqdn" | head -n1) - if [ "$resolved" = "$memos_ip" ]; then - print_success "$memos_fqdn resolves to $resolved" - elif [ -n "$resolved" ]; then - print_error "$memos_fqdn resolves to $resolved (expected $memos_ip)" - dns_ok=false - else - print_error "$memos_fqdn does not resolve" - dns_ok=false - fi - fi - - echo "" - - if [ "$dns_ok" = false ]; then - print_error "DNS validation failed." - print_info "Update DNS records as shown above and wait for propagation." - echo "" - if ! confirm_action "Continue anyway? (SSL certificates will fail without correct DNS)"; then - exit 1 - fi - else - print_success "DNS validation passed" - fi -} - -deploy_ntfy_emergency_app() { - print_header "Deploying ntfy-emergency-app" - - cd "$ANSIBLE_DIR" - - print_info "This deploys the emergency notification interface pointing at ntfy." - echo "" - - if ! confirm_action "Deploy / update the ntfy-emergency-app?"; then - print_warning "Skipped ntfy-emergency-app deployment" - record_summary "${YELLOW}• ntfy-emergency-app${NC}: skipped" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml; then - print_success "ntfy-emergency-app deployed successfully" - record_summary "${GREEN}• ntfy-emergency-app${NC}: deployed" - else - print_error "ntfy-emergency-app deployment failed" - record_summary "${RED}• ntfy-emergency-app${NC}: failed" - fi -} - -deploy_memos() { - print_header "Deploying Memos" - - if [ -z "$(get_hosts_from_inventory "memos-box")" ]; then - print_warning "memos-box not in inventory. Skipping memos deployment." - record_summary "${YELLOW}• memos${NC}: skipped (memos-box missing)" - return 0 - fi - - cd "$ANSIBLE_DIR" - - if ! confirm_action "Deploy / update memos on memos-box?"; then - print_warning "Skipped memos deployment" - record_summary "${YELLOW}• memos${NC}: skipped" - return 0 - fi - - print_info "Running: ansible-playbook -i inventory.ini services/memos/deploy_memos_playbook.yml" - echo "" - - if ansible-playbook -i inventory.ini services/memos/deploy_memos_playbook.yml; then - print_success "Memos deployed successfully" - record_summary "${GREEN}• memos${NC}: deployed" - else - print_error "Memos deployment failed" - record_summary "${RED}• memos${NC}: failed" - fi -} - -verify_services() { - print_header "Verifying Deployments" - - cd "$ANSIBLE_DIR" - - local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') - ssh_key="${ssh_key/#\~/$HOME}" - - local vipy_host - vipy_host=$(get_hosts_from_inventory "vipy") - - if [ -n "$vipy_host" ]; then - print_info "Checking services on vipy ($vipy_host)..." - - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "docker ps | grep ntfy-emergency-app" &>/dev/null; then - print_success "ntfy-emergency-app container running" - else - print_warning "ntfy-emergency-app container not running" - fi - - echo "" - fi - - local memos_host - memos_host=$(get_hosts_from_inventory "memos-box") - if [ -n "$memos_host" ]; then - print_info "Checking memos on memos-box ($memos_host)..." - if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$memos_host "systemctl is-active memos" &>/dev/null; then - print_success "memos systemd service running" - else - print_warning "memos systemd service not running" - fi - echo "" - fi -} - -print_summary() { - print_header "Layer 8 Summary" - - if [ ${#LAYER_SUMMARY[@]} -eq 0 ]; then - print_info "No actions were performed." - return - fi - - for entry in "${LAYER_SUMMARY[@]}"; do - echo -e "$entry" - done - - echo "" - print_info "Next steps:" - echo " • Visit each service's subdomain to complete any manual setup." - echo " • Configure backups for new services if applicable." - echo " • Update Uptime Kuma monitors if additional endpoints are desired." -} - -main() { - print_header "Layer 8: Secondary Services" - - check_prerequisites - check_dns_configuration - - deploy_ntfy_emergency_app - deploy_memos - - verify_services - print_summary -} - -main "$@" - From 0b578ee738d444124e96667846c1136c41cd8f7b Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 8 Dec 2025 10:34:04 +0100 Subject: [PATCH 02/13] stuff --- .../01_user_and_access_setup_playbook.yml | 4 +- .../02_firewall_and_fail2ban_playbook.yml | 4 +- ansible/infra/920_join_headscale_mesh.yml | 7 +- ansible/infra_secrets.yml.example | 3 + .../bitcoin-knots/bitcoin_knots_vars.yml | 32 + .../deploy_bitcoin_knots_playbook.yml | 734 ++++++++++++++ human_script.md | 897 ------------------ tofu/nodito/README.md | 7 - tofu/nodito/main.tf | 10 - tofu/nodito/terraform.tfvars.example | 5 - tofu/nodito/variables.tf | 5 - 11 files changed, 779 insertions(+), 929 deletions(-) create mode 100644 ansible/services/bitcoin-knots/bitcoin_knots_vars.yml create mode 100644 ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml delete mode 100644 human_script.md diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index a86772c..13e5149 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,5 @@ -- name: Secure Debian VPS - hosts: vps +- name: Secure Debian + hosts: all vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index b50a0e3..e83cbcb 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,5 @@ -- name: Secure Debian VPS - hosts: vps +- name: Secure Debian + hosts: all vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 425036c..cd6464c 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -99,7 +99,6 @@ --login-server {{ headscale_domain }} --authkey {{ auth_key }} --accept-dns=true - --advertise-tags "tag:{{ inventory_hostname }}" register: tailscale_up_result changed_when: "'already authenticated' not in tailscale_up_result.stdout" failed_when: tailscale_up_result.rc != 0 and 'already authenticated' not in tailscale_up_result.stdout @@ -117,3 +116,9 @@ debug: msg: "{{ tailscale_status.stdout_lines }}" + - name: Deny all inbound traffic from Tailscale network interface + ufw: + rule: deny + direction: in + interface: tailscale0 + diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index 80b740b..2482160 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -19,3 +19,6 @@ ntfy_password: "your_ntfy_password" headscale_ui_username: "admin" headscale_ui_password: "your_secure_password_here" # headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password + +bitcoin_rpc_user: "bitcoinrpc" +bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml new file mode 100644 index 0000000..fc38c75 --- /dev/null +++ b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml @@ -0,0 +1,32 @@ +# Bitcoin Knots Configuration Variables + +# Version - REQUIRED: Specify exact version/tag to build +bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag +bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) + +# Directories +bitcoin_knots_dir: /opt/bitcoin-knots +bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" +bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets +bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) +bitcoin_conf_dir: /etc/bitcoin + +# Network +bitcoin_rpc_port: 8332 +bitcoin_p2p_port: 8333 +bitcoin_rpc_bind: "127.0.0.1" # Security: localhost only +bitcoin_tailscale_interface: tailscale0 # Tailscale interface for UFW rules + +# Build options +bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores +bitcoin_build_prefix: /usr/local + +# Configuration options +bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) +bitcoin_enable_prune: false # Set to prune amount (e.g., 550) to enable pruning, false for full node (MUST be false for Electrum servers) +bitcoin_max_connections: 125 +# dbcache will be calculated as 90% of host RAM automatically in playbook + +# Service user +bitcoin_user: bitcoin +bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml new file mode 100644 index 0000000..2c7cebb --- /dev/null +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -0,0 +1,734 @@ +- name: Build and Deploy Bitcoin Knots from Source + hosts: knots_box_local + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml + vars: + bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" + bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" + bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" + bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Calculate 90% of system RAM for dbcache + set_fact: + bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" + changed_when: false + + - name: Display calculated dbcache value + debug: + msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" + + + - name: Install build dependencies + apt: + name: + - build-essential + - libtool + - autotools-dev + - automake + - pkg-config + - bsdmainutils + - python3 + - python3-pip + - libevent-dev + - libboost-system-dev + - libboost-filesystem-dev + - libboost-test-dev + - libboost-thread-dev + - libboost-chrono-dev + - libboost-program-options-dev + - libboost-dev + - libssl-dev + - libdb-dev + - libminiupnpc-dev + - libzmq3-dev + - libnatpmp-dev + - libsqlite3-dev + - git + - curl + - wget + - cmake + state: present + update_cache: yes + + - name: Create bitcoin group + group: + name: "{{ bitcoin_group }}" + system: yes + state: present + + - name: Create bitcoin user + user: + name: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ bitcoin_data_dir }}" + create_home: yes + state: present + + - name: Create bitcoin-knots directory + file: + path: "{{ bitcoin_knots_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin-knots source directory + file: + path: "{{ bitcoin_knots_source_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create bitcoin data directory (for config, logs, wallets) + file: + path: "{{ bitcoin_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin large data directory (for blockchain) + file: + path: "{{ bitcoin_large_data_dir }}" + state: directory + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0750' + + - name: Create bitcoin config directory + file: + path: "{{ bitcoin_conf_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Check if bitcoind binary already exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_binary_exists + changed_when: false + + - name: Install gnupg for signature verification + apt: + name: gnupg + state: present + when: not bitcoind_binary_exists.stat.exists + + - name: Import Luke Dashjr's Bitcoin Knots signing key + command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 + register: key_import + changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" + when: not bitcoind_binary_exists.stat.exists + failed_when: key_import.rc != 0 + + - name: Display imported key fingerprint + command: gpg --fingerprint 90C8019E36C2E964 + register: key_fingerprint + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + + - name: Download SHA256SUMS file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Download SHA256SUMS.asc signature file + get_url: + url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" + dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" + mode: '0644' + when: not bitcoind_binary_exists.stat.exists + + - name: Verify PGP signature on SHA256SUMS file + command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS + register: sha256sums_verification + changed_when: false + failed_when: sha256sums_verification.rc != 0 + when: not bitcoind_binary_exists.stat.exists + + + - name: Display SHA256SUMS verification result + debug: + msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Fail if SHA256SUMS signature verification failed + fail: + msg: "SHA256SUMS signature verification failed. Aborting build." + when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) + + - name: Remove any existing tarball to force fresh download + file: + path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Download Bitcoin Knots source tarball + get_url: + url: "{{ bitcoin_source_tarball_url }}" + dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" + mode: '0644' + validate_certs: yes + force: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Calculate SHA256 checksum of downloaded tarball + command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + register: tarball_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Extract expected checksum from SHA256SUMS file + shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' + register: expected_checksum + changed_when: false + when: not bitcoind_binary_exists.stat.exists + failed_when: expected_checksum.stdout == "" + + - name: Display checksum comparison + debug: + msg: + - "Expected: {{ expected_checksum.stdout | trim }}" + - "Actual: {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists + + - name: Verify tarball checksum matches SHA256SUMS + fail: + msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" + when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] + + - name: Remove existing source directory if it exists (to force fresh extraction) + file: + path: "{{ bitcoin_knots_source_dir }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Remove extracted directory if it exists (from previous runs) + file: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + state: absent + when: not bitcoind_binary_exists.stat.exists + + - name: Extract verified source tarball + unarchive: + src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz + dest: "{{ bitcoin_knots_dir }}" + remote_src: yes + when: not bitcoind_binary_exists.stat.exists + + - name: Check if extracted directory exists + stat: + path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" + register: extracted_dir_stat + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Rename extracted directory to expected name + command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" + when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists + + - name: Check if CMakeLists.txt exists + stat: + path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" + register: cmake_exists + changed_when: false + when: not bitcoind_binary_exists.stat.exists + + - name: Create CMake build directory + file: + path: "{{ bitcoin_knots_source_dir }}/build" + state: directory + mode: '0755' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + + - name: Configure Bitcoin Knots build with CMake + command: > + cmake + -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} + -DBUILD_BITCOIN_WALLET=OFF + -DCMAKE_BUILD_TYPE=Release + .. + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: configure_result + changed_when: true + + - name: Fail if CMakeLists.txt not found + fail: + msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." + when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) + + - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) + command: cmake --build . -j{{ bitcoin_build_jobs }} + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + async: 3600 + poll: 0 + register: build_result + changed_when: true + + - name: Check build status + async_status: + jid: "{{ build_result.ansible_job_id }}" + register: build_job_result + until: build_job_result.finished + retries: 120 + delay: 60 + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined + + - name: Fail if build failed + fail: + msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" + when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) + + - name: Install Bitcoin Knots binaries + command: cmake --install . + args: + chdir: "{{ bitcoin_knots_source_dir }}/build" + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + changed_when: true + + - name: Verify bitcoind binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoind" + register: bitcoind_installed + changed_when: false + + - name: Verify bitcoin-cli binary exists + stat: + path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" + register: bitcoin_cli_installed + changed_when: false + + - name: Fail if binaries not found + fail: + msg: "Bitcoin Knots binaries not found after installation" + when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists + + - name: Create bitcoin.conf configuration file + copy: + dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" + content: | + # Bitcoin Knots Configuration + # Generated by Ansible + + # Data directory (blockchain storage) + datadir={{ bitcoin_large_data_dir }} + + # RPC Configuration + server=1 + rpcuser={{ bitcoin_rpc_user }} + rpcpassword={{ bitcoin_rpc_password }} + rpcbind={{ bitcoin_rpc_bind }} + rpcport={{ bitcoin_rpc_port }} + rpcallowip=127.0.0.1 + + # Network Configuration + listen=1 + port={{ bitcoin_p2p_port }} + maxconnections={{ bitcoin_max_connections }} + + # Performance + dbcache={{ bitcoin_dbcache_mb }} + + # Transaction Index (optional) + {% if bitcoin_enable_txindex %} + txindex=1 + {% endif %} + + # Pruning (optional) + {% if bitcoin_enable_prune %} + prune={{ bitcoin_enable_prune }} + {% endif %} + + # Logging + logtimestamps=1 + logfile={{ bitcoin_data_dir }}/debug.log + + # Security + disablewallet=1 + owner: "{{ bitcoin_user }}" + group: "{{ bitcoin_group }}" + mode: '0640' + notify: Restart bitcoind + + - name: Create systemd service file for bitcoind + copy: + dest: /etc/systemd/system/bitcoind.service + content: | + [Unit] + Description=Bitcoin Knots daemon + After=network.target + + [Service] + Type=simple + User={{ bitcoin_user }} + Group={{ bitcoin_group }} + ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf + Restart=always + RestartSec=10 + TimeoutStopSec=600 + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart bitcoind + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start bitcoind service + systemd: + name: bitcoind + enabled: yes + state: started + + - name: Wait for bitcoind RPC to be available + uri: + url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" + method: POST + body_format: json + body: + jsonrpc: "1.0" + id: "healthcheck" + method: "getblockchaininfo" + params: [] + user: "{{ bitcoin_rpc_user }}" + password: "{{ bitcoin_rpc_password }}" + status_code: 200 + timeout: 10 + register: rpc_check + until: rpc_check.status == 200 + retries: 30 + delay: 5 + ignore_errors: yes + + - name: Display RPC connection status + debug: + msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" + + - name: Allow Bitcoin P2P port on Tailscale interface only + ufw: + rule: allow + direction: in + port: "{{ bitcoin_p2p_port }}" + proto: tcp + interface: "{{ bitcoin_tailscale_interface }}" + comment: "Bitcoin Knots P2P (Tailscale only)" + + - name: Allow Bitcoin P2P port (UDP) on Tailscale interface only + ufw: + rule: allow + direction: in + port: "{{ bitcoin_p2p_port }}" + proto: udp + interface: "{{ bitcoin_tailscale_interface }}" + comment: "Bitcoin Knots P2P UDP (Tailscale only)" + + - name: Verify UFW rules for Bitcoin Knots + command: ufw status numbered + register: ufw_status + changed_when: false + + - name: Display UFW status + debug: + msg: "{{ ufw_status.stdout_lines }}" + + - name: Create Bitcoin Knots health check and push script + copy: + dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh + content: | + #!/bin/bash + # + # Bitcoin Knots Health Check and Push to Uptime Kuma + # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma + # + + RPC_HOST="{{ bitcoin_rpc_bind }}" + RPC_PORT={{ bitcoin_rpc_port }} + RPC_USER="{{ bitcoin_rpc_user }}" + RPC_PASSWORD="{{ bitcoin_rpc_password }}" + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if bitcoind RPC is responding + check_bitcoind() { + local response + response=$(curl -s --max-time 5 \ + --user "${RPC_USER}:${RPC_PASSWORD}" \ + --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ + --header 'Content-Type: application/json' \ + "http://${RPC_HOST}:${RPC_PORT}" 2>&1) + + if [ $? -eq 0 ]; then + # Check if response contains error + if echo "$response" | grep -q '"error"'; then + return 1 + else + return 0 + fi + else + return 1 + fi + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode the message + local encoded_msg=$(echo -n "$msg" | curl -Gso /dev/null -w %{url_effective} --data-urlencode "msg=$msg" "" | cut -c 3-) + + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping=" || true + } + + # Main health check + if check_bitcoind; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "bitcoind RPC not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Install curl for health check script + apt: + name: curl + state: present + + - name: Create systemd timer for Bitcoin Knots health check + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer + content: | + [Unit] + Description=Bitcoin Knots Health Check Timer + Requires=bitcoind.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Bitcoin Knots health check + copy: + dest: /etc/systemd/system/bitcoin-knots-healthcheck.service + content: | + [Unit] + Description=Bitcoin Knots Health Check and Push to Uptime Kuma + After=network.target bitcoind.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Bitcoin Knots health check timer + systemd: + name: bitcoin-knots-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_knots_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + # Get push URL from existing monitor + push_id = existing_monitor.get('push_token', existing_monitor.get('id')) + push_url = f"{url}/api/push/{push_id}" + print(f"Push URL: {push_url}") + print("Skipping - monitor already configured") + else: + print(f"Creating push monitor '{monitor_name}'...") + result = api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + # Get push URL from created monitor + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_id = new_monitor.get('push_token', new_monitor.get('id')) + push_url = f"{url}/api/push/{push_id}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Bitcoin Knots" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_bitcoin_knots_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/bitcoin-knots-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: bitcoin-knots-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_knots_monitor.py + - /tmp/ansible_config.yml + + handlers: + - name: Restart bitcoind + systemd: + name: bitcoind + state: restarted + diff --git a/human_script.md b/human_script.md deleted file mode 100644 index 345c6a7..0000000 --- a/human_script.md +++ /dev/null @@ -1,897 +0,0 @@ -# Personal Infrastructure Setup Guide - -This guide walks you through setting up your complete personal infrastructure, layer by layer. Each layer must be completed before moving to the next one. - -**Automated Setup:** Each layer has a bash script that handles the setup process. The scripts will: -- Check prerequisites -- Prompt for required variables -- Set up configuration files -- Execute playbooks -- Verify completion - -## Prerequisites - -Before starting: -- You have a domain name -- You have VPS accounts ready -- You have nodito ready with Proxmox installed, ssh key in place -- You have SSH access to all machines -- You're running this from your laptop (lapy) - ---- - -## Layer 0: Foundation Setup - -**Goal:** Set up your laptop (lapy) as the Ansible control node and configure basic settings. - -**Script:** `./scripts/setup_layer_0.sh` - -### What This Layer Does: -1. Creates Python virtual environment -2. Installs Ansible and required Python packages -3. Installs Ansible Galaxy collections -4. Guides you through creating `inventory.ini` with your machine IPs -5. Guides you through creating `infra_vars.yml` with your domain -6. Creates `services_config.yml` with centralized subdomain settings -7. Creates `infra_secrets.yml` template for Uptime Kuma credentials -8. Validates SSH keys exist -9. Verifies everything is ready for Layer 1 - -### Required Information: -- Your domain name (e.g., `contrapeso.xyz`) -- SSH key path (default: `~/.ssh/counterganzua`) -- IP addresses for your infrastructure: - - vipy (main VPS) - - watchtower (monitoring VPS) - - spacey (headscale VPS) - - nodito (Proxmox server) - optional - - **Note:** VMs (like memos-box) will be created later on Proxmox and added to the `nodito_vms` group - -### Manual Steps: -After running the script, you'll need to: -1. Ensure your SSH key is added to all VPS root users (usually done by VPS provider) -2. Ensure DNS is configured for your domain (nameservers pointing to your DNS provider) - -### Centralized Configuration: - -The script creates `ansible/services_config.yml` which contains all service subdomains in one place: -- Easy to review all subdomains at a glance -- No need to edit multiple vars files -- Consistent Caddy settings across all services -- **Edit this file to customize your subdomains before deploying services** - -### Verification: -The script will verify: -- ✓ Python venv exists and activated -- ✓ Ansible installed -- ✓ Required Python packages installed -- ✓ Ansible Galaxy collections installed -- ✓ `inventory.ini` exists and formatted correctly -- ✓ `infra_vars.yml` exists with domain configured -- ✓ `services_config.yml` created with subdomain settings -- ✓ `infra_secrets.yml` template created -- ✓ SSH key file exists - -### Run the Script: -```bash -cd /home/counterweight/personal_infra -./scripts/setup_layer_0.sh -``` - ---- - -## Layer 1A: VPS Basic Setup - -**Goal:** Configure users, SSH access, firewall, and fail2ban on VPS machines. - -**Script:** `./scripts/setup_layer_1a_vps.sh` - -**Can be run independently** - doesn't require Nodito setup. - -### What This Layer Does: - -For VPSs (vipy, watchtower, spacey): -1. Creates the `counterweight` user with sudo access -2. Configures SSH key authentication -3. Disables root login (by design for security) -4. Sets up UFW firewall with SSH access -5. Installs and configures fail2ban -6. Installs and configures auditd for security logging - -### Prerequisites: -- ✅ Layer 0 complete -- ✅ SSH key added to all VPS root users -- ✅ Root access to VPSs - -### Verification: -The script will verify: -- ✓ Can SSH to all VPSs as root -- ✓ VPS playbooks complete successfully -- ✓ Can SSH to all VPSs as `counterweight` user -- ✓ Firewall is active and configured -- ✓ fail2ban is running - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_1a_vps.sh -``` - -**Note:** After this layer, you will no longer be able to SSH as root to VPSs (by design for security). - ---- - -## Layer 1B: Nodito (Proxmox) Setup - -**Goal:** Configure the Nodito Proxmox server. - -**Script:** `./scripts/setup_layer_1b_nodito.sh` - -**Can be run independently** - doesn't require VPS setup. - -### What This Layer Does: - -For Nodito (Proxmox server): -1. Bootstraps SSH key access for root -2. Creates the `counterweight` user -3. Updates and secures the system -4. Disables root login and password authentication -5. Switches to Proxmox community repositories -6. Optionally sets up ZFS storage pool (if disks configured) -7. Optionally creates Debian cloud template - -### Prerequisites: -- ✅ Layer 0 complete -- ✅ Root password for nodito -- ✅ Nodito configured in inventory.ini - -### Optional: ZFS Setup -For ZFS storage pool (optional): -1. SSH into nodito: `ssh root@` -2. List disk IDs: `ls -la /dev/disk/by-id/ | grep -E "(ata-|scsi-|nvme-)"` -3. Note the disk IDs you want to use -4. The script will help you create `ansible/infra/nodito/nodito_vars.yml` with disk configuration - -⚠️ **Warning:** ZFS setup will DESTROY ALL DATA on specified disks! - -### Verification: -The script will verify: -- ✓ Nodito bootstrap successful -- ✓ Community repos configured -- ✓ Can SSH to nodito as `counterweight` user - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_1b_nodito.sh -``` - -**Note:** After this layer, you will no longer be able to SSH as root to nodito (by design for security). - ---- - -## Layer 2: General Infrastructure Tools - -**Goal:** Install common utilities needed by various services. - -**Script:** `./scripts/setup_layer_2.sh` - -### What This Layer Does: - -Installs essential tools on machines that need them: - -#### rsync -- **Purpose:** Required for backup operations -- **Deployed to:** vipy, watchtower, lapy (and optionally other hosts) -- **Playbook:** `infra/900_install_rsync.yml` - -#### Docker + Docker Compose -- **Purpose:** Required for containerized services -- **Deployed to:** vipy, watchtower (and optionally other hosts) -- **Playbook:** `infra/910_docker_playbook.yml` - -### Prerequisites: -- ✅ Layer 0 complete -- ✅ Layer 1A complete (for VPSs) OR Layer 1B complete (for nodito) -- ✅ SSH access as counterweight user - -### Services That Need These Tools: -- **rsync:** All backup operations (Uptime Kuma, Vaultwarden, LNBits, etc.) -- **docker:** Uptime Kuma, Vaultwarden, ntfy-emergency-app - -### Verification: -The script will verify: -- ✓ rsync installed on specified hosts -- ✓ Docker and Docker Compose installed on specified hosts -- ✓ counterweight user added to docker group -- ✓ Docker service running - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_2.sh -``` - -**Note:** This script is interactive and will let you choose which hosts get which tools. - ---- - -## Layer 3: Reverse Proxy (Caddy) - -**Goal:** Deploy Caddy reverse proxy for HTTPS termination and routing. - -**Script:** `./scripts/setup_layer_3_caddy.sh` - -### What This Layer Does: - -Installs and configures Caddy web server on VPS machines: -- Installs Caddy from official repositories -- Configures Caddy to listen on ports 80/443 -- Opens firewall ports for HTTP/HTTPS -- Creates `/etc/caddy/sites-enabled/` directory structure -- Sets up automatic HTTPS with Let's Encrypt - -**Deployed to:** vipy, watchtower, spacey - -### Why Caddy is Critical: - -Caddy provides: -- **Automatic HTTPS** - Let's Encrypt certificates with auto-renewal -- **Reverse proxy** - Routes traffic to backend services -- **Simple configuration** - Each service adds its own config file -- **HTTP/2 support** - Modern protocol support - -### Prerequisites: -- ✅ Layer 0 complete -- ✅ Layer 1A complete (VPS setup) -- ✅ SSH access as counterweight user -- ✅ Ports 80/443 available on VPSs - -### Services That Need Caddy: -All web services depend on Caddy: -- Uptime Kuma (watchtower) -- ntfy (watchtower) -- Headscale (spacey) -- Vaultwarden (vipy) -- Forgejo (vipy) -- LNBits (vipy) -- ntfy-emergency-app (vipy) - -### Verification: -The script will verify: -- ✓ Caddy installed on all target hosts -- ✓ Caddy service running -- ✓ Ports 80/443 open in firewall -- ✓ Sites-enabled directory created -- ✓ Can reach Caddy default page - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_3_caddy.sh -``` - -**Note:** Caddy starts with an empty configuration. Services will add their own config files in later layers. - ---- - -## Layer 4: Core Monitoring & Notifications - -**Goal:** Deploy ntfy (notifications) and Uptime Kuma (monitoring platform). - -**Script:** `./scripts/setup_layer_4_monitoring.sh` - -### What This Layer Does: - -Deploys core monitoring infrastructure on watchtower: - -#### 4A: ntfy (Notification Service) -- Installs ntfy from official repositories -- Configures ntfy with authentication (deny-all by default) -- Creates admin user for sending notifications -- Sets up Caddy reverse proxy -- **Deployed to:** watchtower - -#### 4B: Uptime Kuma (Monitoring Platform) -- Deploys Uptime Kuma via Docker -- Configures Caddy reverse proxy -- Sets up data persistence -- Optionally sets up backup to lapy -- **Deployed to:** watchtower - -### Prerequisites (Complete BEFORE Running): - -**1. Previous layers complete:** -- ✅ Layer 0, 1A, 2, 3 complete (watchtower must be fully set up) -- ✅ Docker installed on watchtower (from Layer 2) -- ✅ Caddy running on watchtower (from Layer 3) - -**2. Configure subdomains (in centralized config):** -- ✅ Edit `ansible/services_config.yml` and customize subdomains under `subdomains:` section - - Set `ntfy:` to your preferred subdomain (e.g., `ntfy` or `notify`) - - Set `uptime_kuma:` to your preferred subdomain (e.g., `uptime` or `kuma`) - -**3. Create DNS records that match your configured subdomains:** -- ✅ Create A record: `.` → watchtower IP -- ✅ Create A record: `.` → watchtower IP -- ✅ Wait for DNS propagation (can take minutes to hours) -- ✅ Verify with: `dig .` should return watchtower IP - -**4. Prepare ntfy admin credentials:** -- ✅ Decide on username (default: `admin`) -- ✅ Decide on a secure password (script will prompt you) - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_4_monitoring.sh -``` - -The script will prompt you for ntfy admin credentials during deployment. - -### Post-Deployment Steps (Complete AFTER Running): - -**The script will guide you through most of these, but here's what happens:** - -#### Step 1: Set Up Uptime Kuma Admin Account (Manual) -1. Open browser and visit: `https://.` -2. On first visit, you'll see the setup page -3. Create admin username and password -4. Save these credentials securely - -#### Step 2: Update infra_secrets.yml (Manual) -1. Edit `ansible/infra_secrets.yml` -2. Add your Uptime Kuma credentials: - ```yaml - uptime_kuma_username: "your-admin-username" - uptime_kuma_password: "your-admin-password" - ``` -3. Save the file -4. **This is required for automated ntfy setup and Layer 6** - -#### Step 3: Configure ntfy Notification (Automated) -**The script will offer to do this automatically!** If you completed Steps 1 & 2, the script will: -- Connect to Uptime Kuma via API -- Create ntfy notification configuration -- Test the connection -- No manual UI configuration needed! - -**Alternatively (Manual):** -1. In Uptime Kuma web UI, go to **Settings** → **Notifications** -2. Click **Setup Notification**, choose **ntfy** -3. Configure with your ntfy subdomain and credentials - -#### Step 4: Final Verification (Automated) -**The script will automatically verify:** -- ✓ Uptime Kuma credentials in infra_secrets.yml -- ✓ Can connect to Uptime Kuma API -- ✓ ntfy notification is configured -- ✓ All post-deployment steps complete - -If anything is missing, the script will tell you exactly what to do! - -#### Step 5: Subscribe to Notifications on Your Phone (Optional - Manual) -1. Install ntfy app: https://github.com/binwiederhier/ntfy-android -2. Add subscription: - - Server: `https://.` - - Topic: `alerts` (same as configured in Uptime Kuma) - - Username: Your ntfy admin username - - Password: Your ntfy admin password -3. You'll now receive push notifications for all alerts! - -**Pro tip:** Run the script again after completing Steps 1 & 2, and it will automatically configure ntfy and verify everything! - -### Verification: -The script will automatically verify: -- ✓ DNS records are configured correctly (using `dig`) -- ✓ ntfy service running -- ✓ Uptime Kuma container running -- ✓ Caddy configs created for both services - -After post-deployment steps, you can test: -- Visit `https://.` (should load ntfy web UI) -- Visit `https://.` (should load Uptime Kuma) -- Send test notification in Uptime Kuma - -**Note:** DNS validation requires `dig` command. If not available, validation will be skipped (you can continue but SSL may fail). - -### Why This Layer is Critical: -- **All infrastructure monitoring** (Layer 6) depends on Uptime Kuma -- **All alerts** go through ntfy -- Services availability monitoring needs Uptime Kuma -- Without this layer, you won't know when things break! - ---- - -## Layer 5: VPN Infrastructure (Headscale) - -**Goal:** Deploy Headscale for secure mesh networking (like Tailscale, but self-hosted). - -**Script:** `./scripts/setup_layer_5_headscale.sh` - -**This layer is OPTIONAL** - Skip to Layer 6 if you don't need VPN mesh networking. - -### What This Layer Does: - -Deploys Headscale coordination server and optionally joins machines to the mesh: - -#### 5A: Deploy Headscale Server -- Installs Headscale on spacey -- Configures with deny-all ACL policy (you customize later) -- Creates namespace/user for your network -- Sets up Caddy reverse proxy -- Configures embedded DERP server for NAT traversal -- **Deployed to:** spacey - -#### 5B: Join Machines to Mesh (Optional) -- Installs Tailscale client on target machines -- Generates ephemeral pre-auth keys -- Automatically joins machines to your mesh -- Enables Magic DNS -- **Can join:** vipy, watchtower, nodito, lapy, etc. - -### Prerequisites (Complete BEFORE Running): - -**1. Previous layers complete:** -- ✅ Layer 0, 1A, 3 complete (spacey must be set up) -- ✅ Caddy running on spacey (from Layer 3) - -**2. Configure subdomain (in centralized config):** -- ✅ Edit `ansible/services_config.yml` and customize `headscale:` under `subdomains:` section (e.g., `headscale` or `vpn`) - -**3. Create DNS record that matches your configured subdomain:** -- ✅ Create A record: `.` → spacey IP -- ✅ Wait for DNS propagation -- ✅ Verify with: `dig .` should return spacey IP - -**4. Decide on namespace name:** -- ✅ Choose a namespace for your network (default: `counter-net`) -- ✅ This is set in `headscale_vars.yml` as `headscale_namespace` - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_5_headscale.sh -``` - -The script will: -1. Validate DNS configuration -2. Deploy Headscale server -3. Offer to join machines to the mesh - -### Post-Deployment Steps: - -#### Configure ACL Policies (Required for machines to communicate) -1. SSH into spacey: `ssh counterweight@` -2. Edit ACL file: `sudo nano /etc/headscale/acl.json` -3. Configure rules (example - allow all): - ```json - { - "ACLs": [ - {"action": "accept", "src": ["*"], "dst": ["*:*"]} - ] - } - ``` -4. Restart Headscale: `sudo systemctl restart headscale` - -**Default is deny-all for security** - you must configure ACLs for machines to talk! - -#### Join Additional Machines Manually -For machines not in inventory (mobile, desktop): -1. Install Tailscale client on device -2. Generate pre-auth key on spacey: - ```bash - ssh counterweight@ - sudo headscale preauthkeys create --user --reusable - ``` -3. Connect using your Headscale server: - ```bash - tailscale up --login-server https://. --authkey - ``` - -### Automatic Uptime Kuma Monitor: - -**The playbook will automatically create a monitor in Uptime Kuma:** -- ✅ **Headscale** - monitors `https:///health` -- Added to "services" monitor group -- Uses ntfy notification (if configured) -- Check every 60 seconds - -**Prerequisites:** Uptime Kuma credentials must be in `infra_secrets.yml` (from Layer 4) - -### Verification: -The script will automatically verify: -- ✓ DNS records configured correctly -- ✓ Headscale installed and running -- ✓ Namespace created -- ✓ Caddy config created -- ✓ Machines joined (if selected) -- ✓ Monitor created in Uptime Kuma "services" group - -List connected devices: -```bash -ssh counterweight@ -sudo headscale nodes list -``` - -### Why Use Headscale: -- **Secure communication** between all your machines -- **Magic DNS** - access machines by hostname -- **NAT traversal** - works even behind firewalls -- **Self-hosted** - full control of your VPN -- **Mobile support** - use official Tailscale apps - -### Backup: -Optional backup to lapy: -```bash -ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml -``` - ---- - -## Layer 6: Infrastructure Monitoring - -**Goal:** Deploy automated monitoring for disk usage, system health, and CPU temperature. - -**Script:** `./scripts/setup_layer_6_infra_monitoring.sh` - -### What This Layer Does: - -Deploys monitoring scripts that report to Uptime Kuma: - -#### 6A: Disk Usage Monitoring -- Monitors disk usage on specified mount points -- Sends alerts when usage exceeds threshold (default: 80%) -- Creates Uptime Kuma push monitors automatically -- Organizes monitors in host-specific groups -- **Deploys to:** All hosts (selectable) - -#### 6B: System Healthcheck -- Sends regular heartbeat pings to Uptime Kuma -- Alerts if system stops responding -- "No news is good news" monitoring -- **Deploys to:** All hosts (selectable) - -#### 6C: CPU Temperature Monitoring (Nodito only) -- Monitors CPU temperature on Proxmox server -- Alerts when temperature exceeds threshold (default: 80°C) -- **Deploys to:** nodito (if configured) - -### Prerequisites (Complete BEFORE Running): - -**1. Previous layers complete:** -- ✅ Layer 0, 1A/1B, 4 complete -- ✅ Uptime Kuma deployed and configured (Layer 4) -- ✅ **CRITICAL:** `infra_secrets.yml` has Uptime Kuma credentials - -**2. Uptime Kuma API credentials ready:** -- ✅ Must have completed Layer 4 post-deployment steps -- ✅ `ansible/infra_secrets.yml` must contain: - ```yaml - uptime_kuma_username: "your-username" - uptime_kuma_password: "your-password" - ``` - -**3. Python dependencies installed:** -- ✅ `uptime-kuma-api` must be in requirements.txt -- ✅ Should already be installed from Layer 0 -- ✅ Verify: `pip list | grep uptime-kuma-api` - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_6_infra_monitoring.sh -``` - -The script will: -1. Verify Uptime Kuma credentials -2. Offer to deploy disk usage monitoring -3. Offer to deploy system healthchecks -4. Offer to deploy CPU temp monitoring (nodito only) -5. Test monitor creation and alerts - -### What Gets Deployed: - -**For each monitored host:** -- Push monitor in Uptime Kuma (upside-down mode) -- Monitor group named `{hostname} - infra` -- Systemd service for monitoring script -- Systemd timer for periodic execution -- Log file for monitoring history - -**Default settings (customizable):** -- Disk usage threshold: 80% -- Disk check interval: 15 minutes -- Healthcheck interval: 60 seconds -- CPU temp threshold: 80°C -- Monitored mount point: `/` (root) - -### Customization Options: - -Change thresholds and intervals: -```bash -# Disk monitoring with custom settings -ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml \ - -e "disk_usage_threshold_percent=85" \ - -e "disk_check_interval_minutes=10" \ - -e "monitored_mount_point=/home" - -# Healthcheck with custom interval -ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml \ - -e "healthcheck_interval_seconds=30" - -# CPU temp with custom threshold -ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml \ - -e "temp_threshold_celsius=75" -``` - -### Verification: -The script will automatically verify: -- ✓ Uptime Kuma API accessible -- ✓ Monitors created in Uptime Kuma -- ✓ Monitor groups created -- ✓ Systemd services running -- ✓ Can send test alerts - -Check Uptime Kuma web UI: -- Monitors should appear organized by host -- Should receive test pings -- Alerts will show when thresholds exceeded - -### Post-Deployment: - -**Monitor your infrastructure:** -1. Open Uptime Kuma web UI -2. See all monitors organized by host groups -3. Configure notification rules per monitor -4. Set up status pages (optional) - -**Test alerts:** -```bash -# Trigger disk usage alert (fill disk temporarily) -# Trigger healthcheck alert (stop the service) -# Check ntfy for notifications -``` - -### Why This Layer is Important: -- **Proactive monitoring** - Know about issues before users do -- **Disk space alerts** - Prevent services from failing -- **System health** - Detect crashed/frozen machines -- **Temperature monitoring** - Prevent hardware damage -- **Organized** - All monitors grouped by host - ---- - -## Layer 7: Core Services - -**Goal:** Deploy core applications: Vaultwarden, Forgejo, and LNBits. - -**Script:** `./scripts/setup_layer_7_services.sh` - -### What This Layer Does: - -Deploys main services on vipy: - -#### 7A: Vaultwarden (Password Manager) -- Deploys via Docker -- Configures Caddy reverse proxy -- Sets up fail2ban protection -- Enables sign-ups initially (disable after creating first user) -- **Deployed to:** vipy - -#### 7B: Forgejo (Git Server) -- Installs Forgejo binary -- Creates git user and directories -- Configures Caddy reverse proxy -- Enables SSH cloning -- **Deployed to:** vipy - -#### 7C: LNBits (Lightning Wallet) -- Installs system dependencies and uv (Python 3.12 tooling) -- Clones LNBits version v1.3.1 -- Syncs dependencies with uv targeting Python 3.12 -- Configures with FakeWallet backend (for testing) -- Creates systemd service -- Configures Caddy reverse proxy -- **Deployed to:** vipy - -### Prerequisites (Complete BEFORE Running): - -**1. Previous layers complete:** -- ✅ Layer 0, 1A, 2, 3 complete -- ✅ Docker installed on vipy (Layer 2) -- ✅ Caddy running on vipy (Layer 3) - -**2. Configure subdomains (in centralized config):** -- ✅ Edit `ansible/services_config.yml` and customize subdomains under `subdomains:` section: - - Set `vaultwarden:` to your preferred subdomain (e.g., `vault` or `passwords`) - - Set `forgejo:` to your preferred subdomain (e.g., `git` or `code`) - - Set `lnbits:` to your preferred subdomain (e.g., `lnbits` or `wallet`) - -**3. Create DNS records matching your subdomains:** -- ✅ Create A record: `.` → vipy IP -- ✅ Create A record: `.` → vipy IP -- ✅ Create A record: `.` → vipy IP -- ✅ Wait for DNS propagation - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_7_services.sh -``` - -The script will: -1. Validate DNS configuration -2. Offer to deploy each service -3. Configure backups (optional) - -### Post-Deployment Steps: - -#### Vaultwarden: -1. Visit `https://.` -2. Create your first user account -3. **Important:** Disable sign-ups after first user: - ```bash - ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml - ``` -4. Optional: Set up backup to lapy - -#### Forgejo: -1. Visit `https://.` -2. Create admin account on first visit -3. Default: registrations disabled for security -4. SSH cloning works automatically after adding SSH key - -#### LNBits: -1. Visit `https://.` -2. Create superuser on first visit -3. **Important:** Default uses FakeWallet (testing only) -4. Configure real Lightning backend: - - Edit `/opt/lnbits/lnbits/.env` on vipy - - Or use the superuser UI to configure backend -5. Disable new user registration for security -6. Optional: Set up encrypted backup to lapy - -### Backup Configuration: - -After services are stable, set up backups: - -**Vaultwarden backup:** -```bash -ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml -``` - -**LNBits backup (GPG encrypted):** -```bash -ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml -``` - -**Note:** Forgejo backups are not automated - backup manually or set up your own solution. - -### Automatic Uptime Kuma Monitors: - -**The playbooks will automatically create monitors in Uptime Kuma for each service:** -- ✅ **Vaultwarden** - monitors `https:///alive` -- ✅ **Forgejo** - monitors `https:///api/healthz` -- ✅ **LNBits** - monitors `https:///api/v1/health` - -All monitors: -- Added to "services" monitor group -- Use ntfy notification (if configured) -- Check every 60 seconds -- 3 retries before alerting - -**Prerequisites:** Uptime Kuma credentials must be in `infra_secrets.yml` (from Layer 4) - -### Verification: -The script will automatically verify: -- ✓ DNS records configured -- ✓ Services deployed -- ✓ Docker containers running (Vaultwarden) -- ✓ Systemd services running (Forgejo, LNBits) -- ✓ Caddy configs created - -Manual verification: -- Visit each service's subdomain -- Create admin/first user accounts -- Test functionality -- Check Uptime Kuma for new monitors in "services" group - -### Why These Services: -- **Vaultwarden** - Self-hosted password manager (Bitwarden compatible) -- **Forgejo** - Self-hosted Git server (GitHub/GitLab alternative) -- **LNBits** - Lightning Network wallet and accounts system - ---- - -## Layer 8: Secondary Services - -**Goal:** Deploy auxiliary services that depend on the core stack: ntfy-emergency-app and memos. - -**Script:** `./scripts/setup_layer_8_secondary_services.sh` - -### What This Layer Does: -- Deploys the ntfy-emergency-app container on vipy and proxies it through Caddy -- Optionally deploys Memos on `memos-box` (skips automatically if the host is not yet in `inventory.ini`) - -### Prerequisites (Complete BEFORE Running): -- ✅ Layers 0–7 complete (Caddy, ntfy, and Uptime Kuma already online) -- ✅ `ansible/services_config.yml` reviewed so the `ntfy_emergency_app` and `memos` subdomains match your plan -- ✅ `ansible/infra_secrets.yml` contains valid `ntfy_username` and `ntfy_password` -- ✅ DNS A records created for the subdomains (see below) -- ✅ If deploying Memos, ensure `memos-box` exists in `inventory.ini` and is reachable as the `counterweight` user - -### DNS Requirements: -- `.` → vipy IP -- `.` → memos-box IP (skip if memos not yet provisioned) - -The script runs `dig` to validate DNS before deploying and will warn if records are missing or pointing elsewhere. - -### Run the Script: -```bash -source venv/bin/activate -cd /home/counterweight/personal_infra -./scripts/setup_layer_8_secondary_services.sh -``` - -You can deploy each service independently; the script asks for confirmation before running each playbook. - -### Post-Deployment Steps: -- **ntfy-emergency-app:** Visit the emergency subdomain, trigger a test notification, and verify ntfy receives it -- **Memos (if deployed):** Visit the memos subdomain, create the first admin user, and adjust settings from the UI - -### Verification: -- The script checks for the presence of Caddy configs, running containers, and Memos systemd service status -- Review Uptime Kuma or add monitors for these services if you want automatic alerting - -### Optional Follow-Ups: -- Configure backups for any new data stores (e.g., snapshot memos data) -- Add Uptime Kuma monitors for the new services if you want automated alerting - ---- - -## Troubleshooting - -### Common Issues - -#### SSH Connection Fails -- Verify VPS is running and accessible -- Check SSH key is in the correct location -- Ensure SSH key has correct permissions (600) -- Try manual SSH: `ssh -i ~/.ssh/counterganzua root@` - -#### Ansible Not Found -- Make sure you've activated the venv: `source venv/bin/activate` -- Run Layer 0 script again - -#### DNS Not Resolving -- DNS changes can take up to 24-48 hours to propagate -- Use `dig .` to check DNS status -- You can proceed with setup; services will work once DNS propagates - ---- - -## Progress Tracking - -Use this checklist to track your progress: - -- [ ] Layer 0: Foundation Setup -- [ ] Layer 1A: VPS Basic Setup -- [ ] Layer 1B: Nodito (Proxmox) Setup -- [ ] Layer 2: General Infrastructure Tools -- [ ] Layer 3: Reverse Proxy (Caddy) -- [ ] Layer 4: Core Monitoring & Notifications -- [ ] Layer 5: VPN Infrastructure (Headscale) -- [ ] Layer 6: Infrastructure Monitoring -- [ ] Layer 7: Core Services -- [ ] Layer 8: Secondary Services -- [ ] Backups Configured - diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index bb852da..3a0b18f 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -45,13 +45,6 @@ vms = { memory_mb = 2048 disk_size_gb = 20 ipconfig0 = "ip=dhcp" # or "ip=192.168.1.50/24,gw=192.168.1.1" - data_disks = [ - { - size_gb = 50 - # storage defaults to var.zfs_storage_name (proxmox-tank-1) - # optional: slot = "scsi2" - } - ] } } ``` diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf index 4e10a12..bfd400c 100644 --- a/tofu/nodito/main.tf +++ b/tofu/nodito/main.tf @@ -73,16 +73,6 @@ resource "proxmox_vm_qemu" "vm" { # optional flags like iothread/ssd/discard differ by provider versions; keep minimal } - dynamic "disk" { - for_each = try(each.value.data_disks, []) - content { - slot = try(disk.value.slot, format("scsi%s", tonumber(disk.key) + 1)) - type = "disk" - storage = try(disk.value.storage, var.zfs_storage_name) - size = "${disk.value.size_gb}G" - } - } - # Cloud-init CD-ROM so ipconfig0/sshkeys apply disk { slot = "ide2" diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index c957f35..cc88b3f 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -20,11 +20,6 @@ vms = { memory_mb = 2048 disk_size_gb = 20 ipconfig0 = "ip=dhcp" - data_disks = [ - { - size_gb = 50 - } - ] } db1 = { diff --git a/tofu/nodito/variables.tf b/tofu/nodito/variables.tf index 3f16e75..30a1418 100644 --- a/tofu/nodito/variables.tf +++ b/tofu/nodito/variables.tf @@ -55,11 +55,6 @@ variable "vms" { disk_size_gb = number vlan_tag = optional(number) ipconfig0 = optional(string) # e.g. "ip=dhcp" or "ip=192.168.1.50/24,gw=192.168.1.1" - data_disks = optional(list(object({ - size_gb = number - storage = optional(string) - slot = optional(string) - })), []) })) default = {} } From 2893bb77cd6132d2669062420f38fcc6c82e388c Mon Sep 17 00:00:00 2001 From: counterweight Date: Sat, 13 Dec 2025 18:54:46 +0100 Subject: [PATCH 03/13] improve tailscale add --- ansible/infra/920_join_headscale_mesh.yml | 42 ++++++++++++++++++----- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index cd6464c..77158fa 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -77,7 +77,7 @@ - name: Add Tailscale repository apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/debian {{ ansible_lsb.codename }} main" + repo: "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/debian {{ ansible_distribution_release }} main" state: present update_cache: yes @@ -99,6 +99,8 @@ --login-server {{ headscale_domain }} --authkey {{ auth_key }} --accept-dns=true + --hostname={{ ansible_hostname }} + --reset register: tailscale_up_result changed_when: "'already authenticated' not in tailscale_up_result.stdout" failed_when: tailscale_up_result.rc != 0 and 'already authenticated' not in tailscale_up_result.stdout @@ -107,6 +109,37 @@ pause: seconds: 2 + - name: Get node ID from headscale server + delegate_to: "{{ groups['lapy'][0] }}" + become: no + vars: + ssh_args: "{{ ('-i ' + headscale_key + ' ' if headscale_key else '') + '-p ' + headscale_port|string }}" + shell: > + ssh {{ ssh_args }} + {{ headscale_user }}@{{ headscale_host }} + "sudo headscale nodes list -o json" + register: nodes_list_result + changed_when: false + failed_when: nodes_list_result.rc != 0 + + - name: Extract node ID for this host + set_fact: + headscale_node_id: "{{ (nodes_list_result.stdout | from_json) | selectattr('given_name', 'equalto', ansible_hostname) | map(attribute='id') | first }}" + failed_when: headscale_node_id is not defined or headscale_node_id == '' + + - name: Tag node with its hostname + delegate_to: "{{ groups['lapy'][0] }}" + become: no + vars: + ssh_args: "{{ ('-i ' + headscale_key + ' ' if headscale_key else '') + '-p ' + headscale_port|string }}" + shell: > + ssh {{ ssh_args }} + {{ headscale_user }}@{{ headscale_host }} + "sudo headscale nodes tag --tags tag:{{ ansible_hostname }} -i {{ headscale_node_id }}" + register: tag_result + changed_when: true + failed_when: tag_result.rc != 0 + - name: Display Tailscale status command: tailscale status register: tailscale_status @@ -115,10 +148,3 @@ - name: Show Tailscale connection status debug: msg: "{{ tailscale_status.stdout_lines }}" - - - name: Deny all inbound traffic from Tailscale network interface - ufw: - rule: deny - direction: in - interface: tailscale0 - From 8863f800bf7ad56d58c887294626944740f728bd Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 14 Dec 2025 18:52:36 +0100 Subject: [PATCH 04/13] bitcoin node stuff --- ansible/infra/920_join_headscale_mesh.yml | 2 +- ansible/infra_secrets.yml.example | 4 + .../bitcoin-knots/bitcoin_knots_vars.yml | 12 +- .../deploy_bitcoin_knots_playbook.yml | 96 ++-- .../fulcrum/deploy_fulcrum_playbook.yml | 475 ++++++++++++++++++ ansible/services/fulcrum/fulcrum_vars.yml | 40 ++ .../headscale/deploy_headscale_playbook.yml | 8 +- .../deploy_headscale_ui_playbook.yml | 142 ------ 8 files changed, 573 insertions(+), 206 deletions(-) create mode 100644 ansible/services/fulcrum/deploy_fulcrum_playbook.yml create mode 100644 ansible/services/fulcrum/fulcrum_vars.yml delete mode 100644 ansible/services/headscale/deploy_headscale_ui_playbook.yml diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 77158fa..8d06d44 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -44,7 +44,7 @@ shell: > ssh {{ ssh_args }} {{ headscale_user }}@{{ headscale_host }} - "sudo headscale preauthkeys create --user {{ headscale_user_id }} --expiration 1m --output json" + "sudo headscale preauthkeys create --user {{ headscale_user_id }} --expiration 10m --output json" register: preauth_key_result changed_when: true failed_when: preauth_key_result.rc != 0 diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index 2482160..cddc58a 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -22,3 +22,7 @@ headscale_ui_password: "your_secure_password_here" bitcoin_rpc_user: "bitcoinrpc" bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Mempool MariaDB credentials +# Used by: services/mempool/deploy_mempool_playbook.yml +mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml index fc38c75..c9bd7ca 100644 --- a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml +++ b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml @@ -14,8 +14,7 @@ bitcoin_conf_dir: /etc/bitcoin # Network bitcoin_rpc_port: 8332 bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "127.0.0.1" # Security: localhost only -bitcoin_tailscale_interface: tailscale0 # Tailscale interface for UFW rules +bitcoin_rpc_bind: "0.0.0.0" # Build options bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores @@ -23,10 +22,17 @@ bitcoin_build_prefix: /usr/local # Configuration options bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_enable_prune: false # Set to prune amount (e.g., 550) to enable pruning, false for full node (MUST be false for Electrum servers) bitcoin_max_connections: 125 # dbcache will be calculated as 90% of host RAM automatically in playbook +# ZMQ Configuration +bitcoin_zmq_enabled: true +bitcoin_zmq_bind: "tcp://0.0.0.0" +bitcoin_zmq_port_rawblock: 28332 +bitcoin_zmq_port_rawtx: 28333 +bitcoin_zmq_port_hashblock: 28334 +bitcoin_zmq_port_hashtx: 28335 + # Service user bitcoin_user: bitcoin bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index 2c7cebb..47c4f58 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -137,7 +137,6 @@ changed_when: false when: not bitcoind_binary_exists.stat.exists - - name: Download SHA256SUMS file get_url: url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" @@ -156,7 +155,7 @@ command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS register: sha256sums_verification changed_when: false - failed_when: sha256sums_verification.rc != 0 + failed_when: false # Don't fail here - check for 'Good signature' in next task when: not bitcoind_binary_exists.stat.exists @@ -260,6 +259,7 @@ -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} -DBUILD_BITCOIN_WALLET=OFF -DCMAKE_BUILD_TYPE=Release + -DWITH_ZMQ=ON .. args: chdir: "{{ bitcoin_knots_source_dir }}/build" @@ -267,6 +267,15 @@ register: configure_result changed_when: true + - name: Verify CMake enabled ZMQ + shell: | + set -e + cd "{{ bitcoin_knots_source_dir }}/build" + cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' + when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) + register: zmq_check + changed_when: false + - name: Fail if CMakeLists.txt not found fail: msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." @@ -336,7 +345,7 @@ rpcpassword={{ bitcoin_rpc_password }} rpcbind={{ bitcoin_rpc_bind }} rpcport={{ bitcoin_rpc_port }} - rpcallowip=127.0.0.1 + rpcallowip=0.0.0.0/0 # Network Configuration listen=1 @@ -351,14 +360,17 @@ txindex=1 {% endif %} - # Pruning (optional) - {% if bitcoin_enable_prune %} - prune={{ bitcoin_enable_prune }} - {% endif %} - - # Logging + # Logging (to journald via systemd) logtimestamps=1 - logfile={{ bitcoin_data_dir }}/debug.log + printtoconsole=1 + + # ZMQ Configuration + {% if bitcoin_zmq_enabled | default(false) %} + zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} + zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} + zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} + zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} + {% endif %} # Security disablewallet=1 @@ -427,33 +439,6 @@ debug: msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - - name: Allow Bitcoin P2P port on Tailscale interface only - ufw: - rule: allow - direction: in - port: "{{ bitcoin_p2p_port }}" - proto: tcp - interface: "{{ bitcoin_tailscale_interface }}" - comment: "Bitcoin Knots P2P (Tailscale only)" - - - name: Allow Bitcoin P2P port (UDP) on Tailscale interface only - ufw: - rule: allow - direction: in - port: "{{ bitcoin_p2p_port }}" - proto: udp - interface: "{{ bitcoin_tailscale_interface }}" - comment: "Bitcoin Knots P2P UDP (Tailscale only)" - - - name: Verify UFW rules for Bitcoin Knots - command: ufw status numbered - register: ufw_status - changed_when: false - - - name: Display UFW status - debug: - msg: "{{ ufw_status.stdout_lines }}" - - name: Create Bitcoin Knots health check and push script copy: dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh @@ -480,11 +465,12 @@ "http://${RPC_HOST}:${RPC_PORT}" 2>&1) if [ $? -eq 0 ]; then - # Check if response contains error - if echo "$response" | grep -q '"error"'; then - return 1 - else + # Check if response contains a non-null error + # Successful responses have "error": null, failures have "error": {...} + if echo "$response" | grep -q '"error":null\|"error": null'; then return 0 + else + return 1 fi else return 1 @@ -501,11 +487,14 @@ return 1 fi - # URL encode the message - local encoded_msg=$(echo -n "$msg" | curl -Gso /dev/null -w %{url_effective} --data-urlencode "msg=$msg" "" | cut -c 3-) + # URL encode spaces in message + local encoded_msg="${msg// /%20}" - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping=" || true + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi } # Main health check @@ -630,14 +619,14 @@ if existing_monitor: print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - # Get push URL from existing monitor - push_id = existing_monitor.get('push_token', existing_monitor.get('id')) - push_url = f"{url}/api/push/{push_id}" + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" print(f"Push URL: {push_url}") - print("Skipping - monitor already configured") else: print(f"Creating push monitor '{monitor_name}'...") - result = api.add_monitor( + api.add_monitor( type=MonitorType.PUSH, name=monitor_name, parent=group['id'], @@ -646,12 +635,13 @@ retryInterval=60, notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} ) - # Get push URL from created monitor monitors = api.get_monitors() new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) if new_monitor: - push_id = new_monitor.get('push_token', new_monitor.get('id')) - push_url = f"{url}/api/push/{push_id}" + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" print(f"Push URL: {push_url}") api.disconnect() diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml new file mode 100644 index 0000000..1f7f73d --- /dev/null +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -0,0 +1,475 @@ +- name: Deploy Fulcrum Electrum Server + hosts: fulcrum_box_local + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml + vars: + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Calculate 75% of system RAM for db_mem + set_fact: + fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" + changed_when: false + + - name: Display calculated db_mem value + debug: + msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" + + - name: Display Fulcrum version to install + debug: + msg: "Installing Fulcrum version {{ fulcrum_version }}" + + - name: Install required packages + apt: + name: + - curl + - wget + - openssl + state: present + update_cache: yes + + - name: Create fulcrum group + group: + name: "{{ fulcrum_group }}" + system: yes + state: present + + - name: Create fulcrum user + user: + name: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + system: yes + shell: /usr/sbin/nologin + home: /home/{{ fulcrum_user }} + create_home: yes + state: present + + - name: Create Fulcrum database directory (heavy data on special mount) + file: + path: "{{ fulcrum_db_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum config directory + file: + path: "{{ fulcrum_config_dir }}" + state: directory + owner: root + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Create Fulcrum lib directory (for banner and other data files) + file: + path: "{{ fulcrum_lib_dir }}" + state: directory + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0755' + + - name: Check if Fulcrum binary already exists + stat: + path: "{{ fulcrum_binary_path }}" + register: fulcrum_binary_exists + changed_when: false + + - name: Download Fulcrum binary tarball + get_url: + url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + mode: '0644' + when: not fulcrum_binary_exists.stat.exists + + - name: Extract Fulcrum binary + unarchive: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" + dest: "/tmp" + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Install Fulcrum binary + copy: + src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" + dest: "{{ fulcrum_binary_path }}" + owner: root + group: root + mode: '0755' + remote_src: yes + when: not fulcrum_binary_exists.stat.exists + + - name: Verify Fulcrum binary installation + command: "{{ fulcrum_binary_path }} --version" + register: fulcrum_version_check + changed_when: false + + - name: Display Fulcrum version + debug: + msg: "{{ fulcrum_version_check.stdout_lines }}" + + - name: Create Fulcrum banner file + copy: + dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" + content: | + counterinfra + + PER ASPERA AD ASTRA + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + + - name: Create Fulcrum configuration file + copy: + dest: "{{ fulcrum_config_dir }}/fulcrum.conf" + content: | + # Fulcrum Configuration + # Generated by Ansible + + # Bitcoin Core/Knots RPC settings + bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} + rpcuser = {{ bitcoin_rpc_user }} + rpcpassword = {{ bitcoin_rpc_password }} + + # Fulcrum server general settings + datadir = {{ fulcrum_db_dir }} + tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} + peering = {{ 'true' if fulcrum_peering else 'false' }} + zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + + # Anonymize client IP addresses and TxIDs in logs + anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} + + # Max RocksDB Memory in MiB + db_mem = {{ fulcrum_db_mem_mb }}.0 + + # Banner + banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0640' + notify: Restart fulcrum + + - name: Create systemd service file for Fulcrum + copy: + dest: /etc/systemd/system/fulcrum.service + content: | + # MiniBolt: systemd unit for Fulcrum + # /etc/systemd/system/fulcrum.service + + [Unit] + Description=Fulcrum + After=network.target + + StartLimitBurst=2 + StartLimitIntervalSec=20 + + [Service] + ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf + + User={{ fulcrum_user }} + Group={{ fulcrum_group }} + + # Process management + #################### + Type=simple + KillSignal=SIGINT + TimeoutStopSec=300 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + notify: Restart fulcrum + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum service + systemd: + name: fulcrum + enabled: yes + state: started + + - name: Wait for Fulcrum to start + wait_for: + port: "{{ fulcrum_tcp_port }}" + host: "{{ fulcrum_tcp_bind }}" + delay: 5 + timeout: 30 + ignore_errors: yes + + - name: Check Fulcrum service status + systemd: + name: fulcrum + register: fulcrum_service_status + changed_when: false + + - name: Display Fulcrum service status + debug: + msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" + + - name: Create Fulcrum health check and push script + copy: + dest: /usr/local/bin/fulcrum-healthcheck-push.sh + content: | + #!/bin/bash + # + # Fulcrum Health Check and Push to Uptime Kuma + # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma + # + + FULCRUM_HOST="{{ fulcrum_tcp_bind }}" + FULCRUM_PORT={{ fulcrum_tcp_port }} + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + # Check if Fulcrum TCP port is responding + check_fulcrum() { + # Try to connect to TCP port + timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null + return $? + } + + # Push status to Uptime Kuma + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + + # URL encode the message + local encoded_msg=$(echo -n "$msg" | curl -Gso /dev/null -w %{url_effective} --data-urlencode "msg=$msg" "" | cut -c 3-) + + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping=" || true + } + + # Main health check + if check_fulcrum; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Fulcrum TCP port not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create systemd timer for Fulcrum health check + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.timer + content: | + [Unit] + Description=Fulcrum Health Check Timer + Requires=fulcrum.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for Fulcrum health check + copy: + dest: /etc/systemd/system/fulcrum-healthcheck.service + content: | + [Unit] + Description=Fulcrum Health Check and Push to Uptime Kuma + After=network.target fulcrum.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon for health check + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum health check timer + systemd: + name: fulcrum-healthcheck.timer + enabled: yes + state: started + + - name: Create Uptime Kuma push monitor setup script for Fulcrum + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + # Get push URL from existing monitor + push_id = existing_monitor.get('push_token', existing_monitor.get('id')) + push_url = f"{url}/api/push/{push_id}" + print(f"Push URL: {push_url}") + print("Skipping - monitor already configured") + else: + print(f"Creating push monitor '{monitor_name}'...") + result = api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + # Get push URL from created monitor + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_id = new_monitor.get('push_token', new_monitor.get('id')) + push_url = f"{url}/api/push/{push_id}" + print(f"Push URL: {push_url}") + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_name: "Fulcrum" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_fulcrum_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Extract push URL from monitor setup output + set_fact: + uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" + delegate_to: localhost + become: no + when: monitor_setup.stdout is defined + + - name: Display extracted push URL + debug: + msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" + when: uptime_kuma_push_url | default('') != '' + + - name: Set push URL in systemd service environment + lineinfile: + path: /etc/systemd/system/fulcrum-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" + state: present + insertafter: '^\[Service\]' + when: uptime_kuma_push_url | default('') != '' + + - name: Reload systemd daemon after push URL update + systemd: + daemon_reload: yes + when: uptime_kuma_push_url | default('') != '' + + - name: Restart health check timer to pick up new environment + systemd: + name: fulcrum-healthcheck.timer + state: restarted + when: uptime_kuma_push_url | default('') != '' + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_monitor.py + - /tmp/ansible_config.yml + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz + - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux + + handlers: + - name: Restart fulcrum + systemd: + name: fulcrum + state: restarted + diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml new file mode 100644 index 0000000..2be35d6 --- /dev/null +++ b/ansible/services/fulcrum/fulcrum_vars.yml @@ -0,0 +1,40 @@ +# Fulcrum Configuration Variables + +# Version - Pinned to specific release +fulcrum_version: "2.1.0" # Fulcrum version to install + +# Directories +fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) +fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) +fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk +fulcrum_binary_path: /usr/local/bin/Fulcrum + +# Network - Bitcoin RPC connection +# Bitcoin Knots is on a different host (knots_box_local) +# Using RPC user/password authentication (credentials from infra_secrets.yml) +bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) +bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum server +fulcrum_tcp_port: 50001 +# Binding address for Fulcrum TCP server: +# - "127.0.0.1" = localhost only (use when Caddy is on the same box) +# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) +# - Specific IP = bind to specific network interface +fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +# If Caddy is on a different box, set this to the IP address that Caddy will use to connect + +# Performance +# db_mem will be calculated as 75% of available RAM automatically in playbook +fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache + +# Configuration options +fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs +fulcrum_peering: false # Disable peering with other Fulcrum servers +fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications + +# Service user +fulcrum_user: fulcrum +fulcrum_group: fulcrum + diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index e8a2b37..1bcf5bf 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -90,13 +90,7 @@ copy: dest: /etc/headscale/acl.json content: | - { - "ACLs": [], - "Groups": {}, - "Hosts": {}, - "TagOwners": {}, - "Tests": [] - } + {} owner: headscale group: headscale mode: '0640' diff --git a/ansible/services/headscale/deploy_headscale_ui_playbook.yml b/ansible/services/headscale/deploy_headscale_ui_playbook.yml deleted file mode 100644 index 3be792c..0000000 --- a/ansible/services/headscale/deploy_headscale_ui_playbook.yml +++ /dev/null @@ -1,142 +0,0 @@ -- name: Deploy headscale-ui with Docker and configure Caddy reverse proxy - hosts: spacey - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./headscale_vars.yml - vars: - headscale_subdomain: "{{ subdomains.headscale }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - headscale_domain: "{{ headscale_subdomain }}.{{ root_domain }}" - headscale_ui_version: "2025.08.23" - headscale_ui_dir: /opt/headscale-ui - headscale_ui_http_port: 18080 - headscale_ui_https_port: 18443 - - tasks: - - name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: false - - - name: Fail if Docker is not installed - fail: - msg: "Docker is not installed. Please run the docker_playbook.yml first." - when: docker_check.rc != 0 - - - name: Ensure Docker service is running - systemd: - name: docker - state: started - enabled: yes - - - name: Create headscale-ui directory - file: - path: "{{ headscale_ui_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create docker-compose.yml for headscale-ui - copy: - dest: "{{ headscale_ui_dir }}/docker-compose.yml" - content: | - version: "3" - services: - headscale-ui: - image: ghcr.io/gurucomputing/headscale-ui:{{ headscale_ui_version }} - container_name: headscale-ui - restart: unless-stopped - ports: - - "{{ headscale_ui_http_port }}:8080" - - "{{ headscale_ui_https_port }}:8443" - owner: root - group: root - mode: '0644' - - - name: Deploy headscale-ui container with docker compose - command: docker compose up -d - args: - chdir: "{{ headscale_ui_dir }}" - register: docker_compose_result - changed_when: "'Creating' in docker_compose_result.stdout or 'Starting' in docker_compose_result.stdout or docker_compose_result.rc != 0" - - - name: Wait for headscale-ui to be ready - uri: - url: "http://localhost:{{ headscale_ui_http_port }}" - status_code: [200, 404] - register: headscale_ui_ready - until: headscale_ui_ready.status in [200, 404] - retries: 30 - delay: 2 - ignore_errors: yes - - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - - - name: Fail if username is not provided - fail: - msg: "headscale_ui_username must be set in infra_secrets.yml" - when: headscale_ui_username is not defined - - - name: Fail if neither password nor password hash is provided - fail: - msg: "Either headscale_ui_password or headscale_ui_password_hash must be set in infra_secrets.yml" - when: headscale_ui_password is not defined and headscale_ui_password_hash is not defined - - - name: Generate bcrypt hash for headscale-ui password - become: yes - command: caddy hash-password --plaintext "{{ headscale_ui_password }}" - register: headscale_ui_password_hash_result - changed_when: false - no_log: true - when: headscale_ui_password is defined and headscale_ui_password_hash is not defined - - - name: Set headscale-ui password hash from generated value - set_fact: - headscale_ui_password_hash: "{{ headscale_ui_password_hash_result.stdout.strip() }}" - when: headscale_ui_password is defined and headscale_ui_password_hash is not defined - - - name: Update headscale Caddy config to include headscale-ui /web route with authentication - become: yes - copy: - dest: "{{ caddy_sites_dir }}/headscale.conf" - content: | - {{ headscale_domain }} { - @headscale_ui { - path /web* - } - handle @headscale_ui { - basicauth { - {{ headscale_ui_username }} {{ headscale_ui_password_hash }} - } - reverse_proxy http://localhost:{{ headscale_ui_http_port }} - } - # Headscale API is protected by its own API key authentication - # All API operations require a valid Bearer token in the Authorization header - reverse_proxy * http://localhost:{{ headscale_port }} - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - command: systemctl reload caddy - From d82c9afbe52f2c903089750a06b0d7a3b260a22b Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 14 Dec 2025 22:15:29 +0100 Subject: [PATCH 05/13] mempool working --- .../mempool/deploy_mempool_playbook.yml | 751 ++++++++++++++++++ ansible/services/mempool/mempool_vars.yml | 33 + ansible/services_config.yml | 3 + tofu/nodito/main.tf | 11 +- 4 files changed, 788 insertions(+), 10 deletions(-) create mode 100644 ansible/services/mempool/deploy_mempool_playbook.yml create mode 100644 ansible/services/mempool/mempool_vars.yml diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml new file mode 100644 index 0000000..862e21b --- /dev/null +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -0,0 +1,751 @@ +- name: Deploy Mempool Block Explorer with Docker + hosts: mempool_box_local + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml + vars: + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + # =========================================== + # Docker Installation (from 910_docker_playbook.yml) + # =========================================== + - name: Remove old Docker-related packages + apt: + name: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + purge: yes + autoremove: yes + + - name: Update apt cache + apt: + update_cache: yes + + - name: Install prerequisites + apt: + name: + - ca-certificates + - curl + state: present + + - name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download Docker GPG key + get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + + - name: Get Debian architecture + command: dpkg --print-architecture + register: deb_arch + changed_when: false + + - name: Add Docker repository + apt_repository: + repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + update_cache: yes + + - name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + + - name: Ensure Docker is started and enabled + systemd: + name: docker + enabled: yes + state: started + + - name: Add user to docker group + user: + name: "{{ ansible_user }}" + groups: docker + append: yes + + # =========================================== + # Mempool Deployment + # =========================================== + - name: Create mempool directories + file: + path: "{{ item }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + loop: + - "{{ mempool_dir }}" + - "{{ mempool_data_dir }}" + - "{{ mempool_mysql_dir }}" + + - name: Create docker-compose.yml for Mempool + copy: + dest: "{{ mempool_dir }}/docker-compose.yml" + content: | + # All containers use host network for Tailscale MagicDNS resolution + services: + mariadb: + image: mariadb:10.11 + container_name: mempool-db + restart: unless-stopped + network_mode: host + environment: + MYSQL_DATABASE: "{{ mariadb_database }}" + MYSQL_USER: "{{ mariadb_user }}" + MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" + MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" + volumes: + - {{ mempool_mysql_dir }}:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + mempool-backend: + image: mempool/backend:{{ mempool_version }} + container_name: mempool-backend + restart: unless-stopped + network_mode: host + environment: + # Database (localhost since all containers share host network) + DATABASE_ENABLED: "true" + DATABASE_HOST: "127.0.0.1" + DATABASE_DATABASE: "{{ mariadb_database }}" + DATABASE_USERNAME: "{{ mariadb_user }}" + DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" + # Bitcoin Core/Knots (via Tailnet MagicDNS) + CORE_RPC_HOST: "{{ bitcoin_host }}" + CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" + CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" + CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" + # Electrum (Fulcrum via Tailnet MagicDNS) + ELECTRUM_HOST: "{{ fulcrum_host }}" + ELECTRUM_PORT: "{{ fulcrum_port }}" + ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" + # Mempool settings + MEMPOOL_NETWORK: "{{ mempool_network }}" + MEMPOOL_BACKEND: "electrum" + MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" + volumes: + - {{ mempool_data_dir }}:/backend/cache + depends_on: + mariadb: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + mempool-frontend: + image: mempool/frontend:{{ mempool_version }} + container_name: mempool-frontend + restart: unless-stopped + network_mode: host + environment: + FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" + BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" + depends_on: + - mempool-backend + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Pull Mempool images + command: docker compose pull + args: + chdir: "{{ mempool_dir }}" + + - name: Deploy Mempool containers with docker compose + command: docker compose up -d + args: + chdir: "{{ mempool_dir }}" + + - name: Wait for MariaDB to be healthy + command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db + register: mariadb_health + until: mariadb_health.stdout == 'healthy' + retries: 30 + delay: 10 + changed_when: false + + - name: Wait for Mempool backend to start + uri: + url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + method: GET + status_code: 200 + timeout: 10 + register: backend_check + until: backend_check.status == 200 + retries: 30 + delay: 10 + ignore_errors: yes + + - name: Wait for Mempool frontend to be available + uri: + url: "http://localhost:{{ mempool_frontend_port }}" + method: GET + status_code: 200 + timeout: 10 + register: frontend_check + until: frontend_check.status == 200 + retries: 20 + delay: 5 + ignore_errors: yes + + - name: Display deployment status + debug: + msg: + - "Mempool deployment complete!" + - "Frontend: http://localhost:{{ mempool_frontend_port }}" + - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" + - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" + - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" + + # =========================================== + # Health Check Scripts for Uptime Kuma Push Monitors + # =========================================== + - name: Create Mempool MariaDB health check script + copy: + dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + + check_container() { + local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) + [ "$status" = "healthy" ] + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_container; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "MariaDB container unhealthy" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool backend health check script + copy: + dest: /usr/local/bin/mempool-backend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + BACKEND_PORT={{ mempool_backend_port }} + + check_backend() { + curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_backend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Backend API not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + - name: Create Mempool frontend health check script + copy: + dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh + content: | + #!/bin/bash + UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" + FRONTEND_PORT={{ mempool_frontend_port }} + + check_frontend() { + curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 + } + + push_to_uptime_kuma() { + local status=$1 + local msg=$2 + if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then + echo "ERROR: UPTIME_KUMA_PUSH_URL not set" + return 1 + fi + curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true + } + + if check_frontend; then + push_to_uptime_kuma "up" "OK" + exit 0 + else + push_to_uptime_kuma "down" "Frontend not responding" + exit 1 + fi + owner: root + group: root + mode: '0755' + + # =========================================== + # Systemd Timers for Health Checks + # =========================================== + - name: Create systemd services for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" + content: | + [Unit] + Description=Mempool {{ item.label }} Health Check + After=network.target docker.service + + [Service] + Type=oneshot + User=root + ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh + Environment=UPTIME_KUMA_PUSH_URL= + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + loop: + - { name: "mariadb", label: "MariaDB" } + - { name: "backend", label: "Backend" } + - { name: "frontend", label: "Frontend" } + + - name: Create systemd timers for health checks + copy: + dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" + content: | + [Unit] + Description=Mempool {{ item }} Health Check Timer + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + loop: + - mariadb + - backend + - frontend + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start health check timers + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + enabled: yes + state: started + loop: + - mariadb + - backend + - frontend + + # =========================================== + # Uptime Kuma Push Monitor Setup + # =========================================== + - name: Create Uptime Kuma push monitor setup script for Mempool + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_monitors.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitors_to_create = config['monitors'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + results = {} + for monitor_name in monitors_to_create: + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + push_token = existing.get('pushToken') or existing.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + else: + print(f"Creating push monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PUSH, + name=monitor_name, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + monitors = api.get_monitors() + new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + if new_monitor: + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if push_token: + results[monitor_name] = f"{url}/api/push/{push_token}" + print(f"Push URL ({monitor_name}): {results[monitor_name]}") + + api.disconnect() + print("SUCCESS") + + # Write results to file for Ansible to read + with open('/tmp/mempool_push_urls.yml', 'w') as f: + yaml.dump(results, f) + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitors: + - "Mempool MariaDB" + - "Mempool Backend" + - "Mempool Frontend" + mode: '0644' + + - name: Run Uptime Kuma push monitor setup + command: python3 /tmp/setup_mempool_monitors.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Display monitor setup output + debug: + msg: "{{ monitor_setup.stdout_lines }}" + when: monitor_setup.stdout is defined + + - name: Read push URLs from file + slurp: + src: /tmp/mempool_push_urls.yml + delegate_to: localhost + become: no + register: push_urls_file + ignore_errors: yes + + - name: Parse push URLs + set_fact: + push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" + when: push_urls_file.content is defined + ignore_errors: yes + + - name: Update MariaDB health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-mariadb-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool MariaDB'] is defined + + - name: Update Backend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-backend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Backend'] is defined + + - name: Update Frontend health check service with push URL + lineinfile: + path: /etc/systemd/system/mempool-frontend-healthcheck.service + regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' + line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" + insertafter: '^\[Service\]' + when: push_urls is defined and push_urls['Mempool Frontend'] is defined + + - name: Reload systemd after push URL updates + systemd: + daemon_reload: yes + when: push_urls is defined + + - name: Restart health check timers + systemd: + name: "mempool-{{ item }}-healthcheck.timer" + state: restarted + loop: + - mariadb + - backend + - frontend + when: push_urls is defined + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_monitors.py + - /tmp/ansible_mempool_config.yml + - /tmp/mempool_push_urls.yml + +- name: Configure Caddy reverse proxy for Mempool on vipy + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./mempool_vars.yml + vars: + mempool_subdomain: "{{ subdomains.mempool }}" + mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + + tasks: + - name: Ensure Caddy sites-enabled directory exists + file: + path: "{{ caddy_sites_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Ensure Caddyfile includes import directive for sites-enabled + lineinfile: + path: /etc/caddy/Caddyfile + line: 'import sites-enabled/*' + insertafter: EOF + state: present + backup: yes + create: yes + mode: '0644' + + - name: Create Caddy reverse proxy configuration for Mempool + copy: + dest: "{{ caddy_sites_dir }}/mempool.conf" + content: | + {{ mempool_domain }} { + reverse_proxy mempool-box:{{ mempool_frontend_port }} { + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers 100.100.100.100 + } + } + } + owner: root + group: root + mode: '0644' + + - name: Reload Caddy to apply new config + systemd: + name: caddy + state: reloaded + + - name: Display Mempool URL + debug: + msg: "Mempool is now available at https://{{ mempool_domain }}" + + # =========================================== + # Uptime Kuma HTTP Monitor for Public Endpoint + # =========================================== + - name: Create Uptime Kuma HTTP monitor setup script for Mempool + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_mempool_http_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating HTTP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for HTTP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_mempool_http_config.yml + content: | + uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ mempool_domain }}" + monitor_name: "Mempool" + mode: '0644' + + - name: Run Uptime Kuma HTTP monitor setup + command: python3 /tmp/setup_mempool_http_monitor.py + delegate_to: localhost + become: no + register: http_monitor_setup + changed_when: "'SUCCESS' in http_monitor_setup.stdout" + ignore_errors: yes + + - name: Display HTTP monitor setup output + debug: + msg: "{{ http_monitor_setup.stdout_lines }}" + when: http_monitor_setup.stdout is defined + + - name: Clean up HTTP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_mempool_http_monitor.py + - /tmp/ansible_mempool_http_config.yml + diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml new file mode 100644 index 0000000..d3051c3 --- /dev/null +++ b/ansible/services/mempool/mempool_vars.yml @@ -0,0 +1,33 @@ +# Mempool Configuration Variables + +# Version - Pinned to specific release +mempool_version: "v3.2.1" + +# Directories +mempool_dir: /opt/mempool +mempool_data_dir: "{{ mempool_dir }}/data" +mempool_mysql_dir: "{{ mempool_dir }}/mysql" + +# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) +bitcoin_host: "knots-box" +bitcoin_rpc_port: 8332 +# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml + +# Network - Fulcrum Electrum server (via Tailnet Magic DNS) +fulcrum_host: "fulcrum-box" +fulcrum_port: 50001 +fulcrum_tls: "false" + +# Mempool network mode +mempool_network: "mainnet" + +# Container ports (internal) +mempool_frontend_port: 8080 +mempool_backend_port: 8999 + +# MariaDB settings +mariadb_database: "mempool" +mariadb_user: "mempool" +# Note: mariadb_mempool_password is loaded from infra_secrets.yml + + diff --git a/ansible/services_config.yml b/ansible/services_config.yml index b497c51..56cc570 100644 --- a/ansible/services_config.yml +++ b/ansible/services_config.yml @@ -21,6 +21,9 @@ subdomains: # Memos (on memos-box) memos: memos + + # Mempool Block Explorer (on mempool_box, proxied via vipy) + mempool: mempool # Caddy configuration caddy_sites_dir: /etc/caddy/sites-enabled diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf index bfd400c..9123175 100644 --- a/tofu/nodito/main.tf +++ b/tofu/nodito/main.tf @@ -30,16 +30,7 @@ resource "proxmox_vm_qemu" "vm" { lifecycle { prevent_destroy = true - ignore_changes = [ - name, - cpu, - memory, - network, - ipconfig0, - ciuser, - sshkeys, - cicustom, - ] + ignore_changes = all } serial { From e6be6cea513ba92ea02224c22ad99a545015e53e Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 14 Dec 2025 22:33:12 +0100 Subject: [PATCH 06/13] small fixes --- .../fulcrum/deploy_fulcrum_playbook.yml | 28 +++++++++++-------- .../mempool/deploy_mempool_playbook.yml | 4 +-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 1f7f73d..984555f 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -245,11 +245,14 @@ return 1 fi - # URL encode the message - local encoded_msg=$(echo -n "$msg" | curl -Gso /dev/null -w %{url_effective} --data-urlencode "msg=$msg" "" | cut -c 3-) + # URL encode spaces in message + local encoded_msg="${msg// /%20}" - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping=" || true + if ! curl -s --max-time 10 --retry 2 -o /dev/null \ + "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then + echo "ERROR: Failed to push to Uptime Kuma" + return 1 + fi } # Main health check @@ -369,14 +372,14 @@ if existing_monitor: print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - # Get push URL from existing monitor - push_id = existing_monitor.get('push_token', existing_monitor.get('id')) - push_url = f"{url}/api/push/{push_id}" + push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for monitor") + push_url = f"{url}/api/push/{push_token}" print(f"Push URL: {push_url}") - print("Skipping - monitor already configured") else: print(f"Creating push monitor '{monitor_name}'...") - result = api.add_monitor( + api.add_monitor( type=MonitorType.PUSH, name=monitor_name, parent=group['id'], @@ -385,12 +388,13 @@ retryInterval=60, notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} ) - # Get push URL from created monitor monitors = api.get_monitors() new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) if new_monitor: - push_id = new_monitor.get('push_token', new_monitor.get('id')) - push_url = f"{url}/api/push/{push_id}" + push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') + if not push_token: + raise ValueError("Could not find push token for new monitor") + push_url = f"{url}/api/push/{push_token}" print(f"Push URL: {push_url}") api.disconnect() diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml index 862e21b..658180a 100644 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ b/ansible/services/mempool/deploy_mempool_playbook.yml @@ -465,7 +465,7 @@ type=MonitorType.PUSH, name=monitor_name, parent=group['id'], - interval=60, + interval=90, maxretries=3, retryInterval=60, notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} @@ -723,7 +723,7 @@ username: "{{ uptime_kuma_username }}" password: "{{ uptime_kuma_password }}" monitor_url: "https://{{ mempool_domain }}" - monitor_name: "Mempool" + monitor_name: "Mempool Public Access" mode: '0644' - name: Run Uptime Kuma HTTP monitor setup From b70d4bd0e0c604117de1e47a534b5626f4623307 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 15 Dec 2025 19:28:02 +0100 Subject: [PATCH 07/13] memos --- .../services/memos/deploy_memos_playbook.yml | 319 ++++++++++++------ ansible/services/memos/memos_vars.yml | 29 +- .../memos/setup_backup_memos_to_lapy.yml | 106 ++++++ 3 files changed, 350 insertions(+), 104 deletions(-) create mode 100644 ansible/services/memos/setup_backup_memos_to_lapy.yml diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index d3276f5..da56bd6 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,105 +1,111 @@ -- name: Deploy memos and configure Caddy reverse proxy - hosts: memos-box +- name: Deploy Memos on memos-box + hosts: memos_box_local become: yes vars_files: - ../../infra_vars.yml - ../../services_config.yml + - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" tasks: - - name: Install required packages + - name: Ensure required packages are installed apt: name: - wget - - curl - - unzip + - tar state: present - update_cache: yes + update_cache: true - - name: Get latest memos release version - uri: - url: https://api.github.com/repos/usememos/memos/releases/latest - return_content: yes - register: memos_latest_release - - - name: Set memos version and find download URL - set_fact: - memos_version: "{{ memos_latest_release.json.tag_name | regex_replace('^v', '') }}" - - - name: Find linux-amd64 download URL - set_fact: - memos_download_url: "{{ memos_latest_release.json.assets | json_query('[?contains(name, `linux-amd64`) && (contains(name, `.tar.gz`) || contains(name, `.zip`))].browser_download_url') | first }}" - - - name: Display memos version to install - debug: - msg: "Installing memos version {{ memos_version }} from {{ memos_download_url }}" - - - name: Download memos binary - get_url: - url: "{{ memos_download_url }}" - dest: /tmp/memos_archive - mode: '0644' - register: memos_download - - - name: Extract memos binary - unarchive: - src: /tmp/memos_archive - dest: /tmp/memos_extract - remote_src: yes - creates: /tmp/memos_extract/memos - - - name: Install memos binary - copy: - src: /tmp/memos_extract/memos - dest: /usr/local/bin/memos - mode: '0755' - remote_src: yes - notify: Restart memos - - - name: Remove temporary files - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/memos_archive - - /tmp/memos_extract - - - name: Ensure memos user exists + - name: Create memos system user user: - name: memos + name: "{{ memos_user }}" system: yes - shell: /usr/sbin/nologin - home: /var/lib/memos - create_home: yes - state: present + shell: /bin/false + home: "{{ memos_data_dir }}" + create_home: no + comment: "Memos Service" - name: Create memos data directory file: path: "{{ memos_data_dir }}" state: directory - owner: memos - group: memos + owner: "{{ memos_user }}" + group: "{{ memos_user }}" mode: '0750' - - name: Create memos systemd service file + - name: Create memos config directory + file: + path: "{{ memos_config_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Download memos binary archive + get_url: + url: "{{ memos_url }}" + dest: "/tmp/memos.tar.gz" + mode: '0644' + + - name: Extract memos binary + unarchive: + src: "/tmp/memos.tar.gz" + dest: "/tmp" + remote_src: yes + + - name: Move memos binary to /usr/local/bin + copy: + src: "/tmp/memos" + dest: "{{ memos_bin_path }}" + remote_src: yes + mode: '0755' + owner: root + group: root + + - name: Clean up temporary files + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/memos.tar.gz + - /tmp/memos + + - name: Create memos environment file + copy: + dest: "{{ memos_config_dir }}/memos.env" + content: | + MEMOS_MODE=prod + MEMOS_ADDR=0.0.0.0 + MEMOS_PORT={{ memos_port }} + MEMOS_DATA={{ memos_data_dir }} + MEMOS_DRIVER=sqlite + owner: root + group: root + mode: '0644' + notify: Restart memos + + - name: Create memos systemd service copy: dest: /etc/systemd/system/memos.service content: | [Unit] - Description=memos service + Description=Memos - A privacy-first, lightweight note-taking service After=network.target [Service] Type=simple - User=memos - Group=memos - ExecStart=/usr/local/bin/memos --port {{ memos_port }} --data {{ memos_data_dir }} - Restart=on-failure - RestartSec=5s + User={{ memos_user }} + Group={{ memos_user }} + WorkingDirectory={{ memos_data_dir }} + EnvironmentFile={{ memos_config_dir }}/memos.env + ExecStart={{ memos_bin_path }} + Restart=always + RestartSec=3 + StandardOutput=journal + StandardError=journal [Install] WantedBy=multi-user.target @@ -108,35 +114,52 @@ mode: '0644' notify: Restart memos + - name: Reload systemd daemon + systemd: + daemon_reload: yes + - name: Enable and start memos service systemd: name: memos enabled: yes state: started - daemon_reload: yes - name: Wait for memos to be ready uri: - url: "http://localhost:{{ memos_port }}/api/v1/status" + url: "http://127.0.0.1:{{ memos_port }}/healthz" + method: GET status_code: 200 - register: memos_ready - until: memos_ready.status == 200 - retries: 30 - delay: 2 - ignore_errors: yes + register: memos_health + retries: 10 + delay: 3 + until: memos_health.status == 200 - - name: Allow HTTPS through UFW - ufw: - rule: allow - port: '443' - proto: tcp + - name: Display memos status + debug: + msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" - - name: Allow HTTP through UFW (for Let's Encrypt) - ufw: - rule: allow - port: '80' - proto: tcp + handlers: + - name: Restart memos + systemd: + name: memos + state: restarted + +- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./memos_vars.yml + vars: + memos_subdomain: "{{ subdomains.memos }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" + memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: - name: Ensure Caddy sites-enabled directory exists file: path: "{{ caddy_sites_dir }}" @@ -153,12 +176,17 @@ state: present backup: yes - - name: Create Caddy reverse proxy configuration for memos + - name: Create Caddy reverse proxy configuration for memos (via Tailscale) copy: dest: "{{ caddy_sites_dir }}/memos.conf" content: | {{ memos_domain }} { - reverse_proxy localhost:{{ memos_port }} + reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { + # Use Tailscale MagicDNS to resolve the upstream hostname + transport http { + resolvers 100.100.100.100 + } + } } owner: root group: root @@ -167,9 +195,112 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy - handlers: - - name: Restart memos - systemd: - name: memos - state: restarted + - name: Create Uptime Kuma monitor setup script for Memos + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_memos_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + # Load configs + with open('/tmp/ansible_memos_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_url = config['monitor_url'] + monitor_name = config['monitor_name'] + + # Connect to Uptime Kuma + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name='services') + # Refresh to get the group with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing_monitor = None + for monitor in monitors: + if monitor.get('name') == monitor_name: + existing_monitor = monitor + break + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing_monitor: + print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.HTTP, + name=monitor_name, + url=monitor_url, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + error_msg = str(e) if str(e) else repr(e) + print(f"ERROR: {error_msg}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + - name: Create temporary config for monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_memos_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_url: "https://{{ memos_domain }}/healthz" + monitor_name: "Memos" + mode: '0644' + + - name: Run Uptime Kuma monitor setup + command: python3 /tmp/setup_memos_monitor.py + delegate_to: localhost + become: no + register: monitor_setup + changed_when: "'SUCCESS' in monitor_setup.stdout" + ignore_errors: yes + + - name: Clean up temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_memos_monitor.py + - /tmp/ansible_memos_config.yml diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index d027842..dcd0aac 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -1,18 +1,27 @@ -# General -memos_data_dir: /var/lib/memos +# Memos configuration +memos_version: "0.25.3" memos_port: 5230 +memos_user: "memos" +memos_data_dir: "/var/lib/memos" +memos_config_dir: "/etc/memos" +memos_bin_path: "/usr/local/bin/memos" +memos_arch: "linux_amd64" +memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_version }}/memos_v{{ memos_version }}_{{ memos_arch }}.tar.gz" -# (caddy_sites_dir and subdomain now in services_config.yml) +# Tailscale for memos-box (used by vipy Caddy proxy) +memos_tailscale_hostname: "memos-box" +memos_tailscale_ip: "100.64.0.4" -# Remote access -remote_host_name: "memos-box" -remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" -remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" -remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" -remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" +# (caddy_sites_dir and subdomain in services_config.yml) + +# Remote access (for backup from lapy) +backup_host_name: "memos_box_local" +backup_host: "{{ hostvars.get(backup_host_name, {}).get('ansible_host', backup_host_name) }}" +backup_user: "{{ hostvars.get(backup_host_name, {}).get('ansible_user', 'counterweight') }}" +backup_key_file: "{{ hostvars.get(backup_host_name, {}).get('ansible_ssh_private_key_file', '') }}" +backup_port: "{{ hostvars.get(backup_host_name, {}).get('ansible_port', 22) }}" # Local backup local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" - diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml new file mode 100644 index 0000000..3e3718e --- /dev/null +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -0,0 +1,106 @@ +- name: Configure local backup for Memos from memos-box + hosts: lapy + gather_facts: no + vars_files: + - ../../infra_vars.yml + - ./memos_vars.yml + vars: + backup_data_path: "{{ memos_data_dir }}" + + tasks: + - name: Debug remote backup vars + debug: + msg: + - "backup_host={{ backup_host }}" + - "backup_user={{ backup_user }}" + - "backup_data_path='{{ backup_data_path }}'" + - "local_backup_dir={{ local_backup_dir }}" + + - name: Ensure local backup directory exists + file: + path: "{{ local_backup_dir }}" + state: directory + mode: '0755' + + - name: Ensure ~/.local/bin exists + file: + path: "{{ lookup('env', 'HOME') }}/.local/bin" + state: directory + mode: '0755' + + - name: Create backup script + copy: + dest: "{{ backup_script_path }}" + mode: '0750' + content: | + #!/bin/bash + set -euo pipefail + + TIMESTAMP=$(date +'%Y-%m-%d') + BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" + mkdir -p "$BACKUP_DIR" + + {% if backup_key_file %} + SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" + {% else %} + SSH_CMD="ssh -p {{ backup_port }}" + {% endif %} + + rsync -az -e "$SSH_CMD" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" + + # Rotate old backups (keep 14 days) + # Calculate cutoff date (14 days ago) and delete backups older than that + CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') + for dir in "{{ local_backup_dir }}"/20*; do + if [ -d "$dir" ]; then + dir_date=$(basename "$dir") + if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then + rm -rf "$dir" + fi + fi + done + + - name: Ensure cronjob for backup exists + cron: + name: "Memos backup" + user: "{{ lookup('env', 'USER') }}" + job: "{{ backup_script_path }}" + minute: 15 + hour: "9,12,15,18" + + - name: Run the backup script to make the first backup + command: "{{ backup_script_path }}" + + - name: Verify backup was created + block: + - name: Get today's date + command: date +'%Y-%m-%d' + register: today_date + changed_when: false + + - name: Check backup directory exists and contains files + stat: + path: "{{ local_backup_dir }}/{{ today_date.stdout }}" + register: backup_dir_stat + + - name: Verify backup directory exists + assert: + that: + - backup_dir_stat.stat.exists + - backup_dir_stat.stat.isdir + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" + success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" + + - name: Check if backup directory contains files + find: + paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" + recurse: yes + register: backup_files + + - name: Verify backup directory is not empty + assert: + that: + - backup_files.files | length > 0 + fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" + success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" + From 4cc72da4da22474fef301d41a710ff65ee6320d3 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 15 Dec 2025 23:33:08 +0100 Subject: [PATCH 08/13] knots is publicly reachable --- .../deploy_bitcoin_knots_playbook.yml | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index 47c4f58..2684965 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -722,3 +722,195 @@ name: bitcoind state: restarted + +- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./bitcoin_knots_vars.yml + vars: + bitcoin_tailscale_hostname: "knots-box" + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Create Bitcoin P2P proxy socket unit + copy: + dest: /etc/systemd/system/bitcoin-p2p-proxy.socket + content: | + [Unit] + Description=Bitcoin P2P Proxy Socket + + [Socket] + ListenStream={{ bitcoin_p2p_port }} + + [Install] + WantedBy=sockets.target + owner: root + group: root + mode: '0644' + notify: Restart bitcoin-p2p-proxy socket + + - name: Create Bitcoin P2P proxy service unit + copy: + dest: /etc/systemd/system/bitcoin-p2p-proxy.service + content: | + [Unit] + Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} + Requires=bitcoin-p2p-proxy.socket + After=network.target + + [Service] + Type=notify + ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Bitcoin P2P proxy socket + systemd: + name: bitcoin-p2p-proxy.socket + enabled: yes + state: started + + - name: Allow Bitcoin P2P port through UFW + ufw: + rule: allow + port: "{{ bitcoin_p2p_port | string }}" + proto: tcp + comment: "Bitcoin P2P public access" + + - name: Verify connectivity to knots-box via Tailscale + wait_for: + host: "{{ bitcoin_tailscale_hostname }}" + port: "{{ bitcoin_p2p_port }}" + timeout: 10 + ignore_errors: yes + + - name: Display public endpoint + debug: + msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public P2P + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_bitcoin_p2p_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ bitcoin_p2p_port }} + monitor_name: "Bitcoin Knots P2P Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_bitcoin_p2p_tcp_monitor.py + - /tmp/ansible_bitcoin_p2p_config.yml + + handlers: + - name: Restart bitcoin-p2p-proxy socket + systemd: + name: bitcoin-p2p-proxy.socket + state: restarted From 859cd2d8b7ccae3ac8c00beb3787675ab77a3470 Mon Sep 17 00:00:00 2001 From: counterweight Date: Mon, 15 Dec 2025 23:33:20 +0100 Subject: [PATCH 09/13] small memos stuff --- ansible/services/memos/memos_vars.yml | 11 +++++------ ansible/services/memos/setup_backup_memos_to_lapy.yml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index dcd0aac..99618db 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -14,12 +14,11 @@ memos_tailscale_ip: "100.64.0.4" # (caddy_sites_dir and subdomain in services_config.yml) -# Remote access (for backup from lapy) -backup_host_name: "memos_box_local" -backup_host: "{{ hostvars.get(backup_host_name, {}).get('ansible_host', backup_host_name) }}" -backup_user: "{{ hostvars.get(backup_host_name, {}).get('ansible_user', 'counterweight') }}" -backup_key_file: "{{ hostvars.get(backup_host_name, {}).get('ansible_ssh_private_key_file', '') }}" -backup_port: "{{ hostvars.get(backup_host_name, {}).get('ansible_port', 22) }}" +# Remote access (for backup from lapy via Tailscale) +backup_host: "{{ memos_tailscale_hostname }}" +backup_user: "counterweight" +backup_key_file: "~/.ssh/counterganzua" +backup_port: 22 # Local backup local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml index 3e3718e..6d9c161 100644 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ b/ansible/services/memos/setup_backup_memos_to_lapy.yml @@ -46,7 +46,7 @@ SSH_CMD="ssh -p {{ backup_port }}" {% endif %} - rsync -az -e "$SSH_CMD" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" + rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" # Rotate old backups (keep 14 days) # Calculate cutoff date (14 days ago) and delete backups older than that From c6795dc5815c005b33f6348d491568e5f3c83e62 Mon Sep 17 00:00:00 2001 From: counterweight Date: Wed, 24 Dec 2025 10:27:35 +0100 Subject: [PATCH 10/13] redirect fulcrum --- .../deploy_bitcoin_knots_playbook.yml | 2 +- .../fulcrum/deploy_fulcrum_playbook.yml | 237 ++++++++++++++++++ ansible/services/fulcrum/fulcrum_vars.yml | 13 +- 3 files changed, 250 insertions(+), 2 deletions(-) diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml index 2684965..d818945 100644 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml @@ -458,7 +458,7 @@ # Check if bitcoind RPC is responding check_bitcoind() { local response - response=$(curl -s --max-time 5 \ + response=$(curl -s --max-time 30 \ --user "${RPC_USER}:${RPC_PASSWORD}" \ --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ --header 'Content-Type: application/json' \ diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml index 984555f..1255cd6 100644 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml @@ -72,6 +72,44 @@ group: "{{ fulcrum_group }}" mode: '0755' + # =========================================== + # SSL Certificate Generation + # =========================================== + - name: Check if SSL certificate already exists + stat: + path: "{{ fulcrum_ssl_cert_path }}" + register: fulcrum_ssl_cert_exists + when: fulcrum_ssl_enabled | default(false) + + - name: Generate self-signed SSL certificate for Fulcrum + command: > + openssl req -x509 -newkey rsa:4096 + -keyout {{ fulcrum_ssl_key_path }} + -out {{ fulcrum_ssl_cert_path }} + -sha256 -days {{ fulcrum_ssl_cert_days }} + -nodes + -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" + args: + creates: "{{ fulcrum_ssl_cert_path }}" + when: fulcrum_ssl_enabled | default(false) + notify: Restart fulcrum + + - name: Set SSL certificate permissions + file: + path: "{{ fulcrum_ssl_cert_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0644' + when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) + + - name: Set SSL key permissions + file: + path: "{{ fulcrum_ssl_key_path }}" + owner: "{{ fulcrum_user }}" + group: "{{ fulcrum_group }}" + mode: '0600' + when: fulcrum_ssl_enabled | default(false) + - name: Check if Fulcrum binary already exists stat: path: "{{ fulcrum_binary_path }}" @@ -140,6 +178,13 @@ peering = {{ 'true' if fulcrum_peering else 'false' }} zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} + # SSL/TLS Configuration + {% if fulcrum_ssl_enabled | default(false) %} + ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} + cert = {{ fulcrum_ssl_cert_path }} + key = {{ fulcrum_ssl_key_path }} + {% endif %} + # Anonymize client IP addresses and TxIDs in logs anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} @@ -477,3 +522,195 @@ name: fulcrum state: restarted + +- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd + hosts: vipy + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./fulcrum_vars.yml + vars: + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + + tasks: + - name: Create Fulcrum SSL proxy socket unit + copy: + dest: /etc/systemd/system/fulcrum-ssl-proxy.socket + content: | + [Unit] + Description=Fulcrum SSL Proxy Socket + + [Socket] + ListenStream={{ fulcrum_ssl_port }} + + [Install] + WantedBy=sockets.target + owner: root + group: root + mode: '0644' + notify: Restart fulcrum-ssl-proxy socket + + - name: Create Fulcrum SSL proxy service unit + copy: + dest: /etc/systemd/system/fulcrum-ssl-proxy.service + content: | + [Unit] + Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} + Requires=fulcrum-ssl-proxy.socket + After=network.target + + [Service] + Type=notify + ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start Fulcrum SSL proxy socket + systemd: + name: fulcrum-ssl-proxy.socket + enabled: yes + state: started + + - name: Allow Fulcrum SSL port through UFW + ufw: + rule: allow + port: "{{ fulcrum_ssl_port | string }}" + proto: tcp + comment: "Fulcrum SSL public access" + + - name: Verify connectivity to fulcrum-box via Tailscale + wait_for: + host: "{{ fulcrum_tailscale_hostname }}" + port: "{{ fulcrum_ssl_port }}" + timeout: 10 + ignore_errors: yes + + - name: Display public endpoint + debug: + msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" + + # =========================================== + # Uptime Kuma TCP Monitor for Public SSL Port + # =========================================== + - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL + delegate_to: localhost + become: no + copy: + dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import traceback + import yaml + from uptime_kuma_api import UptimeKumaApi, MonitorType + + try: + with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: + config = yaml.safe_load(f) + + url = config['uptime_kuma_url'] + username = config['username'] + password = config['password'] + monitor_host = config['monitor_host'] + monitor_port = config['monitor_port'] + monitor_name = config['monitor_name'] + + api = UptimeKumaApi(url, timeout=30) + api.login(username, password) + + monitors = api.get_monitors() + + # Find or create "services" group + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name='services') + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) + + # Check if monitor already exists + existing = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Get ntfy notification ID + notifications = api.get_notifications() + ntfy_notification_id = None + for notif in notifications: + if notif.get('type') == 'ntfy': + ntfy_notification_id = notif.get('id') + break + + if existing: + print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") + print("Skipping - monitor already configured") + else: + print(f"Creating TCP monitor '{monitor_name}'...") + api.add_monitor( + type=MonitorType.PORT, + name=monitor_name, + hostname=monitor_host, + port=monitor_port, + parent=group['id'], + interval=60, + maxretries=3, + retryInterval=60, + notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} + ) + + api.disconnect() + print("SUCCESS") + + except Exception as e: + print(f"ERROR: {str(e)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + sys.exit(1) + mode: '0755' + + - name: Create temporary config for TCP monitor setup + delegate_to: localhost + become: no + copy: + dest: /tmp/ansible_fulcrum_ssl_config.yml + content: | + uptime_kuma_url: "{{ uptime_kuma_api_url }}" + username: "{{ uptime_kuma_username }}" + password: "{{ uptime_kuma_password }}" + monitor_host: "{{ ansible_host }}" + monitor_port: {{ fulcrum_ssl_port }} + monitor_name: "Fulcrum SSL Public" + mode: '0644' + + - name: Run Uptime Kuma TCP monitor setup + command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py + delegate_to: localhost + become: no + register: tcp_monitor_setup + changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" + ignore_errors: yes + + - name: Display TCP monitor setup output + debug: + msg: "{{ tcp_monitor_setup.stdout_lines }}" + when: tcp_monitor_setup.stdout is defined + + - name: Clean up TCP monitor temporary files + delegate_to: localhost + become: no + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/setup_fulcrum_ssl_tcp_monitor.py + - /tmp/ansible_fulcrum_ssl_config.yml + + handlers: + - name: Restart fulcrum-ssl-proxy socket + systemd: + name: fulcrum-ssl-proxy.socket + state: restarted + diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml index 2be35d6..8486b14 100644 --- a/ansible/services/fulcrum/fulcrum_vars.yml +++ b/ansible/services/fulcrum/fulcrum_vars.yml @@ -18,13 +18,24 @@ bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port # Network - Fulcrum server fulcrum_tcp_port: 50001 -# Binding address for Fulcrum TCP server: +fulcrum_ssl_port: 50002 +# Binding address for Fulcrum TCP/SSL server: # - "127.0.0.1" = localhost only (use when Caddy is on the same box) # - "0.0.0.0" = all interfaces (use when Caddy is on a different box) # - Specific IP = bind to specific network interface fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) +fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port # If Caddy is on a different box, set this to the IP address that Caddy will use to connect +# SSL/TLS Configuration +fulcrum_ssl_enabled: true +fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" +fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" +fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert + +# Port forwarding configuration (for public access via VPS) +fulcrum_tailscale_hostname: "fulcrum-box" + # Performance # db_mem will be calculated as 75% of available RAM automatically in playbook fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache From fe321050c1431c51dbcf3acc2231e69092c98eec Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 4 Jan 2026 23:19:19 +0100 Subject: [PATCH 11/13] monitor zfs --- .../nodito/32_zfs_pool_setup_playbook.yml | 496 ++++++++++++++++++ 1 file changed, 496 insertions(+) diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 4ff0ed4..cb72328 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -170,3 +170,499 @@ fail: msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" + +- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs + hosts: nodito + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + + vars: + zfs_check_interval_seconds: 86400 # 24 hours + zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) + zfs_check_retries: 1 + zfs_monitoring_script_dir: /opt/zfs-monitoring + zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" + zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" + zfs_systemd_health_service_name: zfs-health-monitor + zfs_systemd_scrub_service_name: zfs-monthly-scrub + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "zfs-health-{{ host_name.stdout }}" + monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma ZFS health monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_zfs_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma ZFS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_zfs_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" + "{{ zfs_check_timeout_seconds }}" + "{{ zfs_check_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for ZFS monitoring + package: + name: + - curl + - jq + state: present + + - name: Create monitoring script directory + file: + path: "{{ zfs_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create ZFS health monitoring script + copy: + dest: "{{ zfs_monitoring_script_path }}" + content: | + #!/bin/bash + + # ZFS Pool Health Monitoring Script + # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy + # If any issues detected, does NOT send heartbeat (triggers timeout alert) + + LOG_FILE="{{ zfs_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" + POOL_NAME="{{ zfs_pool_name }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to check pool health using JSON output + check_pool_health() { + local pool="$1" + local issues_found=0 + + # Get pool status as JSON + local pool_json + pool_json=$(zpool status -j "$pool" 2>&1) + + if [ $? -ne 0 ]; then + log_message "ERROR: Failed to get pool status for $pool" + log_message " -> $pool_json" + return 1 + fi + + # Check 1: Pool state must be ONLINE + local pool_state + pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') + + if [ "$pool_state" != "ONLINE" ]; then + log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" + issues_found=1 + else + log_message "OK: Pool state is ONLINE" + fi + + # Check 2: Check all vdevs and devices for non-ONLINE states + local bad_states + bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.state? and .state != "ONLINE") | + "\(.name // "unknown"): \(.state)" + ' 2>/dev/null) + + if [ -n "$bad_states" ]; then + log_message "ISSUE: Found devices not in ONLINE state:" + echo "$bad_states" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: All devices are ONLINE" + fi + + # Check 3: Check for resilvering in progress + local scan_function scan_state + scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + + if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then + local resilver_progress + resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') + log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" + issues_found=1 + fi + + # Check 4: Check for read/write/checksum errors on all devices + # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison + local devices_with_errors + devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | + "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" + ' 2>/dev/null) + + if [ -n "$devices_with_errors" ]; then + log_message "ISSUE: Found devices with I/O errors:" + echo "$devices_with_errors" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: No read/write/checksum errors detected" + fi + + # Check 5: Check for scan errors (from last scrub/resilver) + local scan_errors + scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') + + if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then + log_message "ISSUE: Last scan reported $scan_errors errors" + issues_found=1 + else + log_message "OK: No scan errors" + fi + + return $issues_found + } + + # Function to get last scrub info for status message + get_scrub_info() { + local pool="$1" + local pool_json + pool_json=$(zpool status -j "$pool" 2>/dev/null) + + local scan_func scan_state scan_start + scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') + + if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then + echo "scrub in progress (started $scan_start)" + elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then + echo "last scrub: $scan_start" + else + echo "no scrub history" + fi + } + + # Function to send heartbeat to Uptime Kuma + send_heartbeat() { + local message="$1" + + log_message "Sending heartbeat to Uptime Kuma: $message" + + # URL encode the message + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent successfully (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + # Main health check logic + main() { + log_message "==========================================" + log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" + + # Run all health checks + if check_pool_health "$POOL_NAME"; then + # All checks passed - send heartbeat + local scrub_info + scrub_info=$(get_scrub_info "$POOL_NAME") + + local message="Pool $POOL_NAME healthy ($scrub_info)" + send_heartbeat "$message" + + log_message "Health check completed: ALL OK" + exit 0 + else + # Issues found - do NOT send heartbeat (will trigger timeout alert) + log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" + log_message "Uptime Kuma will alert after timeout due to missing heartbeat" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" + content: | + [Unit] + Description=ZFS Pool Health Monitor + After=zfs.target network.target + + [Service] + Type=oneshot + ExecStart={{ zfs_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for daily ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Pool Health Monitor daily + Requires={{ zfs_systemd_health_service_name }}.service + + [Timer] + OnBootSec=5min + OnUnitActiveSec={{ zfs_check_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for ZFS monthly scrub + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" + content: | + [Unit] + Description=ZFS Monthly Scrub for {{ zfs_pool_name }} + After=zfs.target + + [Service] + Type=oneshot + ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for monthly ZFS scrub + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Scrub on last day of every month at 4:00 AM + Requires={{ zfs_systemd_scrub_service_name }}.service + + [Timer] + OnCalendar=*-*~01 04:00:00 + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start ZFS health monitoring timer + systemd: + name: "{{ zfs_systemd_health_service_name }}.timer" + enabled: yes + state: started + + - name: Enable and start ZFS monthly scrub timer + systemd: + name: "{{ zfs_systemd_scrub_service_name }}.timer" + enabled: yes + state: started + + - name: Test ZFS health monitoring script + command: "{{ zfs_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "ZFS health monitoring script failed - check pool health" + + - name: Display monitoring configuration + debug: + msg: | + ✓ ZFS Pool Health Monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Pool Name: {{ zfs_pool_name }} + + Health Check: + - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) + - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) + - Script: {{ zfs_monitoring_script_path }} + - Log: {{ zfs_log_file }} + - Service: {{ zfs_systemd_health_service_name }}.service + - Timer: {{ zfs_systemd_health_service_name }}.timer + + Monthly Scrub: + - Schedule: Last day of month at 4:00 AM + - Service: {{ zfs_systemd_scrub_service_name }}.service + - Timer: {{ zfs_systemd_scrub_service_name }}.timer + + Conditions monitored: + - Pool state (must be ONLINE) + - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) + - Resilver status (alerts if resilvering) + - Read/Write/Checksum errors + - Scrub errors + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_zfs_monitor.py + state: absent + delegate_to: localhost + become: no From 08281ce349ca6dcb46f6071d105ff06ae3e6ba86 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 11 Jan 2026 22:43:27 +0100 Subject: [PATCH 12/13] ups playbook --- .../nodito/34_nut_ups_setup_playbook.yml | 569 ++++++++++++++++++ ansible/infra/nodito/nodito_vars.yml | 9 + 2 files changed, 578 insertions(+) create mode 100644 ansible/infra/nodito/34_nut_ups_setup_playbook.yml diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml new file mode 100644 index 0000000..02468d5 --- /dev/null +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -0,0 +1,569 @@ +- name: Setup NUT (Network UPS Tools) for CyberPower UPS + hosts: nodito_host + become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml + - nodito_secrets.yml + + tasks: + # ------------------------------------------------------------------ + # Installation + # ------------------------------------------------------------------ + - name: Install NUT packages + apt: + name: + - nut + - nut-client + - nut-server + state: present + update_cache: true + + # ------------------------------------------------------------------ + # Verify UPS is detected + # ------------------------------------------------------------------ + - name: Check if UPS is detected via USB + shell: lsusb | grep -i cyber + register: lsusb_output + changed_when: false + failed_when: false + + - name: Display USB detection result + debug: + msg: "{{ lsusb_output.stdout | default('UPS not detected via USB - ensure it is plugged in') }}" + + - name: Fail if UPS not detected + fail: + msg: "CyberPower UPS not detected via USB. Ensure the USB cable is connected." + when: lsusb_output.rc != 0 + + - name: Reload udev rules for USB permissions + shell: | + udevadm control --reload-rules + udevadm trigger --subsystem-match=usb --action=add + changed_when: true + + - name: Verify USB device has nut group permissions + shell: | + BUS_DEV=$(lsusb | grep -i cyber | grep -oP 'Bus \K\d+|Device \K\d+' | tr '\n' '/' | sed 's/\/$//') + if [ -n "$BUS_DEV" ]; then + BUS=$(echo $BUS_DEV | cut -d'/' -f1) + DEV=$(echo $BUS_DEV | cut -d'/' -f2) + ls -la /dev/bus/usb/$BUS/$DEV + else + echo "UPS device not found" + exit 1 + fi + register: usb_permissions + changed_when: false + + - name: Display USB permissions + debug: + msg: "{{ usb_permissions.stdout }} (should show 'root nut', not 'root root')" + + - name: Scan for UPS with nut-scanner + command: nut-scanner -U + register: nut_scanner_output + changed_when: false + failed_when: false + + - name: Display nut-scanner result + debug: + msg: "{{ nut_scanner_output.stdout_lines }}" + + # ------------------------------------------------------------------ + # Configuration files + # ------------------------------------------------------------------ + - name: Configure NUT mode (standalone) + copy: + dest: /etc/nut/nut.conf + content: | + # Managed by Ansible + MODE=standalone + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure UPS device + copy: + dest: /etc/nut/ups.conf + content: | + # Managed by Ansible + [{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsd to listen on localhost + copy: + dest: /etc/nut/upsd.conf + content: | + # Managed by Ansible + LISTEN 127.0.0.1 3493 + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsd users + copy: + dest: /etc/nut/upsd.users + content: | + # Managed by Ansible + [{{ ups_user }}] + password = {{ ups_password }} + upsmon master + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsmon + copy: + dest: /etc/nut/upsmon.conf + content: | + # Managed by Ansible + MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + + MINSUPPLIES 1 + SHUTDOWNCMD "/sbin/shutdown -h +0" + POLLFREQ 5 + POLLFREQALERT 5 + HOSTSYNC 15 + DEADTIME 15 + POWERDOWNFLAG /etc/killpower + + # Notifications + NOTIFYMSG ONLINE "UPS %s on line power" + NOTIFYMSG ONBATT "UPS %s on battery" + NOTIFYMSG LOWBATT "UPS %s battery is low" + NOTIFYMSG FSD "UPS %s: forced shutdown in progress" + NOTIFYMSG COMMOK "Communications with UPS %s established" + NOTIFYMSG COMMBAD "Communications with UPS %s lost" + NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" + NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + + # Log all events to syslog + NOTIFYFLAG ONLINE SYSLOG + NOTIFYFLAG ONBATT SYSLOG + NOTIFYFLAG LOWBATT SYSLOG + NOTIFYFLAG FSD SYSLOG + NOTIFYFLAG COMMOK SYSLOG + NOTIFYFLAG COMMBAD SYSLOG + NOTIFYFLAG SHUTDOWN SYSLOG + NOTIFYFLAG REPLBATT SYSLOG + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + # ------------------------------------------------------------------ + # Verify late-stage shutdown script + # ------------------------------------------------------------------ + - name: Verify nutshutdown script exists + stat: + path: /lib/systemd/system-shutdown/nutshutdown + register: nutshutdown_script + + - name: Warn if nutshutdown script is missing + debug: + msg: "WARNING: /lib/systemd/system-shutdown/nutshutdown not found. UPS may not cut power after shutdown." + when: not nutshutdown_script.stat.exists + + # ------------------------------------------------------------------ + # Services + # ------------------------------------------------------------------ + - name: Enable and start NUT driver enumerator + systemd: + name: nut-driver-enumerator + enabled: true + state: started + + - name: Enable and start NUT server + systemd: + name: nut-server + enabled: true + state: started + + - name: Enable and start NUT monitor + systemd: + name: nut-monitor + enabled: true + state: started + + # ------------------------------------------------------------------ + # Verification + # ------------------------------------------------------------------ + - name: Wait for NUT services to stabilize + pause: + seconds: 3 + + - name: Verify NUT can communicate with UPS + command: upsc {{ ups_name }}@localhost + register: upsc_output + changed_when: false + failed_when: upsc_output.rc != 0 + + - name: Display UPS status + debug: + msg: "{{ upsc_output.stdout_lines }}" + + - name: Get UPS status summary + shell: | + echo "Status: $(upsc {{ ups_name }}@localhost ups.status 2>/dev/null)" + echo "Battery: $(upsc {{ ups_name }}@localhost battery.charge 2>/dev/null)%" + echo "Runtime: $(upsc {{ ups_name }}@localhost battery.runtime 2>/dev/null)s" + echo "Load: $(upsc {{ ups_name }}@localhost ups.load 2>/dev/null)%" + register: ups_summary + changed_when: false + + - name: Display UPS summary + debug: + msg: "{{ ups_summary.stdout_lines }}" + + - name: Verify low battery thresholds + shell: | + echo "Runtime threshold: $(upsc {{ ups_name }}@localhost battery.runtime.low 2>/dev/null)s" + echo "Charge threshold: $(upsc {{ ups_name }}@localhost battery.charge.low 2>/dev/null)%" + register: thresholds + changed_when: false + + - name: Display low battery thresholds + debug: + msg: "{{ thresholds.stdout_lines }}" + + handlers: + - name: Restart NUT services + systemd: + name: "{{ item }}" + state: restarted + loop: + - nut-driver-enumerator + - nut-server + - nut-monitor + + +- name: Setup UPS Heartbeat Monitoring with Uptime Kuma + hosts: nodito + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + - nodito_secrets.yml + + vars: + ups_heartbeat_interval_seconds: 60 + ups_heartbeat_timeout_seconds: 120 + ups_heartbeat_retries: 1 + ups_monitoring_script_dir: /opt/ups-monitoring + ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" + ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" + ups_systemd_service_name: ups-heartbeat + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "ups-{{ host_name.stdout }}" + monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma UPS monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_ups_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + api.add_monitor(**monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma UPS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_ups_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" + "{{ ups_heartbeat_timeout_seconds }}" + "{{ ups_heartbeat_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL as fact + set_fact: + uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Install required packages for UPS monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ ups_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create UPS heartbeat monitoring script + copy: + dest: "{{ ups_monitoring_script_path }}" + content: | + #!/bin/bash + + # UPS Heartbeat Monitoring Script + # Sends heartbeat to Uptime Kuma only when UPS is on mains power + # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) + + LOG_FILE="{{ ups_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" + UPS_NAME="{{ ups_name }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + send_heartbeat() { + local message="$1" + + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent: $message (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + main() { + local status charge runtime load + + status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) + + if [ -z "$status" ]; then + log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" + exit 1 + fi + + charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) + runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) + load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) + + if [[ "$status" == *"OL"* ]]; then + local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" + send_heartbeat "$message" + exit 0 + else + log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" + content: | + [Unit] + Description=UPS Heartbeat Monitor + After=network.target nut-monitor.service + + [Service] + Type=oneshot + ExecStart={{ ups_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" + content: | + [Unit] + Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds + Requires={{ ups_systemd_service_name }}.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start UPS heartbeat timer + systemd: + name: "{{ ups_systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test UPS heartbeat script + command: "{{ ups_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "UPS heartbeat script failed - check UPS status and communication" + + - name: Display monitoring configuration + debug: + msg: + - "UPS Monitoring configured successfully" + - "" + - "NUT Configuration:" + - " UPS Name: {{ ups_name }}" + - " UPS Description: {{ ups_desc }}" + - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" + - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" + - "" + - "Uptime Kuma Monitoring:" + - " Monitor Name: {{ monitor_friendly_name }}" + - " Monitor Group: {{ uptime_kuma_monitor_group }}" + - " Push URL: {{ uptime_kuma_ups_push_url }}" + - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" + - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" + - "" + - "Scripts and Services:" + - " Script: {{ ups_monitoring_script_path }}" + - " Log: {{ ups_log_file }}" + - " Service: {{ ups_systemd_service_name }}.service" + - " Timer: {{ ups_systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_ups_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml index f9e6b0d..c0002f3 100644 --- a/ansible/infra/nodito/nodito_vars.yml +++ b/ansible/infra/nodito/nodito_vars.yml @@ -17,3 +17,12 @@ zfs_pool_name: "proxmox-tank-1" zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" + +# UPS Configuration (CyberPower CP900EPFCLCD via USB) +ups_name: cyberpower +ups_desc: "CyberPower CP900EPFCLCD" +ups_driver: usbhid-ups +ups_port: auto +ups_user: counterweight +ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power +ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power From c6e1a011672171414c3e5cd52e4afaf9ad4b1ddd Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 8 Feb 2026 18:22:31 +0100 Subject: [PATCH 13/13] thingies --- ansible/infra/910_docker_playbook.yml | 1 + ansible/infra_secrets.yml.example | 5 + ansible/services/forgejo-runner/SETUP.md | 28 ++ .../deploy_forgejo_runner_playbook.yml | 392 ++++++++++++++++++ .../forgejo-runner/forgejo_runner_vars.yml | 9 + 5 files changed, 435 insertions(+) create mode 100644 ansible/services/forgejo-runner/SETUP.md create mode 100644 ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml create mode 100644 ansible/services/forgejo-runner/forgejo_runner_vars.yml diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index 8e8e430..f137b6a 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -25,6 +25,7 @@ name: - ca-certificates - curl + - gnupg state: present - name: Create directory for Docker GPG key diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index cddc58a..14fd498 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -26,3 +26,8 @@ bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" # Mempool MariaDB credentials # Used by: services/mempool/deploy_mempool_playbook.yml mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" + +# Forgejo Runner registration token +# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml +# See: services/forgejo-runner/SETUP.md for how to obtain this token +forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md new file mode 100644 index 0000000..a66d295 --- /dev/null +++ b/ansible/services/forgejo-runner/SETUP.md @@ -0,0 +1,28 @@ +# Forgejo Runner Setup + +## Obtaining the Registration Token + +1. Log in to the Forgejo instance at `https://forgejo.contrapeso.xyz` +2. Go to **Site Administration** > **Actions** > **Runners** +3. Click **Create new runner** +4. Copy the registration token + +## Configuring the Token + +Paste the token into `ansible/infra_secrets.yml`: + +```yaml +forgejo_runner_registration_token: "YOUR_TOKEN_HERE" +``` + +## Running the Playbook + +```bash +ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +``` + +## Verifying + +1. On the VM: `systemctl status forgejo-runner` should show active +2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online +3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml new file mode 100644 index 0000000..a194178 --- /dev/null +++ b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml @@ -0,0 +1,392 @@ +- name: Install Forgejo Runner on Debian 13 + hosts: forgejo_runner_local + become: yes + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - ./forgejo_runner_vars.yml + vars: + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + healthcheck_interval_seconds: 60 + healthcheck_timeout_seconds: 90 + healthcheck_retries: 1 + healthcheck_script_dir: /opt/forgejo-runner-healthcheck + healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" + healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" + healthcheck_service_name: forgejo-runner-healthcheck + + tasks: + # ── 1. Assert Docker is available ────────────────────────────────── + - name: Check if Docker is installed + command: docker --version + register: docker_check + changed_when: false + failed_when: docker_check.rc != 0 + + - name: Fail if Docker is not available + assert: + that: + - docker_check.rc == 0 + fail_msg: > + Docker is not installed or not in PATH. + Please install Docker before running this playbook. + + # ── 2. Download forgejo-runner binary ────────────────────────────── + - name: Download forgejo-runner binary + get_url: + url: "{{ forgejo_runner_url }}" + dest: "{{ forgejo_runner_bin_path }}" + mode: '0755' + + # ── 3. Create runner system user ─────────────────────────────────── + - name: Create runner system user + user: + name: "{{ forgejo_runner_user }}" + system: yes + shell: /usr/sbin/nologin + home: "{{ forgejo_runner_dir }}" + create_home: no + groups: docker + append: yes + comment: 'Forgejo Runner' + + # ── 4. Create working directory ──────────────────────────────────── + - name: Create forgejo-runner working directory + file: + path: "{{ forgejo_runner_dir }}" + state: directory + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + mode: '0750' + + # ── 5. Generate default config ───────────────────────────────────── + - name: Check if config already exists + stat: + path: "{{ forgejo_runner_config_path }}" + register: config_stat + + - name: Generate default config + shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not config_stat.stat.exists + + - name: Set config file ownership + file: + path: "{{ forgejo_runner_config_path }}" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not config_stat.stat.exists + + # ── 6. Register runner ───────────────────────────────────────────── + - name: Check if runner is already registered + stat: + path: "{{ forgejo_runner_dir }}/.runner" + register: runner_stat + + - name: Register runner with Forgejo instance + command: > + {{ forgejo_runner_bin_path }} register --no-interactive + --instance {{ forgejo_instance_url }} + --token {{ forgejo_runner_registration_token }} + --name forgejo-runner-box + --labels "{{ forgejo_runner_labels }}" + args: + chdir: "{{ forgejo_runner_dir }}" + when: not runner_stat.stat.exists + + - name: Set runner registration file ownership + file: + path: "{{ forgejo_runner_dir }}/.runner" + owner: "{{ forgejo_runner_user }}" + group: "{{ forgejo_runner_user }}" + when: not runner_stat.stat.exists + + # ── 7. Create systemd service ────────────────────────────────────── + - name: Create forgejo-runner systemd service + copy: + dest: /etc/systemd/system/forgejo-runner.service + content: | + [Unit] + Description=Forgejo Runner + Documentation=https://forgejo.org/docs/latest/admin/actions/ + After=docker.service + Requires=docker.service + + [Service] + Type=simple + User={{ forgejo_runner_user }} + Group={{ forgejo_runner_user }} + WorkingDirectory={{ forgejo_runner_dir }} + ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + # ── 8. Reload systemd, enable and start ──────────────────────────── + - name: Reload systemd + systemd: + daemon_reload: yes + + - name: Enable and start forgejo-runner service + systemd: + name: forgejo-runner + enabled: yes + state: started + + # ── 9. Verify runner is active ───────────────────────────────────── + - name: Verify forgejo-runner is active + command: systemctl is-active forgejo-runner + register: runner_active + changed_when: false + + - name: Assert runner is running + assert: + that: + - runner_active.stdout == "active" + fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" + + # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── + - name: Create Uptime Kuma push monitor setup script + copy: + dest: /tmp/setup_forgejo_runner_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma push monitor setup + command: > + {{ ansible_playbook_python }} + /tmp/setup_forgejo_runner_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "services" + "forgejo-runner-healthcheck" + "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" + "{{ healthcheck_timeout_seconds }}" + "{{ healthcheck_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL + set_fact: + uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Create healthcheck script directory + file: + path: "{{ healthcheck_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create forgejo-runner healthcheck script + copy: + dest: "{{ healthcheck_script_path }}" + content: | + #!/bin/bash + + # Forgejo Runner Healthcheck Script + # Checks if forgejo-runner is active and pings Uptime Kuma on success + + LOG_FILE="{{ healthcheck_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + main() { + if systemctl is-active --quiet forgejo-runner; then + log_message "forgejo-runner is active, sending ping" + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Ping sent successfully (HTTP $http_code)" + else + log_message "ERROR: Failed to send ping (HTTP $http_code)" + exit 1 + fi + else + log_message "ERROR: forgejo-runner is not active" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create healthcheck systemd service + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" + content: | + [Unit] + Description=Forgejo Runner Healthcheck + After=network.target + + [Service] + Type=oneshot + ExecStart={{ healthcheck_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create healthcheck systemd timer + copy: + dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" + content: | + [Unit] + Description=Run Forgejo Runner Healthcheck every minute + Requires={{ healthcheck_service_name }}.service + + [Timer] + OnBootSec=30sec + OnUnitActiveSec={{ healthcheck_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd for healthcheck units + systemd: + daemon_reload: yes + + - name: Enable and start healthcheck timer + systemd: + name: "{{ healthcheck_service_name }}.timer" + enabled: yes + state: started + + - name: Test healthcheck script + command: "{{ healthcheck_script_path }}" + register: healthcheck_test + changed_when: false + + - name: Verify healthcheck script works + assert: + that: + - healthcheck_test.rc == 0 + fail_msg: "Healthcheck script failed to execute properly" + + - name: Display deployment summary + debug: + msg: | + Forgejo Runner deployed successfully! + + Runner Name: forgejo-runner-box + Instance: {{ forgejo_instance_url }} + Working Directory: {{ forgejo_runner_dir }} + Service: forgejo-runner.service ({{ runner_active.stdout }}) + + Healthcheck Monitor: forgejo-runner-healthcheck + Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s + Timeout: {{ healthcheck_timeout_seconds }}s + + - name: Clean up temporary monitor setup script + file: + path: /tmp/setup_forgejo_runner_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml new file mode 100644 index 0000000..e618fca --- /dev/null +++ b/ansible/services/forgejo-runner/forgejo_runner_vars.yml @@ -0,0 +1,9 @@ +forgejo_runner_version: "6.3.1" +forgejo_runner_arch: "linux-amd64" +forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" +forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" +forgejo_runner_user: "runner" +forgejo_runner_dir: "/opt/forgejo-runner" +forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" +forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" +forgejo_instance_url: "https://forgejo.contrapeso.xyz"