diff --git a/SCRIPT_PLAYBOOK_MAPPING.md b/SCRIPT_PLAYBOOK_MAPPING.md new file mode 100644 index 0000000..38189ab --- /dev/null +++ b/SCRIPT_PLAYBOOK_MAPPING.md @@ -0,0 +1,59 @@ +# Script to Playbook Mapping + +This document describes which playbooks each setup script applies to which machines. + +## Table + +| Script | Playbook | Target Machines/Groups | Notes | +|--------|----------|------------------------|-------| +| **setup_layer_0.sh** | None | N/A | Initial setup script - creates venv, config files | +| **setup_layer_1a_vps.sh** | `infra/01_user_and_access_setup_playbook.yml` | `vps` (vipy, watchtower, spacey) | Creates counterweight user, configures SSH | +| **setup_layer_1a_vps.sh** | `infra/02_firewall_and_fail2ban_playbook.yml` | `vps` (vipy, watchtower, spacey) | Configures UFW firewall and fail2ban | +| **setup_layer_1b_nodito.sh** | `infra/nodito/30_proxmox_bootstrap_playbook.yml` | `nodito_host` (nodito) | Initial Proxmox bootstrap | +| **setup_layer_1b_nodito.sh** | `infra/nodito/31_proxmox_community_repos_playbook.yml` | `nodito_host` (nodito) | Configures Proxmox community repositories | +| **setup_layer_1b_nodito.sh** | `infra/nodito/32_zfs_pool_setup_playbook.yml` | `nodito_host` (nodito) | Sets up ZFS pool on Proxmox | +| **setup_layer_1b_nodito.sh** | `infra/nodito/33_proxmox_debian_cloud_template.yml` | `nodito_host` (nodito) | Creates Debian cloud template for VMs | +| **setup_layer_2.sh** | `infra/900_install_rsync.yml` | `all` (vipy, watchtower, spacey, nodito) | Installs rsync on all machines | +| **setup_layer_2.sh** | `infra/910_docker_playbook.yml` | `all` (vipy, watchtower, spacey, nodito) | Installs Docker on all machines | +| **setup_layer_3_caddy.sh** | `services/caddy_playbook.yml` | `vps` (vipy, watchtower, spacey) | Installs and configures Caddy reverse proxy | +| **setup_layer_4_monitoring.sh** | `services/ntfy/deploy_ntfy_playbook.yml` | `watchtower` | Deploys ntfy notification service | +| **setup_layer_4_monitoring.sh** | `services/uptime_kuma/deploy_uptime_kuma_playbook.yml` | `watchtower` | Deploys Uptime Kuma monitoring | +| **setup_layer_4_monitoring.sh** | `services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml` | `lapy` (localhost) | Configures backup of Uptime Kuma to laptop | +| **setup_layer_4_monitoring.sh** | `services/ntfy/setup_ntfy_uptime_kuma_notification.yml` | `watchtower` | Configures ntfy notifications for Uptime Kuma | +| **setup_layer_5_headscale.sh** | `services/headscale/deploy_headscale_playbook.yml` | `spacey` | Deploys Headscale mesh VPN server | +| **setup_layer_5_headscale.sh** | `infra/920_join_headscale_mesh.yml` | `all` (vipy, watchtower, spacey, nodito) | Joins all machines to Headscale mesh (with --limit) | +| **setup_layer_5_headscale.sh** | `services/headscale/setup_backup_headscale_to_lapy.yml` | `lapy` (localhost) | Configures backup of Headscale to laptop | +| **setup_layer_6_infra_monitoring.sh** | `infra/410_disk_usage_alerts.yml` | `all` (vipy, watchtower, spacey, nodito, lapy) | Sets up disk usage monitoring alerts | +| **setup_layer_6_infra_monitoring.sh** | `infra/420_system_healthcheck.yml` | `all` (vipy, watchtower, spacey, nodito, lapy) | Sets up system health checks | +| **setup_layer_6_infra_monitoring.sh** | `infra/430_cpu_temp_alerts.yml` | `nodito_host` (nodito) | Sets up CPU temperature alerts for Proxmox | +| **setup_layer_7_services.sh** | `services/vaultwarden/deploy_vaultwarden_playbook.yml` | `vipy` | Deploys Vaultwarden password manager | +| **setup_layer_7_services.sh** | `services/forgejo/deploy_forgejo_playbook.yml` | `vipy` | Deploys Forgejo Git server | +| **setup_layer_7_services.sh** | `services/lnbits/deploy_lnbits_playbook.yml` | `vipy` | Deploys LNbits Lightning wallet | +| **setup_layer_7_services.sh** | `services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml` | `lapy` (localhost) | Configures backup of Vaultwarden to laptop | +| **setup_layer_7_services.sh** | `services/lnbits/setup_backup_lnbits_to_lapy.yml` | `lapy` (localhost) | Configures backup of LNbits to laptop | +| **setup_layer_8_secondary_services.sh** | `services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml` | `vipy` | Deploys emergency ntfy app | +| **setup_layer_8_secondary_services.sh** | `services/memos/deploy_memos_playbook.yml` | `memos-box` (VM on nodito) | Deploys Memos note-taking service | + +## Machine Groups Reference + +- **vps**: vipy, watchtower, spacey (VPS servers) +- **nodito_host**: nodito (Proxmox server) +- **nodito_vms**: memos-box and other VMs created on nodito +- **lapy**: localhost (your laptop) +- **all**: All machines in inventory +- **watchtower**: Single VPS for monitoring services +- **vipy**: Single VPS for main services +- **spacey**: Single VPS for Headscale +- **memos-box**: VM on nodito for Memos service + +## Notes + +- Scripts use `--limit` flag to restrict playbooks that target `all` to specific hosts +- Backup playbooks run on `lapy` (localhost) to configure backup jobs +- Some playbooks are optional and may be skipped if hosts aren't configured +- Layer 0 is a prerequisite for all other layers + + + + + diff --git a/ansible/backup.infra_vars.yml b/ansible/backup.infra_vars.yml new file mode 100644 index 0000000..952df93 --- /dev/null +++ b/ansible/backup.infra_vars.yml @@ -0,0 +1,4 @@ +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: contrapeso.xyz diff --git a/ansible/inventory.ini.example b/ansible/example.inventory.ini similarity index 100% rename from ansible/inventory.ini.example rename to ansible/example.inventory.ini diff --git a/ansible/infra/01_user_and_access_setup_playbook.yml b/ansible/infra/01_user_and_access_setup_playbook.yml index 13e5149..a86772c 100644 --- a/ansible/infra/01_user_and_access_setup_playbook.yml +++ b/ansible/infra/01_user_and_access_setup_playbook.yml @@ -1,5 +1,5 @@ -- name: Secure Debian - hosts: all +- name: Secure Debian VPS + hosts: vps vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/02_firewall_and_fail2ban_playbook.yml b/ansible/infra/02_firewall_and_fail2ban_playbook.yml index e83cbcb..b50a0e3 100644 --- a/ansible/infra/02_firewall_and_fail2ban_playbook.yml +++ b/ansible/infra/02_firewall_and_fail2ban_playbook.yml @@ -1,5 +1,5 @@ -- name: Secure Debian - hosts: all +- name: Secure Debian VPS + hosts: vps vars_files: - ../infra_vars.yml become: true diff --git a/ansible/infra/910_docker_playbook.yml b/ansible/infra/910_docker_playbook.yml index f137b6a..8e8e430 100644 --- a/ansible/infra/910_docker_playbook.yml +++ b/ansible/infra/910_docker_playbook.yml @@ -25,7 +25,6 @@ name: - ca-certificates - curl - - gnupg state: present - name: Create directory for Docker GPG key diff --git a/ansible/infra/920_join_headscale_mesh.yml b/ansible/infra/920_join_headscale_mesh.yml index 8d06d44..a0c3b5a 100644 --- a/ansible/infra/920_join_headscale_mesh.yml +++ b/ansible/infra/920_join_headscale_mesh.yml @@ -44,7 +44,7 @@ shell: > ssh {{ ssh_args }} {{ headscale_user }}@{{ headscale_host }} - "sudo headscale preauthkeys create --user {{ headscale_user_id }} --expiration 10m --output json" + "sudo headscale preauthkeys create --user {{ headscale_user_id }} --expiration 1m --output json" register: preauth_key_result changed_when: true failed_when: preauth_key_result.rc != 0 @@ -77,7 +77,7 @@ - name: Add Tailscale repository apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/debian {{ ansible_distribution_release }} main" + repo: "deb [signed-by=/etc/apt/keyrings/tailscale.gpg] https://pkgs.tailscale.com/stable/debian {{ ansible_lsb.codename }} main" state: present update_cache: yes @@ -99,8 +99,6 @@ --login-server {{ headscale_domain }} --authkey {{ auth_key }} --accept-dns=true - --hostname={{ ansible_hostname }} - --reset register: tailscale_up_result changed_when: "'already authenticated' not in tailscale_up_result.stdout" failed_when: tailscale_up_result.rc != 0 and 'already authenticated' not in tailscale_up_result.stdout @@ -109,37 +107,6 @@ pause: seconds: 2 - - name: Get node ID from headscale server - delegate_to: "{{ groups['lapy'][0] }}" - become: no - vars: - ssh_args: "{{ ('-i ' + headscale_key + ' ' if headscale_key else '') + '-p ' + headscale_port|string }}" - shell: > - ssh {{ ssh_args }} - {{ headscale_user }}@{{ headscale_host }} - "sudo headscale nodes list -o json" - register: nodes_list_result - changed_when: false - failed_when: nodes_list_result.rc != 0 - - - name: Extract node ID for this host - set_fact: - headscale_node_id: "{{ (nodes_list_result.stdout | from_json) | selectattr('given_name', 'equalto', ansible_hostname) | map(attribute='id') | first }}" - failed_when: headscale_node_id is not defined or headscale_node_id == '' - - - name: Tag node with its hostname - delegate_to: "{{ groups['lapy'][0] }}" - become: no - vars: - ssh_args: "{{ ('-i ' + headscale_key + ' ' if headscale_key else '') + '-p ' + headscale_port|string }}" - shell: > - ssh {{ ssh_args }} - {{ headscale_user }}@{{ headscale_host }} - "sudo headscale nodes tag --tags tag:{{ ansible_hostname }} -i {{ headscale_node_id }}" - register: tag_result - changed_when: true - failed_when: tag_result.rc != 0 - - name: Display Tailscale status command: tailscale status register: tailscale_status @@ -148,3 +115,4 @@ - name: Show Tailscale connection status debug: msg: "{{ tailscale_status.stdout_lines }}" + diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index cb72328..4ff0ed4 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -170,499 +170,3 @@ fail: msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" - -- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs - hosts: nodito - become: true - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml - - vars: - zfs_check_interval_seconds: 86400 # 24 hours - zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) - zfs_check_retries: 1 - zfs_monitoring_script_dir: /opt/zfs-monitoring - zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" - zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" - zfs_systemd_health_service_name: zfs-health-monitor - zfs_systemd_scrub_service_name: zfs-monthly-scrub - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "zfs-health-{{ host_name.stdout }}" - monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma ZFS health monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_zfs_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma ZFS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_zfs_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" - "{{ zfs_check_timeout_seconds }}" - "{{ zfs_check_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - - name: Install required packages for ZFS monitoring - package: - name: - - curl - - jq - state: present - - - name: Create monitoring script directory - file: - path: "{{ zfs_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create ZFS health monitoring script - copy: - dest: "{{ zfs_monitoring_script_path }}" - content: | - #!/bin/bash - - # ZFS Pool Health Monitoring Script - # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy - # If any issues detected, does NOT send heartbeat (triggers timeout alert) - - LOG_FILE="{{ zfs_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" - POOL_NAME="{{ zfs_pool_name }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to check pool health using JSON output - check_pool_health() { - local pool="$1" - local issues_found=0 - - # Get pool status as JSON - local pool_json - pool_json=$(zpool status -j "$pool" 2>&1) - - if [ $? -ne 0 ]; then - log_message "ERROR: Failed to get pool status for $pool" - log_message " -> $pool_json" - return 1 - fi - - # Check 1: Pool state must be ONLINE - local pool_state - pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') - - if [ "$pool_state" != "ONLINE" ]; then - log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" - issues_found=1 - else - log_message "OK: Pool state is ONLINE" - fi - - # Check 2: Check all vdevs and devices for non-ONLINE states - local bad_states - bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.state? and .state != "ONLINE") | - "\(.name // "unknown"): \(.state)" - ' 2>/dev/null) - - if [ -n "$bad_states" ]; then - log_message "ISSUE: Found devices not in ONLINE state:" - echo "$bad_states" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: All devices are ONLINE" - fi - - # Check 3: Check for resilvering in progress - local scan_function scan_state - scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - - if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then - local resilver_progress - resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') - log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" - issues_found=1 - fi - - # Check 4: Check for read/write/checksum errors on all devices - # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison - local devices_with_errors - devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | - "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" - ' 2>/dev/null) - - if [ -n "$devices_with_errors" ]; then - log_message "ISSUE: Found devices with I/O errors:" - echo "$devices_with_errors" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: No read/write/checksum errors detected" - fi - - # Check 5: Check for scan errors (from last scrub/resilver) - local scan_errors - scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') - - if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then - log_message "ISSUE: Last scan reported $scan_errors errors" - issues_found=1 - else - log_message "OK: No scan errors" - fi - - return $issues_found - } - - # Function to get last scrub info for status message - get_scrub_info() { - local pool="$1" - local pool_json - pool_json=$(zpool status -j "$pool" 2>/dev/null) - - local scan_func scan_state scan_start - scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') - - if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then - echo "scrub in progress (started $scan_start)" - elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then - echo "last scrub: $scan_start" - else - echo "no scrub history" - fi - } - - # Function to send heartbeat to Uptime Kuma - send_heartbeat() { - local message="$1" - - log_message "Sending heartbeat to Uptime Kuma: $message" - - # URL encode the message - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent successfully (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - # Main health check logic - main() { - log_message "==========================================" - log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" - - # Run all health checks - if check_pool_health "$POOL_NAME"; then - # All checks passed - send heartbeat - local scrub_info - scrub_info=$(get_scrub_info "$POOL_NAME") - - local message="Pool $POOL_NAME healthy ($scrub_info)" - send_heartbeat "$message" - - log_message "Health check completed: ALL OK" - exit 0 - else - # Issues found - do NOT send heartbeat (will trigger timeout alert) - log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" - log_message "Uptime Kuma will alert after timeout due to missing heartbeat" - exit 1 - fi - } - - # Run main function - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" - content: | - [Unit] - Description=ZFS Pool Health Monitor - After=zfs.target network.target - - [Service] - Type=oneshot - ExecStart={{ zfs_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for daily ZFS health monitoring - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Pool Health Monitor daily - Requires={{ zfs_systemd_health_service_name }}.service - - [Timer] - OnBootSec=5min - OnUnitActiveSec={{ zfs_check_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for ZFS monthly scrub - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" - content: | - [Unit] - Description=ZFS Monthly Scrub for {{ zfs_pool_name }} - After=zfs.target - - [Service] - Type=oneshot - ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for monthly ZFS scrub - copy: - dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Scrub on last day of every month at 4:00 AM - Requires={{ zfs_systemd_scrub_service_name }}.service - - [Timer] - OnCalendar=*-*~01 04:00:00 - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start ZFS health monitoring timer - systemd: - name: "{{ zfs_systemd_health_service_name }}.timer" - enabled: yes - state: started - - - name: Enable and start ZFS monthly scrub timer - systemd: - name: "{{ zfs_systemd_scrub_service_name }}.timer" - enabled: yes - state: started - - - name: Test ZFS health monitoring script - command: "{{ zfs_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "ZFS health monitoring script failed - check pool health" - - - name: Display monitoring configuration - debug: - msg: | - ✓ ZFS Pool Health Monitoring deployed successfully! - - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} - Pool Name: {{ zfs_pool_name }} - - Health Check: - - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) - - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) - - Script: {{ zfs_monitoring_script_path }} - - Log: {{ zfs_log_file }} - - Service: {{ zfs_systemd_health_service_name }}.service - - Timer: {{ zfs_systemd_health_service_name }}.timer - - Monthly Scrub: - - Schedule: Last day of month at 4:00 AM - - Service: {{ zfs_systemd_scrub_service_name }}.service - - Timer: {{ zfs_systemd_scrub_service_name }}.timer - - Conditions monitored: - - Pool state (must be ONLINE) - - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) - - Resilver status (alerts if resilvering) - - Read/Write/Checksum errors - - Scrub errors - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_zfs_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml deleted file mode 100644 index 02468d5..0000000 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ /dev/null @@ -1,569 +0,0 @@ -- name: Setup NUT (Network UPS Tools) for CyberPower UPS - hosts: nodito_host - become: true - vars_files: - - ../../infra_vars.yml - - nodito_vars.yml - - nodito_secrets.yml - - tasks: - # ------------------------------------------------------------------ - # Installation - # ------------------------------------------------------------------ - - name: Install NUT packages - apt: - name: - - nut - - nut-client - - nut-server - state: present - update_cache: true - - # ------------------------------------------------------------------ - # Verify UPS is detected - # ------------------------------------------------------------------ - - name: Check if UPS is detected via USB - shell: lsusb | grep -i cyber - register: lsusb_output - changed_when: false - failed_when: false - - - name: Display USB detection result - debug: - msg: "{{ lsusb_output.stdout | default('UPS not detected via USB - ensure it is plugged in') }}" - - - name: Fail if UPS not detected - fail: - msg: "CyberPower UPS not detected via USB. Ensure the USB cable is connected." - when: lsusb_output.rc != 0 - - - name: Reload udev rules for USB permissions - shell: | - udevadm control --reload-rules - udevadm trigger --subsystem-match=usb --action=add - changed_when: true - - - name: Verify USB device has nut group permissions - shell: | - BUS_DEV=$(lsusb | grep -i cyber | grep -oP 'Bus \K\d+|Device \K\d+' | tr '\n' '/' | sed 's/\/$//') - if [ -n "$BUS_DEV" ]; then - BUS=$(echo $BUS_DEV | cut -d'/' -f1) - DEV=$(echo $BUS_DEV | cut -d'/' -f2) - ls -la /dev/bus/usb/$BUS/$DEV - else - echo "UPS device not found" - exit 1 - fi - register: usb_permissions - changed_when: false - - - name: Display USB permissions - debug: - msg: "{{ usb_permissions.stdout }} (should show 'root nut', not 'root root')" - - - name: Scan for UPS with nut-scanner - command: nut-scanner -U - register: nut_scanner_output - changed_when: false - failed_when: false - - - name: Display nut-scanner result - debug: - msg: "{{ nut_scanner_output.stdout_lines }}" - - # ------------------------------------------------------------------ - # Configuration files - # ------------------------------------------------------------------ - - name: Configure NUT mode (standalone) - copy: - dest: /etc/nut/nut.conf - content: | - # Managed by Ansible - MODE=standalone - owner: root - group: nut - mode: "0640" - notify: Restart NUT services - - - name: Configure UPS device - copy: - dest: /etc/nut/ups.conf - content: | - # Managed by Ansible - [{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} - owner: root - group: nut - mode: "0640" - notify: Restart NUT services - - - name: Configure upsd to listen on localhost - copy: - dest: /etc/nut/upsd.conf - content: | - # Managed by Ansible - LISTEN 127.0.0.1 3493 - owner: root - group: nut - mode: "0640" - notify: Restart NUT services - - - name: Configure upsd users - copy: - dest: /etc/nut/upsd.users - content: | - # Managed by Ansible - [{{ ups_user }}] - password = {{ ups_password }} - upsmon master - owner: root - group: nut - mode: "0640" - notify: Restart NUT services - - - name: Configure upsmon - copy: - dest: /etc/nut/upsmon.conf - content: | - # Managed by Ansible - MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - - MINSUPPLIES 1 - SHUTDOWNCMD "/sbin/shutdown -h +0" - POLLFREQ 5 - POLLFREQALERT 5 - HOSTSYNC 15 - DEADTIME 15 - POWERDOWNFLAG /etc/killpower - - # Notifications - NOTIFYMSG ONLINE "UPS %s on line power" - NOTIFYMSG ONBATT "UPS %s on battery" - NOTIFYMSG LOWBATT "UPS %s battery is low" - NOTIFYMSG FSD "UPS %s: forced shutdown in progress" - NOTIFYMSG COMMOK "Communications with UPS %s established" - NOTIFYMSG COMMBAD "Communications with UPS %s lost" - NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" - NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - - # Log all events to syslog - NOTIFYFLAG ONLINE SYSLOG - NOTIFYFLAG ONBATT SYSLOG - NOTIFYFLAG LOWBATT SYSLOG - NOTIFYFLAG FSD SYSLOG - NOTIFYFLAG COMMOK SYSLOG - NOTIFYFLAG COMMBAD SYSLOG - NOTIFYFLAG SHUTDOWN SYSLOG - NOTIFYFLAG REPLBATT SYSLOG - owner: root - group: nut - mode: "0640" - notify: Restart NUT services - - # ------------------------------------------------------------------ - # Verify late-stage shutdown script - # ------------------------------------------------------------------ - - name: Verify nutshutdown script exists - stat: - path: /lib/systemd/system-shutdown/nutshutdown - register: nutshutdown_script - - - name: Warn if nutshutdown script is missing - debug: - msg: "WARNING: /lib/systemd/system-shutdown/nutshutdown not found. UPS may not cut power after shutdown." - when: not nutshutdown_script.stat.exists - - # ------------------------------------------------------------------ - # Services - # ------------------------------------------------------------------ - - name: Enable and start NUT driver enumerator - systemd: - name: nut-driver-enumerator - enabled: true - state: started - - - name: Enable and start NUT server - systemd: - name: nut-server - enabled: true - state: started - - - name: Enable and start NUT monitor - systemd: - name: nut-monitor - enabled: true - state: started - - # ------------------------------------------------------------------ - # Verification - # ------------------------------------------------------------------ - - name: Wait for NUT services to stabilize - pause: - seconds: 3 - - - name: Verify NUT can communicate with UPS - command: upsc {{ ups_name }}@localhost - register: upsc_output - changed_when: false - failed_when: upsc_output.rc != 0 - - - name: Display UPS status - debug: - msg: "{{ upsc_output.stdout_lines }}" - - - name: Get UPS status summary - shell: | - echo "Status: $(upsc {{ ups_name }}@localhost ups.status 2>/dev/null)" - echo "Battery: $(upsc {{ ups_name }}@localhost battery.charge 2>/dev/null)%" - echo "Runtime: $(upsc {{ ups_name }}@localhost battery.runtime 2>/dev/null)s" - echo "Load: $(upsc {{ ups_name }}@localhost ups.load 2>/dev/null)%" - register: ups_summary - changed_when: false - - - name: Display UPS summary - debug: - msg: "{{ ups_summary.stdout_lines }}" - - - name: Verify low battery thresholds - shell: | - echo "Runtime threshold: $(upsc {{ ups_name }}@localhost battery.runtime.low 2>/dev/null)s" - echo "Charge threshold: $(upsc {{ ups_name }}@localhost battery.charge.low 2>/dev/null)%" - register: thresholds - changed_when: false - - - name: Display low battery thresholds - debug: - msg: "{{ thresholds.stdout_lines }}" - - handlers: - - name: Restart NUT services - systemd: - name: "{{ item }}" - state: restarted - loop: - - nut-driver-enumerator - - nut-server - - nut-monitor - - -- name: Setup UPS Heartbeat Monitoring with Uptime Kuma - hosts: nodito - become: true - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml - - nodito_secrets.yml - - vars: - ups_heartbeat_interval_seconds: 60 - ups_heartbeat_timeout_seconds: 120 - ups_heartbeat_retries: 1 - ups_monitoring_script_dir: /opt/ups-monitoring - ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" - ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" - ups_systemd_service_name: ups-heartbeat - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - - tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "ups-{{ host_name.stdout }}" - monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma UPS monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_ups_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - api.add_monitor(**monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma UPS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_ups_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" - "{{ ups_heartbeat_timeout_seconds }}" - "{{ ups_heartbeat_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL as fact - set_fact: - uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Install required packages for UPS monitoring - package: - name: - - curl - state: present - - - name: Create monitoring script directory - file: - path: "{{ ups_monitoring_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create UPS heartbeat monitoring script - copy: - dest: "{{ ups_monitoring_script_path }}" - content: | - #!/bin/bash - - # UPS Heartbeat Monitoring Script - # Sends heartbeat to Uptime Kuma only when UPS is on mains power - # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) - - LOG_FILE="{{ ups_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" - UPS_NAME="{{ ups_name }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - send_heartbeat() { - local message="$1" - - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent: $message (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - main() { - local status charge runtime load - - status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) - - if [ -z "$status" ]; then - log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" - exit 1 - fi - - charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) - runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) - load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) - - if [[ "$status" == *"OL"* ]]; then - local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" - send_heartbeat "$message" - exit 0 - else - log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create systemd service for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" - content: | - [Unit] - Description=UPS Heartbeat Monitor - After=network.target nut-monitor.service - - [Service] - Type=oneshot - ExecStart={{ ups_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create systemd timer for UPS heartbeat - copy: - dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" - content: | - [Unit] - Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds - Requires={{ ups_systemd_service_name }}.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start UPS heartbeat timer - systemd: - name: "{{ ups_systemd_service_name }}.timer" - enabled: yes - state: started - - - name: Test UPS heartbeat script - command: "{{ ups_monitoring_script_path }}" - register: script_test - changed_when: false - - - name: Verify script execution - assert: - that: - - script_test.rc == 0 - fail_msg: "UPS heartbeat script failed - check UPS status and communication" - - - name: Display monitoring configuration - debug: - msg: - - "UPS Monitoring configured successfully" - - "" - - "NUT Configuration:" - - " UPS Name: {{ ups_name }}" - - " UPS Description: {{ ups_desc }}" - - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" - - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" - - "" - - "Uptime Kuma Monitoring:" - - " Monitor Name: {{ monitor_friendly_name }}" - - " Monitor Group: {{ uptime_kuma_monitor_group }}" - - " Push URL: {{ uptime_kuma_ups_push_url }}" - - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" - - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" - - "" - - "Scripts and Services:" - - " Script: {{ ups_monitoring_script_path }}" - - " Log: {{ ups_log_file }}" - - " Service: {{ ups_systemd_service_name }}.service" - - " Timer: {{ ups_systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_ups_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml index c0002f3..f9e6b0d 100644 --- a/ansible/infra/nodito/nodito_vars.yml +++ b/ansible/infra/nodito/nodito_vars.yml @@ -17,12 +17,3 @@ zfs_pool_name: "proxmox-tank-1" zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" - -# UPS Configuration (CyberPower CP900EPFCLCD via USB) -ups_name: cyberpower -ups_desc: "CyberPower CP900EPFCLCD" -ups_driver: usbhid-ups -ups_port: auto -ups_user: counterweight -ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power -ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power diff --git a/ansible/infra_secrets.yml.example b/ansible/infra_secrets.yml.example index 14fd498..07ee552 100644 --- a/ansible/infra_secrets.yml.example +++ b/ansible/infra_secrets.yml.example @@ -9,25 +9,3 @@ uptime_kuma_password: "your_password_here" ntfy_username: "your_ntfy_username" ntfy_password: "your_ntfy_password" - -# headscale-ui credentials -# Used for HTTP basic authentication via Caddy -# Provide either: -# - headscale_ui_password: plain text password (will be hashed automatically) -# - headscale_ui_password_hash: pre-hashed bcrypt password (more secure, use caddy hash-password to generate) - -headscale_ui_username: "admin" -headscale_ui_password: "your_secure_password_here" -# headscale_ui_password_hash: "$2a$14$..." # Optional: pre-hashed password - -bitcoin_rpc_user: "bitcoinrpc" -bitcoin_rpc_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Mempool MariaDB credentials -# Used by: services/mempool/deploy_mempool_playbook.yml -mariadb_mempool_password: "CHANGE_ME_TO_SECURE_PASSWORD" - -# Forgejo Runner registration token -# Used by: services/forgejo-runner/deploy_forgejo_runner_playbook.yml -# See: services/forgejo-runner/SETUP.md for how to obtain this token -forgejo_runner_registration_token: "YOUR_RUNNER_TOKEN_HERE" diff --git a/ansible/infra_vars.yml b/ansible/infra_vars.yml index 952df93..a719e68 100644 --- a/ansible/infra_vars.yml +++ b/ansible/infra_vars.yml @@ -1,3 +1,6 @@ +# Infrastructure Variables +# Generated by setup_layer_0.sh + new_user: counterweight ssh_port: 22 allow_ssh_from: "any" diff --git a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml b/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml deleted file mode 100644 index c9bd7ca..0000000 --- a/ansible/services/bitcoin-knots/bitcoin_knots_vars.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Bitcoin Knots Configuration Variables - -# Version - REQUIRED: Specify exact version/tag to build -bitcoin_knots_version: "v29.2.knots20251110" # Must specify exact version/tag -bitcoin_knots_version_short: "29.2.knots20251110" # Version without 'v' prefix (for tarball URLs) - -# Directories -bitcoin_knots_dir: /opt/bitcoin-knots -bitcoin_knots_source_dir: "{{ bitcoin_knots_dir }}/source" -bitcoin_data_dir: /var/lib/bitcoin # Standard location for config, logs, wallets -bitcoin_large_data_dir: /mnt/knots_data # Custom location for blockchain data (blocks, chainstate) -bitcoin_conf_dir: /etc/bitcoin - -# Network -bitcoin_rpc_port: 8332 -bitcoin_p2p_port: 8333 -bitcoin_rpc_bind: "0.0.0.0" - -# Build options -bitcoin_build_jobs: 4 # Parallel build jobs (-j flag), adjust based on CPU cores -bitcoin_build_prefix: /usr/local - -# Configuration options -bitcoin_enable_txindex: true # Set to true if transaction index needed (REQUIRED for Electrum servers like Electrs/ElectrumX) -bitcoin_max_connections: 125 -# dbcache will be calculated as 90% of host RAM automatically in playbook - -# ZMQ Configuration -bitcoin_zmq_enabled: true -bitcoin_zmq_bind: "tcp://0.0.0.0" -bitcoin_zmq_port_rawblock: 28332 -bitcoin_zmq_port_rawtx: 28333 -bitcoin_zmq_port_hashblock: 28334 -bitcoin_zmq_port_hashtx: 28335 - -# Service user -bitcoin_user: bitcoin -bitcoin_group: bitcoin diff --git a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml b/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml deleted file mode 100644 index d818945..0000000 --- a/ansible/services/bitcoin-knots/deploy_bitcoin_knots_playbook.yml +++ /dev/null @@ -1,916 +0,0 @@ -- name: Build and Deploy Bitcoin Knots from Source - hosts: knots_box_local - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml - vars: - bitcoin_repo_url: "https://github.com/bitcoinknots/bitcoin.git" - bitcoin_sigs_base_url: "https://raw.githubusercontent.com/bitcoinknots/guix.sigs/knots" - bitcoin_version_major: "{{ bitcoin_knots_version_short | regex_replace('^(\\d+)\\..*', '\\1') }}" - bitcoin_source_tarball_url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Calculate 90% of system RAM for dbcache - set_fact: - bitcoin_dbcache_mb: "{{ (ansible_memtotal_mb | float * 0.9) | int }}" - changed_when: false - - - name: Display calculated dbcache value - debug: - msg: "Setting dbcache to {{ bitcoin_dbcache_mb }} MB (90% of {{ ansible_memtotal_mb }} MB total RAM)" - - - - name: Install build dependencies - apt: - name: - - build-essential - - libtool - - autotools-dev - - automake - - pkg-config - - bsdmainutils - - python3 - - python3-pip - - libevent-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-test-dev - - libboost-thread-dev - - libboost-chrono-dev - - libboost-program-options-dev - - libboost-dev - - libssl-dev - - libdb-dev - - libminiupnpc-dev - - libzmq3-dev - - libnatpmp-dev - - libsqlite3-dev - - git - - curl - - wget - - cmake - state: present - update_cache: yes - - - name: Create bitcoin group - group: - name: "{{ bitcoin_group }}" - system: yes - state: present - - - name: Create bitcoin user - user: - name: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ bitcoin_data_dir }}" - create_home: yes - state: present - - - name: Create bitcoin-knots directory - file: - path: "{{ bitcoin_knots_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin-knots source directory - file: - path: "{{ bitcoin_knots_source_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create bitcoin data directory (for config, logs, wallets) - file: - path: "{{ bitcoin_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin large data directory (for blockchain) - file: - path: "{{ bitcoin_large_data_dir }}" - state: directory - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0750' - - - name: Create bitcoin config directory - file: - path: "{{ bitcoin_conf_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Check if bitcoind binary already exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_binary_exists - changed_when: false - - - name: Install gnupg for signature verification - apt: - name: gnupg - state: present - when: not bitcoind_binary_exists.stat.exists - - - name: Import Luke Dashjr's Bitcoin Knots signing key - command: gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 90C8019E36C2E964 - register: key_import - changed_when: "'already in secret keyring' not in key_import.stdout and 'already in public keyring' not in key_import.stdout" - when: not bitcoind_binary_exists.stat.exists - failed_when: key_import.rc != 0 - - - name: Display imported key fingerprint - command: gpg --fingerprint 90C8019E36C2E964 - register: key_fingerprint - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Download SHA256SUMS.asc signature file - get_url: - url: "https://bitcoinknots.org/files/{{ bitcoin_version_major }}.x/{{ bitcoin_knots_version_short }}/SHA256SUMS.asc" - dest: "/tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc" - mode: '0644' - when: not bitcoind_binary_exists.stat.exists - - - name: Verify PGP signature on SHA256SUMS file - command: gpg --verify /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS.asc /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS - register: sha256sums_verification - changed_when: false - failed_when: false # Don't fail here - check for 'Good signature' in next task - when: not bitcoind_binary_exists.stat.exists - - - - name: Display SHA256SUMS verification result - debug: - msg: "{{ sha256sums_verification.stdout_lines + sha256sums_verification.stderr_lines }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Fail if SHA256SUMS signature verification failed - fail: - msg: "SHA256SUMS signature verification failed. Aborting build." - when: not bitcoind_binary_exists.stat.exists and ('Good signature' not in sha256sums_verification.stdout and 'Good signature' not in sha256sums_verification.stderr) - - - name: Remove any existing tarball to force fresh download - file: - path: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Download Bitcoin Knots source tarball - get_url: - url: "{{ bitcoin_source_tarball_url }}" - dest: "/tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" - mode: '0644' - validate_certs: yes - force: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Calculate SHA256 checksum of downloaded tarball - command: sha256sum /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - register: tarball_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Extract expected checksum from SHA256SUMS file - shell: grep "bitcoin-{{ bitcoin_knots_version_short }}.tar.gz" /tmp/bitcoin-knots-{{ bitcoin_knots_version_short }}-SHA256SUMS | awk '{print $1}' - register: expected_checksum - changed_when: false - when: not bitcoind_binary_exists.stat.exists - failed_when: expected_checksum.stdout == "" - - - name: Display checksum comparison - debug: - msg: - - "Expected: {{ expected_checksum.stdout | trim }}" - - "Actual: {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists - - - name: Verify tarball checksum matches SHA256SUMS - fail: - msg: "Tarball checksum mismatch! Expected {{ expected_checksum.stdout | trim }}, got {{ tarball_checksum.stdout.split()[0] }}" - when: not bitcoind_binary_exists.stat.exists and expected_checksum.stdout | trim != tarball_checksum.stdout.split()[0] - - - name: Remove existing source directory if it exists (to force fresh extraction) - file: - path: "{{ bitcoin_knots_source_dir }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Remove extracted directory if it exists (from previous runs) - file: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - state: absent - when: not bitcoind_binary_exists.stat.exists - - - name: Extract verified source tarball - unarchive: - src: /tmp/bitcoin-{{ bitcoin_knots_version_short }}.tar.gz - dest: "{{ bitcoin_knots_dir }}" - remote_src: yes - when: not bitcoind_binary_exists.stat.exists - - - name: Check if extracted directory exists - stat: - path: "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" - register: extracted_dir_stat - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Rename extracted directory to expected name - command: mv "{{ bitcoin_knots_dir }}/bitcoin-{{ bitcoin_knots_version_short }}" "{{ bitcoin_knots_source_dir }}" - when: not bitcoind_binary_exists.stat.exists and extracted_dir_stat.stat.exists - - - name: Check if CMakeLists.txt exists - stat: - path: "{{ bitcoin_knots_source_dir }}/CMakeLists.txt" - register: cmake_exists - changed_when: false - when: not bitcoind_binary_exists.stat.exists - - - name: Create CMake build directory - file: - path: "{{ bitcoin_knots_source_dir }}/build" - state: directory - mode: '0755' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - - - name: Configure Bitcoin Knots build with CMake - command: > - cmake - -DCMAKE_INSTALL_PREFIX={{ bitcoin_build_prefix }} - -DBUILD_BITCOIN_WALLET=OFF - -DCMAKE_BUILD_TYPE=Release - -DWITH_ZMQ=ON - .. - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: configure_result - changed_when: true - - - name: Verify CMake enabled ZMQ - shell: | - set -e - cd "{{ bitcoin_knots_source_dir }}/build" - cmake -LAH .. | grep -iE 'ZMQ|WITH_ZMQ|ENABLE_ZMQ|USE_ZMQ' - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - register: zmq_check - changed_when: false - - - name: Fail if CMakeLists.txt not found - fail: - msg: "CMakeLists.txt not found in {{ bitcoin_knots_source_dir }}. Cannot build Bitcoin Knots." - when: not bitcoind_binary_exists.stat.exists and not (cmake_exists.stat.exists | default(false)) - - - name: Build Bitcoin Knots with CMake (this may take 30-60+ minutes) - command: cmake --build . -j{{ bitcoin_build_jobs }} - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - async: 3600 - poll: 0 - register: build_result - changed_when: true - - - name: Check build status - async_status: - jid: "{{ build_result.ansible_job_id }}" - register: build_job_result - until: build_job_result.finished - retries: 120 - delay: 60 - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined - - - name: Fail if build failed - fail: - msg: "Bitcoin Knots build failed: {{ build_job_result.msg }}" - when: not bitcoind_binary_exists.stat.exists and build_result.ansible_job_id is defined and build_job_result.failed | default(false) - - - name: Install Bitcoin Knots binaries - command: cmake --install . - args: - chdir: "{{ bitcoin_knots_source_dir }}/build" - when: not bitcoind_binary_exists.stat.exists and cmake_exists.stat.exists | default(false) - changed_when: true - - - name: Verify bitcoind binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoind" - register: bitcoind_installed - changed_when: false - - - name: Verify bitcoin-cli binary exists - stat: - path: "{{ bitcoin_build_prefix }}/bin/bitcoin-cli" - register: bitcoin_cli_installed - changed_when: false - - - name: Fail if binaries not found - fail: - msg: "Bitcoin Knots binaries not found after installation" - when: not bitcoind_installed.stat.exists or not bitcoin_cli_installed.stat.exists - - - name: Create bitcoin.conf configuration file - copy: - dest: "{{ bitcoin_conf_dir }}/bitcoin.conf" - content: | - # Bitcoin Knots Configuration - # Generated by Ansible - - # Data directory (blockchain storage) - datadir={{ bitcoin_large_data_dir }} - - # RPC Configuration - server=1 - rpcuser={{ bitcoin_rpc_user }} - rpcpassword={{ bitcoin_rpc_password }} - rpcbind={{ bitcoin_rpc_bind }} - rpcport={{ bitcoin_rpc_port }} - rpcallowip=0.0.0.0/0 - - # Network Configuration - listen=1 - port={{ bitcoin_p2p_port }} - maxconnections={{ bitcoin_max_connections }} - - # Performance - dbcache={{ bitcoin_dbcache_mb }} - - # Transaction Index (optional) - {% if bitcoin_enable_txindex %} - txindex=1 - {% endif %} - - # Logging (to journald via systemd) - logtimestamps=1 - printtoconsole=1 - - # ZMQ Configuration - {% if bitcoin_zmq_enabled | default(false) %} - zmqpubrawblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawblock }} - zmqpubrawtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_rawtx }} - zmqpubhashblock={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashblock }} - zmqpubhashtx={{ bitcoin_zmq_bind }}:{{ bitcoin_zmq_port_hashtx }} - {% endif %} - - # Security - disablewallet=1 - owner: "{{ bitcoin_user }}" - group: "{{ bitcoin_group }}" - mode: '0640' - notify: Restart bitcoind - - - name: Create systemd service file for bitcoind - copy: - dest: /etc/systemd/system/bitcoind.service - content: | - [Unit] - Description=Bitcoin Knots daemon - After=network.target - - [Service] - Type=simple - User={{ bitcoin_user }} - Group={{ bitcoin_group }} - ExecStart={{ bitcoin_build_prefix }}/bin/bitcoind -conf={{ bitcoin_conf_dir }}/bitcoin.conf - Restart=always - RestartSec=10 - TimeoutStopSec=600 - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoind - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start bitcoind service - systemd: - name: bitcoind - enabled: yes - state: started - - - name: Wait for bitcoind RPC to be available - uri: - url: "http://{{ bitcoin_rpc_bind }}:{{ bitcoin_rpc_port }}" - method: POST - body_format: json - body: - jsonrpc: "1.0" - id: "healthcheck" - method: "getblockchaininfo" - params: [] - user: "{{ bitcoin_rpc_user }}" - password: "{{ bitcoin_rpc_password }}" - status_code: 200 - timeout: 10 - register: rpc_check - until: rpc_check.status == 200 - retries: 30 - delay: 5 - ignore_errors: yes - - - name: Display RPC connection status - debug: - msg: "Bitcoin Knots RPC is {{ 'available' if rpc_check.status == 200 else 'not yet available' }}" - - - name: Create Bitcoin Knots health check and push script - copy: - dest: /usr/local/bin/bitcoin-knots-healthcheck-push.sh - content: | - #!/bin/bash - # - # Bitcoin Knots Health Check and Push to Uptime Kuma - # Checks if bitcoind RPC is responding and pushes status to Uptime Kuma - # - - RPC_HOST="{{ bitcoin_rpc_bind }}" - RPC_PORT={{ bitcoin_rpc_port }} - RPC_USER="{{ bitcoin_rpc_user }}" - RPC_PASSWORD="{{ bitcoin_rpc_password }}" - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if bitcoind RPC is responding - check_bitcoind() { - local response - response=$(curl -s --max-time 30 \ - --user "${RPC_USER}:${RPC_PASSWORD}" \ - --data-binary '{"jsonrpc":"1.0","id":"healthcheck","method":"getblockchaininfo","params":[]}' \ - --header 'Content-Type: application/json' \ - "http://${RPC_HOST}:${RPC_PORT}" 2>&1) - - if [ $? -eq 0 ]; then - # Check if response contains a non-null error - # Successful responses have "error": null, failures have "error": {...} - if echo "$response" | grep -q '"error":null\|"error": null'; then - return 0 - else - return 1 - fi - else - return 1 - fi - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_bitcoind; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "bitcoind RPC not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Install curl for health check script - apt: - name: curl - state: present - - - name: Create systemd timer for Bitcoin Knots health check - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.timer - content: | - [Unit] - Description=Bitcoin Knots Health Check Timer - Requires=bitcoind.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Bitcoin Knots health check - copy: - dest: /etc/systemd/system/bitcoin-knots-healthcheck.service - content: | - [Unit] - Description=Bitcoin Knots Health Check and Push to Uptime Kuma - After=network.target bitcoind.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/bitcoin-knots-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin Knots health check timer - systemd: - name: bitcoin-knots-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Bitcoin Knots - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_knots_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Bitcoin Knots" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_bitcoin_knots_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/bitcoin-knots-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: bitcoin-knots-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_knots_monitor.py - - /tmp/ansible_config.yml - - handlers: - - name: Restart bitcoind - systemd: - name: bitcoind - state: restarted - - -- name: Setup public Bitcoin P2P forwarding on vipy via systemd-socket-proxyd - hosts: vipy - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./bitcoin_knots_vars.yml - vars: - bitcoin_tailscale_hostname: "knots-box" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Create Bitcoin P2P proxy socket unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.socket - content: | - [Unit] - Description=Bitcoin P2P Proxy Socket - - [Socket] - ListenStream={{ bitcoin_p2p_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart bitcoin-p2p-proxy socket - - - name: Create Bitcoin P2P proxy service unit - copy: - dest: /etc/systemd/system/bitcoin-p2p-proxy.service - content: | - [Unit] - Description=Bitcoin P2P Proxy to {{ bitcoin_tailscale_hostname }} - Requires=bitcoin-p2p-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ bitcoin_tailscale_hostname }}:{{ bitcoin_p2p_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Bitcoin P2P proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - enabled: yes - state: started - - - name: Allow Bitcoin P2P port through UFW - ufw: - rule: allow - port: "{{ bitcoin_p2p_port | string }}" - proto: tcp - comment: "Bitcoin P2P public access" - - - name: Verify connectivity to knots-box via Tailscale - wait_for: - host: "{{ bitcoin_tailscale_hostname }}" - port: "{{ bitcoin_p2p_port }}" - timeout: 10 - ignore_errors: yes - - - name: Display public endpoint - debug: - msg: "Bitcoin P2P public endpoint: {{ ansible_host }}:{{ bitcoin_p2p_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public P2P - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Bitcoin P2P - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_bitcoin_p2p_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_bitcoin_p2p_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_bitcoin_p2p_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ bitcoin_p2p_port }} - monitor_name: "Bitcoin Knots P2P Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - command: python3 /tmp/setup_bitcoin_p2p_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_bitcoin_p2p_tcp_monitor.py - - /tmp/ansible_bitcoin_p2p_config.yml - - handlers: - - name: Restart bitcoin-p2p-proxy socket - systemd: - name: bitcoin-p2p-proxy.socket - state: restarted diff --git a/ansible/services/forgejo-runner/SETUP.md b/ansible/services/forgejo-runner/SETUP.md deleted file mode 100644 index a66d295..0000000 --- a/ansible/services/forgejo-runner/SETUP.md +++ /dev/null @@ -1,28 +0,0 @@ -# Forgejo Runner Setup - -## Obtaining the Registration Token - -1. Log in to the Forgejo instance at `https://forgejo.contrapeso.xyz` -2. Go to **Site Administration** > **Actions** > **Runners** -3. Click **Create new runner** -4. Copy the registration token - -## Configuring the Token - -Paste the token into `ansible/infra_secrets.yml`: - -```yaml -forgejo_runner_registration_token: "YOUR_TOKEN_HERE" -``` - -## Running the Playbook - -```bash -ansible-playbook ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml -``` - -## Verifying - -1. On the VM: `systemctl status forgejo-runner` should show active -2. In Forgejo: **Site Administration** > **Actions** > **Runners** should show the runner as online -3. In Uptime Kuma: the `forgejo-runner-healthcheck` push monitor should be receiving pings diff --git a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml b/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml deleted file mode 100644 index a194178..0000000 --- a/ansible/services/forgejo-runner/deploy_forgejo_runner_playbook.yml +++ /dev/null @@ -1,392 +0,0 @@ -- name: Install Forgejo Runner on Debian 13 - hosts: forgejo_runner_local - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./forgejo_runner_vars.yml - vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" - healthcheck_interval_seconds: 60 - healthcheck_timeout_seconds: 90 - healthcheck_retries: 1 - healthcheck_script_dir: /opt/forgejo-runner-healthcheck - healthcheck_script_path: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.sh" - healthcheck_log_file: "{{ healthcheck_script_dir }}/forgejo_runner_healthcheck.log" - healthcheck_service_name: forgejo-runner-healthcheck - - tasks: - # ── 1. Assert Docker is available ────────────────────────────────── - - name: Check if Docker is installed - command: docker --version - register: docker_check - changed_when: false - failed_when: docker_check.rc != 0 - - - name: Fail if Docker is not available - assert: - that: - - docker_check.rc == 0 - fail_msg: > - Docker is not installed or not in PATH. - Please install Docker before running this playbook. - - # ── 2. Download forgejo-runner binary ────────────────────────────── - - name: Download forgejo-runner binary - get_url: - url: "{{ forgejo_runner_url }}" - dest: "{{ forgejo_runner_bin_path }}" - mode: '0755' - - # ── 3. Create runner system user ─────────────────────────────────── - - name: Create runner system user - user: - name: "{{ forgejo_runner_user }}" - system: yes - shell: /usr/sbin/nologin - home: "{{ forgejo_runner_dir }}" - create_home: no - groups: docker - append: yes - comment: 'Forgejo Runner' - - # ── 4. Create working directory ──────────────────────────────────── - - name: Create forgejo-runner working directory - file: - path: "{{ forgejo_runner_dir }}" - state: directory - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - mode: '0750' - - # ── 5. Generate default config ───────────────────────────────────── - - name: Check if config already exists - stat: - path: "{{ forgejo_runner_config_path }}" - register: config_stat - - - name: Generate default config - shell: "{{ forgejo_runner_bin_path }} generate-config > {{ forgejo_runner_config_path }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not config_stat.stat.exists - - - name: Set config file ownership - file: - path: "{{ forgejo_runner_config_path }}" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not config_stat.stat.exists - - # ── 6. Register runner ───────────────────────────────────────────── - - name: Check if runner is already registered - stat: - path: "{{ forgejo_runner_dir }}/.runner" - register: runner_stat - - - name: Register runner with Forgejo instance - command: > - {{ forgejo_runner_bin_path }} register --no-interactive - --instance {{ forgejo_instance_url }} - --token {{ forgejo_runner_registration_token }} - --name forgejo-runner-box - --labels "{{ forgejo_runner_labels }}" - args: - chdir: "{{ forgejo_runner_dir }}" - when: not runner_stat.stat.exists - - - name: Set runner registration file ownership - file: - path: "{{ forgejo_runner_dir }}/.runner" - owner: "{{ forgejo_runner_user }}" - group: "{{ forgejo_runner_user }}" - when: not runner_stat.stat.exists - - # ── 7. Create systemd service ────────────────────────────────────── - - name: Create forgejo-runner systemd service - copy: - dest: /etc/systemd/system/forgejo-runner.service - content: | - [Unit] - Description=Forgejo Runner - Documentation=https://forgejo.org/docs/latest/admin/actions/ - After=docker.service - Requires=docker.service - - [Service] - Type=simple - User={{ forgejo_runner_user }} - Group={{ forgejo_runner_user }} - WorkingDirectory={{ forgejo_runner_dir }} - ExecStart={{ forgejo_runner_bin_path }} daemon --config {{ forgejo_runner_config_path }} - Restart=on-failure - RestartSec=10 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - # ── 8. Reload systemd, enable and start ──────────────────────────── - - name: Reload systemd - systemd: - daemon_reload: yes - - - name: Enable and start forgejo-runner service - systemd: - name: forgejo-runner - enabled: yes - state: started - - # ── 9. Verify runner is active ───────────────────────────────────── - - name: Verify forgejo-runner is active - command: systemctl is-active forgejo-runner - register: runner_active - changed_when: false - - - name: Assert runner is running - assert: - that: - - runner_active.stdout == "active" - fail_msg: "forgejo-runner service is not active: {{ runner_active.stdout }}" - - # ── 10. Set up Uptime Kuma push monitor ──────────────────────────── - - name: Create Uptime Kuma push monitor setup script - copy: - dest: /tmp/setup_forgejo_runner_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=60, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma push monitor setup - command: > - {{ ansible_playbook_python }} - /tmp/setup_forgejo_runner_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "services" - "forgejo-runner-healthcheck" - "Forgejo Runner healthcheck - ping every {{ healthcheck_interval_seconds }}s" - "{{ healthcheck_timeout_seconds }}" - "{{ healthcheck_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL - set_fact: - uptime_kuma_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - - name: Create healthcheck script directory - file: - path: "{{ healthcheck_script_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Create forgejo-runner healthcheck script - copy: - dest: "{{ healthcheck_script_path }}" - content: | - #!/bin/bash - - # Forgejo Runner Healthcheck Script - # Checks if forgejo-runner is active and pings Uptime Kuma on success - - LOG_FILE="{{ healthcheck_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_push_url }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - main() { - if systemctl is-active --quiet forgejo-runner; then - log_message "forgejo-runner is active, sending ping" - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=forgejo-runner%20is%20active" 2>&1) - http_code=$(echo "$response" | tail -n1) - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Ping sent successfully (HTTP $http_code)" - else - log_message "ERROR: Failed to send ping (HTTP $http_code)" - exit 1 - fi - else - log_message "ERROR: forgejo-runner is not active" - exit 1 - fi - } - - main - owner: root - group: root - mode: '0755' - - - name: Create healthcheck systemd service - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service" - content: | - [Unit] - Description=Forgejo Runner Healthcheck - After=network.target - - [Service] - Type=oneshot - ExecStart={{ healthcheck_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Create healthcheck systemd timer - copy: - dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer" - content: | - [Unit] - Description=Run Forgejo Runner Healthcheck every minute - Requires={{ healthcheck_service_name }}.service - - [Timer] - OnBootSec=30sec - OnUnitActiveSec={{ healthcheck_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd for healthcheck units - systemd: - daemon_reload: yes - - - name: Enable and start healthcheck timer - systemd: - name: "{{ healthcheck_service_name }}.timer" - enabled: yes - state: started - - - name: Test healthcheck script - command: "{{ healthcheck_script_path }}" - register: healthcheck_test - changed_when: false - - - name: Verify healthcheck script works - assert: - that: - - healthcheck_test.rc == 0 - fail_msg: "Healthcheck script failed to execute properly" - - - name: Display deployment summary - debug: - msg: | - Forgejo Runner deployed successfully! - - Runner Name: forgejo-runner-box - Instance: {{ forgejo_instance_url }} - Working Directory: {{ forgejo_runner_dir }} - Service: forgejo-runner.service ({{ runner_active.stdout }}) - - Healthcheck Monitor: forgejo-runner-healthcheck - Healthcheck Interval: Every {{ healthcheck_interval_seconds }}s - Timeout: {{ healthcheck_timeout_seconds }}s - - - name: Clean up temporary monitor setup script - file: - path: /tmp/setup_forgejo_runner_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/services/forgejo-runner/forgejo_runner_vars.yml b/ansible/services/forgejo-runner/forgejo_runner_vars.yml deleted file mode 100644 index e618fca..0000000 --- a/ansible/services/forgejo-runner/forgejo_runner_vars.yml +++ /dev/null @@ -1,9 +0,0 @@ -forgejo_runner_version: "6.3.1" -forgejo_runner_arch: "linux-amd64" -forgejo_runner_url: "https://code.forgejo.org/forgejo/runner/releases/download/v{{ forgejo_runner_version }}/forgejo-runner-{{ forgejo_runner_version }}-{{ forgejo_runner_arch }}" -forgejo_runner_bin_path: "/usr/local/bin/forgejo-runner" -forgejo_runner_user: "runner" -forgejo_runner_dir: "/opt/forgejo-runner" -forgejo_runner_config_path: "{{ forgejo_runner_dir }}/config.yml" -forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm" -forgejo_instance_url: "https://forgejo.contrapeso.xyz" diff --git a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml b/ansible/services/fulcrum/deploy_fulcrum_playbook.yml deleted file mode 100644 index 1255cd6..0000000 --- a/ansible/services/fulcrum/deploy_fulcrum_playbook.yml +++ /dev/null @@ -1,716 +0,0 @@ -- name: Deploy Fulcrum Electrum Server - hosts: fulcrum_box_local - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml - vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Calculate 75% of system RAM for db_mem - set_fact: - fulcrum_db_mem_mb: "{{ (ansible_memtotal_mb | float * fulcrum_db_mem_percent) | int }}" - changed_when: false - - - name: Display calculated db_mem value - debug: - msg: "Setting db_mem to {{ fulcrum_db_mem_mb }} MB ({{ (fulcrum_db_mem_percent * 100) | int }}% of {{ ansible_memtotal_mb }} MB total RAM)" - - - name: Display Fulcrum version to install - debug: - msg: "Installing Fulcrum version {{ fulcrum_version }}" - - - name: Install required packages - apt: - name: - - curl - - wget - - openssl - state: present - update_cache: yes - - - name: Create fulcrum group - group: - name: "{{ fulcrum_group }}" - system: yes - state: present - - - name: Create fulcrum user - user: - name: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - system: yes - shell: /usr/sbin/nologin - home: /home/{{ fulcrum_user }} - create_home: yes - state: present - - - name: Create Fulcrum database directory (heavy data on special mount) - file: - path: "{{ fulcrum_db_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum config directory - file: - path: "{{ fulcrum_config_dir }}" - state: directory - owner: root - group: "{{ fulcrum_group }}" - mode: '0755' - - - name: Create Fulcrum lib directory (for banner and other data files) - file: - path: "{{ fulcrum_lib_dir }}" - state: directory - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0755' - - # =========================================== - # SSL Certificate Generation - # =========================================== - - name: Check if SSL certificate already exists - stat: - path: "{{ fulcrum_ssl_cert_path }}" - register: fulcrum_ssl_cert_exists - when: fulcrum_ssl_enabled | default(false) - - - name: Generate self-signed SSL certificate for Fulcrum - command: > - openssl req -x509 -newkey rsa:4096 - -keyout {{ fulcrum_ssl_key_path }} - -out {{ fulcrum_ssl_cert_path }} - -sha256 -days {{ fulcrum_ssl_cert_days }} - -nodes - -subj "/C=XX/ST=Decentralized/L=Bitcoin/O=Fulcrum/OU=Electrum/CN=fulcrum.local" - args: - creates: "{{ fulcrum_ssl_cert_path }}" - when: fulcrum_ssl_enabled | default(false) - notify: Restart fulcrum - - - name: Set SSL certificate permissions - file: - path: "{{ fulcrum_ssl_cert_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - when: fulcrum_ssl_enabled | default(false) and fulcrum_ssl_cert_exists.stat.exists | default(false) or fulcrum_ssl_enabled | default(false) - - - name: Set SSL key permissions - file: - path: "{{ fulcrum_ssl_key_path }}" - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0600' - when: fulcrum_ssl_enabled | default(false) - - - name: Check if Fulcrum binary already exists - stat: - path: "{{ fulcrum_binary_path }}" - register: fulcrum_binary_exists - changed_when: false - - - name: Download Fulcrum binary tarball - get_url: - url: "https://github.com/cculianu/Fulcrum/releases/download/v{{ fulcrum_version }}/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - mode: '0644' - when: not fulcrum_binary_exists.stat.exists - - - name: Extract Fulcrum binary - unarchive: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz" - dest: "/tmp" - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Install Fulcrum binary - copy: - src: "/tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux/Fulcrum" - dest: "{{ fulcrum_binary_path }}" - owner: root - group: root - mode: '0755' - remote_src: yes - when: not fulcrum_binary_exists.stat.exists - - - name: Verify Fulcrum binary installation - command: "{{ fulcrum_binary_path }} --version" - register: fulcrum_version_check - changed_when: false - - - name: Display Fulcrum version - debug: - msg: "{{ fulcrum_version_check.stdout_lines }}" - - - name: Create Fulcrum banner file - copy: - dest: "{{ fulcrum_lib_dir }}/fulcrum-banner.txt" - content: | - counterinfra - - PER ASPERA AD ASTRA - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0644' - - - name: Create Fulcrum configuration file - copy: - dest: "{{ fulcrum_config_dir }}/fulcrum.conf" - content: | - # Fulcrum Configuration - # Generated by Ansible - - # Bitcoin Core/Knots RPC settings - bitcoind = {{ bitcoin_rpc_host }}:{{ bitcoin_rpc_port }} - rpcuser = {{ bitcoin_rpc_user }} - rpcpassword = {{ bitcoin_rpc_password }} - - # Fulcrum server general settings - datadir = {{ fulcrum_db_dir }} - tcp = {{ fulcrum_tcp_bind }}:{{ fulcrum_tcp_port }} - peering = {{ 'true' if fulcrum_peering else 'false' }} - zmq_allow_hashtx = {{ 'true' if fulcrum_zmq_allow_hashtx else 'false' }} - - # SSL/TLS Configuration - {% if fulcrum_ssl_enabled | default(false) %} - ssl = {{ fulcrum_ssl_bind }}:{{ fulcrum_ssl_port }} - cert = {{ fulcrum_ssl_cert_path }} - key = {{ fulcrum_ssl_key_path }} - {% endif %} - - # Anonymize client IP addresses and TxIDs in logs - anon_logs = {{ 'true' if fulcrum_anon_logs else 'false' }} - - # Max RocksDB Memory in MiB - db_mem = {{ fulcrum_db_mem_mb }}.0 - - # Banner - banner = {{ fulcrum_lib_dir }}/fulcrum-banner.txt - owner: "{{ fulcrum_user }}" - group: "{{ fulcrum_group }}" - mode: '0640' - notify: Restart fulcrum - - - name: Create systemd service file for Fulcrum - copy: - dest: /etc/systemd/system/fulcrum.service - content: | - # MiniBolt: systemd unit for Fulcrum - # /etc/systemd/system/fulcrum.service - - [Unit] - Description=Fulcrum - After=network.target - - StartLimitBurst=2 - StartLimitIntervalSec=20 - - [Service] - ExecStart={{ fulcrum_binary_path }} {{ fulcrum_config_dir }}/fulcrum.conf - - User={{ fulcrum_user }} - Group={{ fulcrum_group }} - - # Process management - #################### - Type=simple - KillSignal=SIGINT - TimeoutStopSec=300 - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum service - systemd: - name: fulcrum - enabled: yes - state: started - - - name: Wait for Fulcrum to start - wait_for: - port: "{{ fulcrum_tcp_port }}" - host: "{{ fulcrum_tcp_bind }}" - delay: 5 - timeout: 30 - ignore_errors: yes - - - name: Check Fulcrum service status - systemd: - name: fulcrum - register: fulcrum_service_status - changed_when: false - - - name: Display Fulcrum service status - debug: - msg: "Fulcrum service is {{ 'running' if fulcrum_service_status.status.ActiveState == 'active' else 'not running' }}" - - - name: Create Fulcrum health check and push script - copy: - dest: /usr/local/bin/fulcrum-healthcheck-push.sh - content: | - #!/bin/bash - # - # Fulcrum Health Check and Push to Uptime Kuma - # Checks if Fulcrum TCP port is responding and pushes status to Uptime Kuma - # - - FULCRUM_HOST="{{ fulcrum_tcp_bind }}" - FULCRUM_PORT={{ fulcrum_tcp_port }} - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - # Check if Fulcrum TCP port is responding - check_fulcrum() { - # Try to connect to TCP port - timeout 5 bash -c "echo > /dev/tcp/${FULCRUM_HOST}/${FULCRUM_PORT}" 2>/dev/null - return $? - } - - # Push status to Uptime Kuma - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - - # URL encode spaces in message - local encoded_msg="${msg// /%20}" - - if ! curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${encoded_msg}&ping="; then - echo "ERROR: Failed to push to Uptime Kuma" - return 1 - fi - } - - # Main health check - if check_fulcrum; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Fulcrum TCP port not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create systemd timer for Fulcrum health check - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.timer - content: | - [Unit] - Description=Fulcrum Health Check Timer - Requires=fulcrum.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - - - name: Create systemd service for Fulcrum health check - copy: - dest: /etc/systemd/system/fulcrum-healthcheck.service - content: | - [Unit] - Description=Fulcrum Health Check and Push to Uptime Kuma - After=network.target fulcrum.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/fulcrum-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon for health check - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum health check timer - systemd: - name: fulcrum-healthcheck.timer - enabled: yes - state: started - - - name: Create Uptime Kuma push monitor setup script for Fulcrum - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - push_token = existing_monitor.get('pushToken') or existing_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if not push_token: - raise ValueError("Could not find push token for new monitor") - push_url = f"{url}/api/push/{push_token}" - print(f"Push URL: {push_url}") - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_name: "Fulcrum" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_fulcrum_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Extract push URL from monitor setup output - set_fact: - uptime_kuma_push_url: "{{ monitor_setup.stdout | regex_search('Push URL: (https?://[^\\s]+)', '\\1') | first | default('') }}" - delegate_to: localhost - become: no - when: monitor_setup.stdout is defined - - - name: Display extracted push URL - debug: - msg: "Uptime Kuma Push URL: {{ uptime_kuma_push_url }}" - when: uptime_kuma_push_url | default('') != '' - - - name: Set push URL in systemd service environment - lineinfile: - path: /etc/systemd/system/fulcrum-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ uptime_kuma_push_url }}" - state: present - insertafter: '^\[Service\]' - when: uptime_kuma_push_url | default('') != '' - - - name: Reload systemd daemon after push URL update - systemd: - daemon_reload: yes - when: uptime_kuma_push_url | default('') != '' - - - name: Restart health check timer to pick up new environment - systemd: - name: fulcrum-healthcheck.timer - state: restarted - when: uptime_kuma_push_url | default('') != '' - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_monitor.py - - /tmp/ansible_config.yml - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux.tar.gz - - /tmp/Fulcrum-{{ fulcrum_version }}-x86_64-linux - - handlers: - - name: Restart fulcrum - systemd: - name: fulcrum - state: restarted - - -- name: Setup public Fulcrum SSL forwarding on vipy via systemd-socket-proxyd - hosts: vipy - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./fulcrum_vars.yml - vars: - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - - name: Create Fulcrum SSL proxy socket unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.socket - content: | - [Unit] - Description=Fulcrum SSL Proxy Socket - - [Socket] - ListenStream={{ fulcrum_ssl_port }} - - [Install] - WantedBy=sockets.target - owner: root - group: root - mode: '0644' - notify: Restart fulcrum-ssl-proxy socket - - - name: Create Fulcrum SSL proxy service unit - copy: - dest: /etc/systemd/system/fulcrum-ssl-proxy.service - content: | - [Unit] - Description=Fulcrum SSL Proxy to {{ fulcrum_tailscale_hostname }} - Requires=fulcrum-ssl-proxy.socket - After=network.target - - [Service] - Type=notify - ExecStart=/lib/systemd/systemd-socket-proxyd {{ fulcrum_tailscale_hostname }}:{{ fulcrum_ssl_port }} - owner: root - group: root - mode: '0644' - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start Fulcrum SSL proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - enabled: yes - state: started - - - name: Allow Fulcrum SSL port through UFW - ufw: - rule: allow - port: "{{ fulcrum_ssl_port | string }}" - proto: tcp - comment: "Fulcrum SSL public access" - - - name: Verify connectivity to fulcrum-box via Tailscale - wait_for: - host: "{{ fulcrum_tailscale_hostname }}" - port: "{{ fulcrum_ssl_port }}" - timeout: 10 - ignore_errors: yes - - - name: Display public endpoint - debug: - msg: "Fulcrum SSL public endpoint: {{ ansible_host }}:{{ fulcrum_ssl_port }}" - - # =========================================== - # Uptime Kuma TCP Monitor for Public SSL Port - # =========================================== - - name: Create Uptime Kuma TCP monitor setup script for Fulcrum SSL - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_fulcrum_ssl_tcp_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_fulcrum_ssl_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_host = config['monitor_host'] - monitor_port = config['monitor_port'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating TCP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - hostname=monitor_host, - port=monitor_port, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for TCP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_fulcrum_ssl_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_host: "{{ ansible_host }}" - monitor_port: {{ fulcrum_ssl_port }} - monitor_name: "Fulcrum SSL Public" - mode: '0644' - - - name: Run Uptime Kuma TCP monitor setup - command: python3 /tmp/setup_fulcrum_ssl_tcp_monitor.py - delegate_to: localhost - become: no - register: tcp_monitor_setup - changed_when: "'SUCCESS' in tcp_monitor_setup.stdout" - ignore_errors: yes - - - name: Display TCP monitor setup output - debug: - msg: "{{ tcp_monitor_setup.stdout_lines }}" - when: tcp_monitor_setup.stdout is defined - - - name: Clean up TCP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_fulcrum_ssl_tcp_monitor.py - - /tmp/ansible_fulcrum_ssl_config.yml - - handlers: - - name: Restart fulcrum-ssl-proxy socket - systemd: - name: fulcrum-ssl-proxy.socket - state: restarted - diff --git a/ansible/services/fulcrum/fulcrum_vars.yml b/ansible/services/fulcrum/fulcrum_vars.yml deleted file mode 100644 index 8486b14..0000000 --- a/ansible/services/fulcrum/fulcrum_vars.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Fulcrum Configuration Variables - -# Version - Pinned to specific release -fulcrum_version: "2.1.0" # Fulcrum version to install - -# Directories -fulcrum_db_dir: /mnt/fulcrum_data/fulcrum_db # Database directory (heavy data on special mount) -fulcrum_config_dir: /etc/fulcrum # Config file location (standard OS path) -fulcrum_lib_dir: /var/lib/fulcrum # Other data files (banner, etc.) on OS disk -fulcrum_binary_path: /usr/local/bin/Fulcrum - -# Network - Bitcoin RPC connection -# Bitcoin Knots is on a different host (knots_box_local) -# Using RPC user/password authentication (credentials from infra_secrets.yml) -bitcoin_rpc_host: "192.168.1.140" # Bitcoin Knots RPC host (IP of knots_box_local) -bitcoin_rpc_port: 8332 # Bitcoin Knots RPC port -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum server -fulcrum_tcp_port: 50001 -fulcrum_ssl_port: 50002 -# Binding address for Fulcrum TCP/SSL server: -# - "127.0.0.1" = localhost only (use when Caddy is on the same box) -# - "0.0.0.0" = all interfaces (use when Caddy is on a different box) -# - Specific IP = bind to specific network interface -fulcrum_tcp_bind: "0.0.0.0" # Default: localhost (change to "0.0.0.0" if Caddy is on different box) -fulcrum_ssl_bind: "0.0.0.0" # Binding address for SSL port -# If Caddy is on a different box, set this to the IP address that Caddy will use to connect - -# SSL/TLS Configuration -fulcrum_ssl_enabled: true -fulcrum_ssl_cert_path: "{{ fulcrum_config_dir }}/fulcrum.crt" -fulcrum_ssl_key_path: "{{ fulcrum_config_dir }}/fulcrum.key" -fulcrum_ssl_cert_days: 3650 # 10 years validity for self-signed cert - -# Port forwarding configuration (for public access via VPS) -fulcrum_tailscale_hostname: "fulcrum-box" - -# Performance -# db_mem will be calculated as 75% of available RAM automatically in playbook -fulcrum_db_mem_percent: 0.75 # 75% of RAM for database cache - -# Configuration options -fulcrum_anon_logs: true # Anonymize client IPs and TxIDs in logs -fulcrum_peering: false # Disable peering with other Fulcrum servers -fulcrum_zmq_allow_hashtx: true # Allow ZMQ hashtx notifications - -# Service user -fulcrum_user: fulcrum -fulcrum_group: fulcrum - diff --git a/ansible/services/headscale/deploy_headscale_playbook.yml b/ansible/services/headscale/deploy_headscale_playbook.yml index 1bcf5bf..e8a2b37 100644 --- a/ansible/services/headscale/deploy_headscale_playbook.yml +++ b/ansible/services/headscale/deploy_headscale_playbook.yml @@ -90,7 +90,13 @@ copy: dest: /etc/headscale/acl.json content: | - {} + { + "ACLs": [], + "Groups": {}, + "Hosts": {}, + "TagOwners": {}, + "Tests": [] + } owner: headscale group: headscale mode: '0640' diff --git a/ansible/services/memos/deploy_memos_playbook.yml b/ansible/services/memos/deploy_memos_playbook.yml index da56bd6..d3276f5 100644 --- a/ansible/services/memos/deploy_memos_playbook.yml +++ b/ansible/services/memos/deploy_memos_playbook.yml @@ -1,111 +1,105 @@ -- name: Deploy Memos on memos-box - hosts: memos_box_local +- name: Deploy memos and configure Caddy reverse proxy + hosts: memos-box become: yes vars_files: - ../../infra_vars.yml - ../../services_config.yml - - ../../infra_secrets.yml - ./memos_vars.yml vars: memos_subdomain: "{{ subdomains.memos }}" + caddy_sites_dir: "{{ caddy_sites_dir }}" memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" tasks: - - name: Ensure required packages are installed + - name: Install required packages apt: name: - wget - - tar + - curl + - unzip state: present - update_cache: true + update_cache: yes - - name: Create memos system user + - name: Get latest memos release version + uri: + url: https://api.github.com/repos/usememos/memos/releases/latest + return_content: yes + register: memos_latest_release + + - name: Set memos version and find download URL + set_fact: + memos_version: "{{ memos_latest_release.json.tag_name | regex_replace('^v', '') }}" + + - name: Find linux-amd64 download URL + set_fact: + memos_download_url: "{{ memos_latest_release.json.assets | json_query('[?contains(name, `linux-amd64`) && (contains(name, `.tar.gz`) || contains(name, `.zip`))].browser_download_url') | first }}" + + - name: Display memos version to install + debug: + msg: "Installing memos version {{ memos_version }} from {{ memos_download_url }}" + + - name: Download memos binary + get_url: + url: "{{ memos_download_url }}" + dest: /tmp/memos_archive + mode: '0644' + register: memos_download + + - name: Extract memos binary + unarchive: + src: /tmp/memos_archive + dest: /tmp/memos_extract + remote_src: yes + creates: /tmp/memos_extract/memos + + - name: Install memos binary + copy: + src: /tmp/memos_extract/memos + dest: /usr/local/bin/memos + mode: '0755' + remote_src: yes + notify: Restart memos + + - name: Remove temporary files + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/memos_archive + - /tmp/memos_extract + + - name: Ensure memos user exists user: - name: "{{ memos_user }}" + name: memos system: yes - shell: /bin/false - home: "{{ memos_data_dir }}" - create_home: no - comment: "Memos Service" + shell: /usr/sbin/nologin + home: /var/lib/memos + create_home: yes + state: present - name: Create memos data directory file: path: "{{ memos_data_dir }}" state: directory - owner: "{{ memos_user }}" - group: "{{ memos_user }}" + owner: memos + group: memos mode: '0750' - - name: Create memos config directory - file: - path: "{{ memos_config_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Download memos binary archive - get_url: - url: "{{ memos_url }}" - dest: "/tmp/memos.tar.gz" - mode: '0644' - - - name: Extract memos binary - unarchive: - src: "/tmp/memos.tar.gz" - dest: "/tmp" - remote_src: yes - - - name: Move memos binary to /usr/local/bin - copy: - src: "/tmp/memos" - dest: "{{ memos_bin_path }}" - remote_src: yes - mode: '0755' - owner: root - group: root - - - name: Clean up temporary files - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/memos.tar.gz - - /tmp/memos - - - name: Create memos environment file - copy: - dest: "{{ memos_config_dir }}/memos.env" - content: | - MEMOS_MODE=prod - MEMOS_ADDR=0.0.0.0 - MEMOS_PORT={{ memos_port }} - MEMOS_DATA={{ memos_data_dir }} - MEMOS_DRIVER=sqlite - owner: root - group: root - mode: '0644' - notify: Restart memos - - - name: Create memos systemd service + - name: Create memos systemd service file copy: dest: /etc/systemd/system/memos.service content: | [Unit] - Description=Memos - A privacy-first, lightweight note-taking service + Description=memos service After=network.target [Service] Type=simple - User={{ memos_user }} - Group={{ memos_user }} - WorkingDirectory={{ memos_data_dir }} - EnvironmentFile={{ memos_config_dir }}/memos.env - ExecStart={{ memos_bin_path }} - Restart=always - RestartSec=3 - StandardOutput=journal - StandardError=journal + User=memos + Group=memos + ExecStart=/usr/local/bin/memos --port {{ memos_port }} --data {{ memos_data_dir }} + Restart=on-failure + RestartSec=5s [Install] WantedBy=multi-user.target @@ -114,52 +108,35 @@ mode: '0644' notify: Restart memos - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - name: Enable and start memos service systemd: name: memos enabled: yes state: started + daemon_reload: yes - name: Wait for memos to be ready uri: - url: "http://127.0.0.1:{{ memos_port }}/healthz" - method: GET + url: "http://localhost:{{ memos_port }}/api/v1/status" status_code: 200 - register: memos_health - retries: 10 - delay: 3 - until: memos_health.status == 200 + register: memos_ready + until: memos_ready.status == 200 + retries: 30 + delay: 2 + ignore_errors: yes - - name: Display memos status - debug: - msg: "Memos is running on port {{ memos_port }}. Access via Tailscale at http://{{ memos_tailscale_hostname }}:{{ memos_port }}" + - name: Allow HTTPS through UFW + ufw: + rule: allow + port: '443' + proto: tcp - handlers: - - name: Restart memos - systemd: - name: memos - state: restarted + - name: Allow HTTP through UFW (for Let's Encrypt) + ufw: + rule: allow + port: '80' + proto: tcp - -- name: Configure Caddy reverse proxy for Memos on vipy (proxying via Tailscale) - hosts: vipy - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./memos_vars.yml - vars: - memos_subdomain: "{{ subdomains.memos }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - memos_domain: "{{ memos_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - name: Ensure Caddy sites-enabled directory exists file: path: "{{ caddy_sites_dir }}" @@ -176,17 +153,12 @@ state: present backup: yes - - name: Create Caddy reverse proxy configuration for memos (via Tailscale) + - name: Create Caddy reverse proxy configuration for memos copy: dest: "{{ caddy_sites_dir }}/memos.conf" content: | {{ memos_domain }} { - reverse_proxy {{ memos_tailscale_hostname }}:{{ memos_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } + reverse_proxy localhost:{{ memos_port }} } owner: root group: root @@ -195,112 +167,9 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy - - name: Create Uptime Kuma monitor setup script for Memos - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_memos_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_memos_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' + handlers: + - name: Restart memos + systemd: + name: memos + state: restarted - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_memos_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ memos_domain }}/healthz" - monitor_name: "Memos" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_memos_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_memos_monitor.py - - /tmp/ansible_memos_config.yml diff --git a/ansible/services/memos/memos_vars.yml b/ansible/services/memos/memos_vars.yml index 99618db..d027842 100644 --- a/ansible/services/memos/memos_vars.yml +++ b/ansible/services/memos/memos_vars.yml @@ -1,26 +1,18 @@ -# Memos configuration -memos_version: "0.25.3" +# General +memos_data_dir: /var/lib/memos memos_port: 5230 -memos_user: "memos" -memos_data_dir: "/var/lib/memos" -memos_config_dir: "/etc/memos" -memos_bin_path: "/usr/local/bin/memos" -memos_arch: "linux_amd64" -memos_url: "https://github.com/usememos/memos/releases/download/v{{ memos_version }}/memos_v{{ memos_version }}_{{ memos_arch }}.tar.gz" -# Tailscale for memos-box (used by vipy Caddy proxy) -memos_tailscale_hostname: "memos-box" -memos_tailscale_ip: "100.64.0.4" +# (caddy_sites_dir and subdomain now in services_config.yml) -# (caddy_sites_dir and subdomain in services_config.yml) - -# Remote access (for backup from lapy via Tailscale) -backup_host: "{{ memos_tailscale_hostname }}" -backup_user: "counterweight" -backup_key_file: "~/.ssh/counterganzua" -backup_port: 22 +# Remote access +remote_host_name: "memos-box" +remote_host: "{{ hostvars.get(remote_host_name, {}).get('ansible_host', remote_host_name) }}" +remote_user: "{{ hostvars.get(remote_host_name, {}).get('ansible_user', 'counterweight') }}" +remote_key_file: "{{ hostvars.get(remote_host_name, {}).get('ansible_ssh_private_key_file', '') }}" +remote_port: "{{ hostvars.get(remote_host_name, {}).get('ansible_port', 22) }}" # Local backup local_backup_dir: "{{ lookup('env', 'HOME') }}/memos-backups" backup_script_path: "{{ lookup('env', 'HOME') }}/.local/bin/memos_backup.sh" + diff --git a/ansible/services/memos/setup_backup_memos_to_lapy.yml b/ansible/services/memos/setup_backup_memos_to_lapy.yml deleted file mode 100644 index 6d9c161..0000000 --- a/ansible/services/memos/setup_backup_memos_to_lapy.yml +++ /dev/null @@ -1,106 +0,0 @@ -- name: Configure local backup for Memos from memos-box - hosts: lapy - gather_facts: no - vars_files: - - ../../infra_vars.yml - - ./memos_vars.yml - vars: - backup_data_path: "{{ memos_data_dir }}" - - tasks: - - name: Debug remote backup vars - debug: - msg: - - "backup_host={{ backup_host }}" - - "backup_user={{ backup_user }}" - - "backup_data_path='{{ backup_data_path }}'" - - "local_backup_dir={{ local_backup_dir }}" - - - name: Ensure local backup directory exists - file: - path: "{{ local_backup_dir }}" - state: directory - mode: '0755' - - - name: Ensure ~/.local/bin exists - file: - path: "{{ lookup('env', 'HOME') }}/.local/bin" - state: directory - mode: '0755' - - - name: Create backup script - copy: - dest: "{{ backup_script_path }}" - mode: '0750' - content: | - #!/bin/bash - set -euo pipefail - - TIMESTAMP=$(date +'%Y-%m-%d') - BACKUP_DIR="{{ local_backup_dir }}/$TIMESTAMP" - mkdir -p "$BACKUP_DIR" - - {% if backup_key_file %} - SSH_CMD="ssh -i {{ backup_key_file }} -p {{ backup_port }}" - {% else %} - SSH_CMD="ssh -p {{ backup_port }}" - {% endif %} - - rsync -az -e "$SSH_CMD" --rsync-path="sudo rsync" --delete {{ backup_user }}@{{ backup_host }}:{{ backup_data_path }}/ "$BACKUP_DIR/" - - # Rotate old backups (keep 14 days) - # Calculate cutoff date (14 days ago) and delete backups older than that - CUTOFF_DATE=$(date -d '14 days ago' +'%Y-%m-%d') - for dir in "{{ local_backup_dir }}"/20*; do - if [ -d "$dir" ]; then - dir_date=$(basename "$dir") - if [ "$dir_date" != "$TIMESTAMP" ] && [ "$dir_date" \< "$CUTOFF_DATE" ]; then - rm -rf "$dir" - fi - fi - done - - - name: Ensure cronjob for backup exists - cron: - name: "Memos backup" - user: "{{ lookup('env', 'USER') }}" - job: "{{ backup_script_path }}" - minute: 15 - hour: "9,12,15,18" - - - name: Run the backup script to make the first backup - command: "{{ backup_script_path }}" - - - name: Verify backup was created - block: - - name: Get today's date - command: date +'%Y-%m-%d' - register: today_date - changed_when: false - - - name: Check backup directory exists and contains files - stat: - path: "{{ local_backup_dir }}/{{ today_date.stdout }}" - register: backup_dir_stat - - - name: Verify backup directory exists - assert: - that: - - backup_dir_stat.stat.exists - - backup_dir_stat.stat.isdir - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} was not created" - success_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists" - - - name: Check if backup directory contains files - find: - paths: "{{ local_backup_dir }}/{{ today_date.stdout }}" - recurse: yes - register: backup_files - - - name: Verify backup directory is not empty - assert: - that: - - backup_files.files | length > 0 - fail_msg: "Backup directory {{ local_backup_dir }}/{{ today_date.stdout }} exists but is empty" - success_msg: "Backup directory contains {{ backup_files.files | length }} file(s)" - diff --git a/ansible/services/mempool/deploy_mempool_playbook.yml b/ansible/services/mempool/deploy_mempool_playbook.yml deleted file mode 100644 index 658180a..0000000 --- a/ansible/services/mempool/deploy_mempool_playbook.yml +++ /dev/null @@ -1,751 +0,0 @@ -- name: Deploy Mempool Block Explorer with Docker - hosts: mempool_box_local - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml - vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - - tasks: - # =========================================== - # Docker Installation (from 910_docker_playbook.yml) - # =========================================== - - name: Remove old Docker-related packages - apt: - name: - - docker.io - - docker-doc - - docker-compose - - podman-docker - - containerd - - runc - state: absent - purge: yes - autoremove: yes - - - name: Update apt cache - apt: - update_cache: yes - - - name: Install prerequisites - apt: - name: - - ca-certificates - - curl - state: present - - - name: Create directory for Docker GPG key - file: - path: /etc/apt/keyrings - state: directory - mode: '0755' - - - name: Download Docker GPG key - get_url: - url: https://download.docker.com/linux/debian/gpg - dest: /etc/apt/keyrings/docker.asc - mode: '0644' - - - name: Get Debian architecture - command: dpkg --print-architecture - register: deb_arch - changed_when: false - - - name: Add Docker repository - apt_repository: - repo: "deb [arch={{ deb_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" - filename: docker - state: present - update_cache: yes - - - name: Install Docker packages - apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - - - name: Ensure Docker is started and enabled - systemd: - name: docker - enabled: yes - state: started - - - name: Add user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - - # =========================================== - # Mempool Deployment - # =========================================== - - name: Create mempool directories - file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0755' - loop: - - "{{ mempool_dir }}" - - "{{ mempool_data_dir }}" - - "{{ mempool_mysql_dir }}" - - - name: Create docker-compose.yml for Mempool - copy: - dest: "{{ mempool_dir }}/docker-compose.yml" - content: | - # All containers use host network for Tailscale MagicDNS resolution - services: - mariadb: - image: mariadb:10.11 - container_name: mempool-db - restart: unless-stopped - network_mode: host - environment: - MYSQL_DATABASE: "{{ mariadb_database }}" - MYSQL_USER: "{{ mariadb_user }}" - MYSQL_PASSWORD: "{{ mariadb_mempool_password }}" - MYSQL_ROOT_PASSWORD: "{{ mariadb_mempool_password }}" - volumes: - - {{ mempool_mysql_dir }}:/var/lib/mysql - healthcheck: - test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - mempool-backend: - image: mempool/backend:{{ mempool_version }} - container_name: mempool-backend - restart: unless-stopped - network_mode: host - environment: - # Database (localhost since all containers share host network) - DATABASE_ENABLED: "true" - DATABASE_HOST: "127.0.0.1" - DATABASE_DATABASE: "{{ mariadb_database }}" - DATABASE_USERNAME: "{{ mariadb_user }}" - DATABASE_PASSWORD: "{{ mariadb_mempool_password }}" - # Bitcoin Core/Knots (via Tailnet MagicDNS) - CORE_RPC_HOST: "{{ bitcoin_host }}" - CORE_RPC_PORT: "{{ bitcoin_rpc_port }}" - CORE_RPC_USERNAME: "{{ bitcoin_rpc_user }}" - CORE_RPC_PASSWORD: "{{ bitcoin_rpc_password }}" - # Electrum (Fulcrum via Tailnet MagicDNS) - ELECTRUM_HOST: "{{ fulcrum_host }}" - ELECTRUM_PORT: "{{ fulcrum_port }}" - ELECTRUM_TLS_ENABLED: "{{ fulcrum_tls }}" - # Mempool settings - MEMPOOL_NETWORK: "{{ mempool_network }}" - MEMPOOL_BACKEND: "electrum" - MEMPOOL_CLEAR_PROTECTION_MINUTES: "20" - MEMPOOL_INDEXING_BLOCKS_AMOUNT: "52560" - volumes: - - {{ mempool_data_dir }}:/backend/cache - depends_on: - mariadb: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8999/api/v1/backend-info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - mempool-frontend: - image: mempool/frontend:{{ mempool_version }} - container_name: mempool-frontend - restart: unless-stopped - network_mode: host - environment: - FRONTEND_HTTP_PORT: "{{ mempool_frontend_port }}" - BACKEND_MAINNET_HTTP_HOST: "127.0.0.1" - depends_on: - - mempool-backend - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:{{ mempool_frontend_port }}"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' - - - name: Pull Mempool images - command: docker compose pull - args: - chdir: "{{ mempool_dir }}" - - - name: Deploy Mempool containers with docker compose - command: docker compose up -d - args: - chdir: "{{ mempool_dir }}" - - - name: Wait for MariaDB to be healthy - command: docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db - register: mariadb_health - until: mariadb_health.stdout == 'healthy' - retries: 30 - delay: 10 - changed_when: false - - - name: Wait for Mempool backend to start - uri: - url: "http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - method: GET - status_code: 200 - timeout: 10 - register: backend_check - until: backend_check.status == 200 - retries: 30 - delay: 10 - ignore_errors: yes - - - name: Wait for Mempool frontend to be available - uri: - url: "http://localhost:{{ mempool_frontend_port }}" - method: GET - status_code: 200 - timeout: 10 - register: frontend_check - until: frontend_check.status == 200 - retries: 20 - delay: 5 - ignore_errors: yes - - - name: Display deployment status - debug: - msg: - - "Mempool deployment complete!" - - "Frontend: http://localhost:{{ mempool_frontend_port }}" - - "Backend API: http://localhost:{{ mempool_backend_port }}/api/v1/backend-info" - - "Backend check: {{ 'OK' if backend_check.status == 200 else 'Still initializing...' }}" - - "Frontend check: {{ 'OK' if frontend_check.status == 200 else 'Still initializing...' }}" - - # =========================================== - # Health Check Scripts for Uptime Kuma Push Monitors - # =========================================== - - name: Create Mempool MariaDB health check script - copy: - dest: /usr/local/bin/mempool-mariadb-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - - check_container() { - local status=$(docker inspect --format='{{ '{{' }}.State.Health.Status{{ '}}' }}' mempool-db 2>/dev/null) - [ "$status" = "healthy" ] - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_container; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "MariaDB container unhealthy" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool backend health check script - copy: - dest: /usr/local/bin/mempool-backend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - BACKEND_PORT={{ mempool_backend_port }} - - check_backend() { - curl -sf --max-time 5 "http://localhost:${BACKEND_PORT}/api/v1/backend-info" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_backend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Backend API not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - - name: Create Mempool frontend health check script - copy: - dest: /usr/local/bin/mempool-frontend-healthcheck-push.sh - content: | - #!/bin/bash - UPTIME_KUMA_PUSH_URL="${UPTIME_KUMA_PUSH_URL}" - FRONTEND_PORT={{ mempool_frontend_port }} - - check_frontend() { - curl -sf --max-time 5 "http://localhost:${FRONTEND_PORT}" > /dev/null 2>&1 - } - - push_to_uptime_kuma() { - local status=$1 - local msg=$2 - if [ -z "$UPTIME_KUMA_PUSH_URL" ]; then - echo "ERROR: UPTIME_KUMA_PUSH_URL not set" - return 1 - fi - curl -s --max-time 10 --retry 2 -o /dev/null \ - "${UPTIME_KUMA_PUSH_URL}?status=${status}&msg=${msg// /%20}&ping=" || true - } - - if check_frontend; then - push_to_uptime_kuma "up" "OK" - exit 0 - else - push_to_uptime_kuma "down" "Frontend not responding" - exit 1 - fi - owner: root - group: root - mode: '0755' - - # =========================================== - # Systemd Timers for Health Checks - # =========================================== - - name: Create systemd services for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item.name }}-healthcheck.service" - content: | - [Unit] - Description=Mempool {{ item.label }} Health Check - After=network.target docker.service - - [Service] - Type=oneshot - User=root - ExecStart=/usr/local/bin/mempool-{{ item.name }}-healthcheck-push.sh - Environment=UPTIME_KUMA_PUSH_URL= - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target - owner: root - group: root - mode: '0644' - loop: - - { name: "mariadb", label: "MariaDB" } - - { name: "backend", label: "Backend" } - - { name: "frontend", label: "Frontend" } - - - name: Create systemd timers for health checks - copy: - dest: "/etc/systemd/system/mempool-{{ item }}-healthcheck.timer" - content: | - [Unit] - Description=Mempool {{ item }} Health Check Timer - - [Timer] - OnBootSec=2min - OnUnitActiveSec=1min - Persistent=true - - [Install] - WantedBy=timers.target - owner: root - group: root - mode: '0644' - loop: - - mariadb - - backend - - frontend - - - name: Reload systemd daemon - systemd: - daemon_reload: yes - - - name: Enable and start health check timers - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - enabled: yes - state: started - loop: - - mariadb - - backend - - frontend - - # =========================================== - # Uptime Kuma Push Monitor Setup - # =========================================== - - name: Create Uptime Kuma push monitor setup script for Mempool - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_monitors.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitors_to_create = config['monitors'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - results = {} - for monitor_name in monitors_to_create: - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - push_token = existing.get('pushToken') or existing.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - else: - print(f"Creating push monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.PUSH, - name=monitor_name, - parent=group['id'], - interval=90, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - monitors = api.get_monitors() - new_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - if new_monitor: - push_token = new_monitor.get('pushToken') or new_monitor.get('push_token') - if push_token: - results[monitor_name] = f"{url}/api/push/{push_token}" - print(f"Push URL ({monitor_name}): {results[monitor_name]}") - - api.disconnect() - print("SUCCESS") - - # Write results to file for Ansible to read - with open('/tmp/mempool_push_urls.yml', 'w') as f: - yaml.dump(results, f) - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitors: - - "Mempool MariaDB" - - "Mempool Backend" - - "Mempool Frontend" - mode: '0644' - - - name: Run Uptime Kuma push monitor setup - command: python3 /tmp/setup_mempool_monitors.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Display monitor setup output - debug: - msg: "{{ monitor_setup.stdout_lines }}" - when: monitor_setup.stdout is defined - - - name: Read push URLs from file - slurp: - src: /tmp/mempool_push_urls.yml - delegate_to: localhost - become: no - register: push_urls_file - ignore_errors: yes - - - name: Parse push URLs - set_fact: - push_urls: "{{ push_urls_file.content | b64decode | from_yaml }}" - when: push_urls_file.content is defined - ignore_errors: yes - - - name: Update MariaDB health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-mariadb-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool MariaDB'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool MariaDB'] is defined - - - name: Update Backend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-backend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Backend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Backend'] is defined - - - name: Update Frontend health check service with push URL - lineinfile: - path: /etc/systemd/system/mempool-frontend-healthcheck.service - regexp: '^Environment=UPTIME_KUMA_PUSH_URL=' - line: "Environment=UPTIME_KUMA_PUSH_URL={{ push_urls['Mempool Frontend'] }}" - insertafter: '^\[Service\]' - when: push_urls is defined and push_urls['Mempool Frontend'] is defined - - - name: Reload systemd after push URL updates - systemd: - daemon_reload: yes - when: push_urls is defined - - - name: Restart health check timers - systemd: - name: "mempool-{{ item }}-healthcheck.timer" - state: restarted - loop: - - mariadb - - backend - - frontend - when: push_urls is defined - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_monitors.py - - /tmp/ansible_mempool_config.yml - - /tmp/mempool_push_urls.yml - -- name: Configure Caddy reverse proxy for Mempool on vipy - hosts: vipy - become: yes - vars_files: - - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - ./mempool_vars.yml - vars: - mempool_subdomain: "{{ subdomains.mempool }}" - mempool_domain: "{{ mempool_subdomain }}.{{ root_domain }}" - caddy_sites_dir: "{{ caddy_sites_dir }}" - - tasks: - - name: Ensure Caddy sites-enabled directory exists - file: - path: "{{ caddy_sites_dir }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Ensure Caddyfile includes import directive for sites-enabled - lineinfile: - path: /etc/caddy/Caddyfile - line: 'import sites-enabled/*' - insertafter: EOF - state: present - backup: yes - create: yes - mode: '0644' - - - name: Create Caddy reverse proxy configuration for Mempool - copy: - dest: "{{ caddy_sites_dir }}/mempool.conf" - content: | - {{ mempool_domain }} { - reverse_proxy mempool-box:{{ mempool_frontend_port }} { - # Use Tailscale MagicDNS to resolve the upstream hostname - transport http { - resolvers 100.100.100.100 - } - } - } - owner: root - group: root - mode: '0644' - - - name: Reload Caddy to apply new config - systemd: - name: caddy - state: reloaded - - - name: Display Mempool URL - debug: - msg: "Mempool is now available at https://{{ mempool_domain }}" - - # =========================================== - # Uptime Kuma HTTP Monitor for Public Endpoint - # =========================================== - - name: Create Uptime Kuma HTTP monitor setup script for Mempool - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_mempool_http_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - with open('/tmp/ansible_mempool_http_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name='services') - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing: - print(f"Monitor '{monitor_name}' already exists (ID: {existing['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating HTTP monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - print(f"ERROR: {str(e)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for HTTP monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_mempool_http_config.yml - content: | - uptime_kuma_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ mempool_domain }}" - monitor_name: "Mempool Public Access" - mode: '0644' - - - name: Run Uptime Kuma HTTP monitor setup - command: python3 /tmp/setup_mempool_http_monitor.py - delegate_to: localhost - become: no - register: http_monitor_setup - changed_when: "'SUCCESS' in http_monitor_setup.stdout" - ignore_errors: yes - - - name: Display HTTP monitor setup output - debug: - msg: "{{ http_monitor_setup.stdout_lines }}" - when: http_monitor_setup.stdout is defined - - - name: Clean up HTTP monitor temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_mempool_http_monitor.py - - /tmp/ansible_mempool_http_config.yml - diff --git a/ansible/services/mempool/mempool_vars.yml b/ansible/services/mempool/mempool_vars.yml deleted file mode 100644 index d3051c3..0000000 --- a/ansible/services/mempool/mempool_vars.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Mempool Configuration Variables - -# Version - Pinned to specific release -mempool_version: "v3.2.1" - -# Directories -mempool_dir: /opt/mempool -mempool_data_dir: "{{ mempool_dir }}/data" -mempool_mysql_dir: "{{ mempool_dir }}/mysql" - -# Network - Bitcoin Core/Knots connection (via Tailnet Magic DNS) -bitcoin_host: "knots-box" -bitcoin_rpc_port: 8332 -# Note: bitcoin_rpc_user and bitcoin_rpc_password are loaded from infra_secrets.yml - -# Network - Fulcrum Electrum server (via Tailnet Magic DNS) -fulcrum_host: "fulcrum-box" -fulcrum_port: 50001 -fulcrum_tls: "false" - -# Mempool network mode -mempool_network: "mainnet" - -# Container ports (internal) -mempool_frontend_port: 8080 -mempool_backend_port: 8999 - -# MariaDB settings -mariadb_database: "mempool" -mariadb_user: "mempool" -# Note: mariadb_mempool_password is loaded from infra_secrets.yml - - diff --git a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml index b8c0064..18a3b72 100644 --- a/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml +++ b/ansible/services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml @@ -14,7 +14,6 @@ ntfy_emergency_app_ntfy_url: "https://{{ ntfy_service_domain }}" ntfy_emergency_app_ntfy_user: "{{ ntfy_username | default('') }}" ntfy_emergency_app_ntfy_password: "{{ ntfy_password | default('') }}" - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" tasks: - name: Create ntfy-emergency-app directory @@ -78,113 +77,3 @@ - name: Reload Caddy to apply new config command: systemctl reload caddy - - - name: Create Uptime Kuma monitor setup script for ntfy-emergency-app - delegate_to: localhost - become: no - copy: - dest: /tmp/setup_ntfy_emergency_app_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import traceback - import yaml - from uptime_kuma_api import UptimeKumaApi, MonitorType - - try: - # Load configs - with open('/tmp/ansible_config.yml', 'r') as f: - config = yaml.safe_load(f) - - url = config['uptime_kuma_url'] - username = config['username'] - password = config['password'] - monitor_url = config['monitor_url'] - monitor_name = config['monitor_name'] - - # Connect to Uptime Kuma - api = UptimeKumaApi(url, timeout=30) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Find or create "services" group - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name='services') - # Refresh to get the group with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == 'services' and m.get('type') == 'group'), None) - - # Check if monitor already exists - existing_monitor = None - for monitor in monitors: - if monitor.get('name') == monitor_name: - existing_monitor = monitor - break - - # Get ntfy notification ID - notifications = api.get_notifications() - ntfy_notification_id = None - for notif in notifications: - if notif.get('type') == 'ntfy': - ntfy_notification_id = notif.get('id') - break - - if existing_monitor: - print(f"Monitor '{monitor_name}' already exists (ID: {existing_monitor['id']})") - print("Skipping - monitor already configured") - else: - print(f"Creating monitor '{monitor_name}'...") - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=monitor_url, - parent=group['id'], - interval=60, - maxretries=3, - retryInterval=60, - notificationIDList={ntfy_notification_id: True} if ntfy_notification_id else {} - ) - - api.disconnect() - print("SUCCESS") - - except Exception as e: - error_msg = str(e) if str(e) else repr(e) - print(f"ERROR: {error_msg}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(1) - mode: '0755' - - - name: Create temporary config for monitor setup - delegate_to: localhost - become: no - copy: - dest: /tmp/ansible_config.yml - content: | - uptime_kuma_url: "{{ uptime_kuma_api_url }}" - username: "{{ uptime_kuma_username }}" - password: "{{ uptime_kuma_password }}" - monitor_url: "https://{{ ntfy_emergency_app_domain }}" - monitor_name: "ntfy-emergency-app" - mode: '0644' - - - name: Run Uptime Kuma monitor setup - command: python3 /tmp/setup_ntfy_emergency_app_monitor.py - delegate_to: localhost - become: no - register: monitor_setup - changed_when: "'SUCCESS' in monitor_setup.stdout" - ignore_errors: yes - - - name: Clean up temporary files - delegate_to: localhost - become: no - file: - path: "{{ item }}" - state: absent - loop: - - /tmp/setup_ntfy_emergency_app_monitor.py - - /tmp/ansible_config.yml diff --git a/ansible/services_config.yml b/ansible/services_config.yml index 56cc570..5c0dcbd 100644 --- a/ansible/services_config.yml +++ b/ansible/services_config.yml @@ -16,14 +16,11 @@ subdomains: lnbits: wallet # Secondary Services (on vipy) - ntfy_emergency_app: avisame + ntfy_emergency_app: emergency personal_blog: pablohere # Memos (on memos-box) memos: memos - - # Mempool Block Explorer (on mempool_box, proxied via vipy) - mempool: mempool # Caddy configuration caddy_sites_dir: /etc/caddy/sites-enabled diff --git a/ansible/services_config.yml.example b/ansible/services_config.yml.example new file mode 100644 index 0000000..972b685 --- /dev/null +++ b/ansible/services_config.yml.example @@ -0,0 +1,32 @@ +# Centralized Services Configuration +# Copy this to services_config.yml and customize + +# Edit these subdomains to match your preferences +subdomains: + # Monitoring Services (on watchtower) + ntfy: ntfy + uptime_kuma: uptime + + # VPN Infrastructure (on spacey) + headscale: headscale + + # Core Services (on vipy) + vaultwarden: vault + forgejo: git + lnbits: lnbits + + # Secondary Services (on vipy) + ntfy_emergency_app: emergency + + # Memos (on memos-box) + memos: memos + +# Caddy configuration +caddy_sites_dir: /etc/caddy/sites-enabled + +# Service-specific settings shared across playbooks +service_settings: + ntfy: + topic: alerts + headscale: + namespace: counter-net diff --git a/backup.inventory.ini b/backup.inventory.ini new file mode 100644 index 0000000..dec2de3 --- /dev/null +++ b/backup.inventory.ini @@ -0,0 +1,16 @@ +[vps] +vipy ansible_host=207.154.226.192 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +watchtower ansible_host=206.189.63.167 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua +spacey ansible_host=165.232.73.4 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + +[nodito_host] +nodito ansible_host=192.168.1.139 ansible_user=counterweight ansible_port=22 ansible_ssh_pass=noesfacilvivirenunmundocentralizado ansible_ssh_private_key_file=~/.ssh/counterganzua + +[nodito_vms] +memos-box ansible_host=192.168.1.149 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=~/.ssh/counterganzua + + +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +# Make sure to adjust the username +[lapy] +localhost ansible_connection=local ansible_user=counterweight gpg_recipient=counterweightoperator@protonmail.com gpg_key_id=883EDBAA726BD96C \ No newline at end of file diff --git a/human_script.md b/human_script.md new file mode 100644 index 0000000..345c6a7 --- /dev/null +++ b/human_script.md @@ -0,0 +1,897 @@ +# Personal Infrastructure Setup Guide + +This guide walks you through setting up your complete personal infrastructure, layer by layer. Each layer must be completed before moving to the next one. + +**Automated Setup:** Each layer has a bash script that handles the setup process. The scripts will: +- Check prerequisites +- Prompt for required variables +- Set up configuration files +- Execute playbooks +- Verify completion + +## Prerequisites + +Before starting: +- You have a domain name +- You have VPS accounts ready +- You have nodito ready with Proxmox installed, ssh key in place +- You have SSH access to all machines +- You're running this from your laptop (lapy) + +--- + +## Layer 0: Foundation Setup + +**Goal:** Set up your laptop (lapy) as the Ansible control node and configure basic settings. + +**Script:** `./scripts/setup_layer_0.sh` + +### What This Layer Does: +1. Creates Python virtual environment +2. Installs Ansible and required Python packages +3. Installs Ansible Galaxy collections +4. Guides you through creating `inventory.ini` with your machine IPs +5. Guides you through creating `infra_vars.yml` with your domain +6. Creates `services_config.yml` with centralized subdomain settings +7. Creates `infra_secrets.yml` template for Uptime Kuma credentials +8. Validates SSH keys exist +9. Verifies everything is ready for Layer 1 + +### Required Information: +- Your domain name (e.g., `contrapeso.xyz`) +- SSH key path (default: `~/.ssh/counterganzua`) +- IP addresses for your infrastructure: + - vipy (main VPS) + - watchtower (monitoring VPS) + - spacey (headscale VPS) + - nodito (Proxmox server) - optional + - **Note:** VMs (like memos-box) will be created later on Proxmox and added to the `nodito_vms` group + +### Manual Steps: +After running the script, you'll need to: +1. Ensure your SSH key is added to all VPS root users (usually done by VPS provider) +2. Ensure DNS is configured for your domain (nameservers pointing to your DNS provider) + +### Centralized Configuration: + +The script creates `ansible/services_config.yml` which contains all service subdomains in one place: +- Easy to review all subdomains at a glance +- No need to edit multiple vars files +- Consistent Caddy settings across all services +- **Edit this file to customize your subdomains before deploying services** + +### Verification: +The script will verify: +- ✓ Python venv exists and activated +- ✓ Ansible installed +- ✓ Required Python packages installed +- ✓ Ansible Galaxy collections installed +- ✓ `inventory.ini` exists and formatted correctly +- ✓ `infra_vars.yml` exists with domain configured +- ✓ `services_config.yml` created with subdomain settings +- ✓ `infra_secrets.yml` template created +- ✓ SSH key file exists + +### Run the Script: +```bash +cd /home/counterweight/personal_infra +./scripts/setup_layer_0.sh +``` + +--- + +## Layer 1A: VPS Basic Setup + +**Goal:** Configure users, SSH access, firewall, and fail2ban on VPS machines. + +**Script:** `./scripts/setup_layer_1a_vps.sh` + +**Can be run independently** - doesn't require Nodito setup. + +### What This Layer Does: + +For VPSs (vipy, watchtower, spacey): +1. Creates the `counterweight` user with sudo access +2. Configures SSH key authentication +3. Disables root login (by design for security) +4. Sets up UFW firewall with SSH access +5. Installs and configures fail2ban +6. Installs and configures auditd for security logging + +### Prerequisites: +- ✅ Layer 0 complete +- ✅ SSH key added to all VPS root users +- ✅ Root access to VPSs + +### Verification: +The script will verify: +- ✓ Can SSH to all VPSs as root +- ✓ VPS playbooks complete successfully +- ✓ Can SSH to all VPSs as `counterweight` user +- ✓ Firewall is active and configured +- ✓ fail2ban is running + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_1a_vps.sh +``` + +**Note:** After this layer, you will no longer be able to SSH as root to VPSs (by design for security). + +--- + +## Layer 1B: Nodito (Proxmox) Setup + +**Goal:** Configure the Nodito Proxmox server. + +**Script:** `./scripts/setup_layer_1b_nodito.sh` + +**Can be run independently** - doesn't require VPS setup. + +### What This Layer Does: + +For Nodito (Proxmox server): +1. Bootstraps SSH key access for root +2. Creates the `counterweight` user +3. Updates and secures the system +4. Disables root login and password authentication +5. Switches to Proxmox community repositories +6. Optionally sets up ZFS storage pool (if disks configured) +7. Optionally creates Debian cloud template + +### Prerequisites: +- ✅ Layer 0 complete +- ✅ Root password for nodito +- ✅ Nodito configured in inventory.ini + +### Optional: ZFS Setup +For ZFS storage pool (optional): +1. SSH into nodito: `ssh root@` +2. List disk IDs: `ls -la /dev/disk/by-id/ | grep -E "(ata-|scsi-|nvme-)"` +3. Note the disk IDs you want to use +4. The script will help you create `ansible/infra/nodito/nodito_vars.yml` with disk configuration + +⚠️ **Warning:** ZFS setup will DESTROY ALL DATA on specified disks! + +### Verification: +The script will verify: +- ✓ Nodito bootstrap successful +- ✓ Community repos configured +- ✓ Can SSH to nodito as `counterweight` user + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_1b_nodito.sh +``` + +**Note:** After this layer, you will no longer be able to SSH as root to nodito (by design for security). + +--- + +## Layer 2: General Infrastructure Tools + +**Goal:** Install common utilities needed by various services. + +**Script:** `./scripts/setup_layer_2.sh` + +### What This Layer Does: + +Installs essential tools on machines that need them: + +#### rsync +- **Purpose:** Required for backup operations +- **Deployed to:** vipy, watchtower, lapy (and optionally other hosts) +- **Playbook:** `infra/900_install_rsync.yml` + +#### Docker + Docker Compose +- **Purpose:** Required for containerized services +- **Deployed to:** vipy, watchtower (and optionally other hosts) +- **Playbook:** `infra/910_docker_playbook.yml` + +### Prerequisites: +- ✅ Layer 0 complete +- ✅ Layer 1A complete (for VPSs) OR Layer 1B complete (for nodito) +- ✅ SSH access as counterweight user + +### Services That Need These Tools: +- **rsync:** All backup operations (Uptime Kuma, Vaultwarden, LNBits, etc.) +- **docker:** Uptime Kuma, Vaultwarden, ntfy-emergency-app + +### Verification: +The script will verify: +- ✓ rsync installed on specified hosts +- ✓ Docker and Docker Compose installed on specified hosts +- ✓ counterweight user added to docker group +- ✓ Docker service running + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_2.sh +``` + +**Note:** This script is interactive and will let you choose which hosts get which tools. + +--- + +## Layer 3: Reverse Proxy (Caddy) + +**Goal:** Deploy Caddy reverse proxy for HTTPS termination and routing. + +**Script:** `./scripts/setup_layer_3_caddy.sh` + +### What This Layer Does: + +Installs and configures Caddy web server on VPS machines: +- Installs Caddy from official repositories +- Configures Caddy to listen on ports 80/443 +- Opens firewall ports for HTTP/HTTPS +- Creates `/etc/caddy/sites-enabled/` directory structure +- Sets up automatic HTTPS with Let's Encrypt + +**Deployed to:** vipy, watchtower, spacey + +### Why Caddy is Critical: + +Caddy provides: +- **Automatic HTTPS** - Let's Encrypt certificates with auto-renewal +- **Reverse proxy** - Routes traffic to backend services +- **Simple configuration** - Each service adds its own config file +- **HTTP/2 support** - Modern protocol support + +### Prerequisites: +- ✅ Layer 0 complete +- ✅ Layer 1A complete (VPS setup) +- ✅ SSH access as counterweight user +- ✅ Ports 80/443 available on VPSs + +### Services That Need Caddy: +All web services depend on Caddy: +- Uptime Kuma (watchtower) +- ntfy (watchtower) +- Headscale (spacey) +- Vaultwarden (vipy) +- Forgejo (vipy) +- LNBits (vipy) +- ntfy-emergency-app (vipy) + +### Verification: +The script will verify: +- ✓ Caddy installed on all target hosts +- ✓ Caddy service running +- ✓ Ports 80/443 open in firewall +- ✓ Sites-enabled directory created +- ✓ Can reach Caddy default page + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_3_caddy.sh +``` + +**Note:** Caddy starts with an empty configuration. Services will add their own config files in later layers. + +--- + +## Layer 4: Core Monitoring & Notifications + +**Goal:** Deploy ntfy (notifications) and Uptime Kuma (monitoring platform). + +**Script:** `./scripts/setup_layer_4_monitoring.sh` + +### What This Layer Does: + +Deploys core monitoring infrastructure on watchtower: + +#### 4A: ntfy (Notification Service) +- Installs ntfy from official repositories +- Configures ntfy with authentication (deny-all by default) +- Creates admin user for sending notifications +- Sets up Caddy reverse proxy +- **Deployed to:** watchtower + +#### 4B: Uptime Kuma (Monitoring Platform) +- Deploys Uptime Kuma via Docker +- Configures Caddy reverse proxy +- Sets up data persistence +- Optionally sets up backup to lapy +- **Deployed to:** watchtower + +### Prerequisites (Complete BEFORE Running): + +**1. Previous layers complete:** +- ✅ Layer 0, 1A, 2, 3 complete (watchtower must be fully set up) +- ✅ Docker installed on watchtower (from Layer 2) +- ✅ Caddy running on watchtower (from Layer 3) + +**2. Configure subdomains (in centralized config):** +- ✅ Edit `ansible/services_config.yml` and customize subdomains under `subdomains:` section + - Set `ntfy:` to your preferred subdomain (e.g., `ntfy` or `notify`) + - Set `uptime_kuma:` to your preferred subdomain (e.g., `uptime` or `kuma`) + +**3. Create DNS records that match your configured subdomains:** +- ✅ Create A record: `.` → watchtower IP +- ✅ Create A record: `.` → watchtower IP +- ✅ Wait for DNS propagation (can take minutes to hours) +- ✅ Verify with: `dig .` should return watchtower IP + +**4. Prepare ntfy admin credentials:** +- ✅ Decide on username (default: `admin`) +- ✅ Decide on a secure password (script will prompt you) + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_4_monitoring.sh +``` + +The script will prompt you for ntfy admin credentials during deployment. + +### Post-Deployment Steps (Complete AFTER Running): + +**The script will guide you through most of these, but here's what happens:** + +#### Step 1: Set Up Uptime Kuma Admin Account (Manual) +1. Open browser and visit: `https://.` +2. On first visit, you'll see the setup page +3. Create admin username and password +4. Save these credentials securely + +#### Step 2: Update infra_secrets.yml (Manual) +1. Edit `ansible/infra_secrets.yml` +2. Add your Uptime Kuma credentials: + ```yaml + uptime_kuma_username: "your-admin-username" + uptime_kuma_password: "your-admin-password" + ``` +3. Save the file +4. **This is required for automated ntfy setup and Layer 6** + +#### Step 3: Configure ntfy Notification (Automated) +**The script will offer to do this automatically!** If you completed Steps 1 & 2, the script will: +- Connect to Uptime Kuma via API +- Create ntfy notification configuration +- Test the connection +- No manual UI configuration needed! + +**Alternatively (Manual):** +1. In Uptime Kuma web UI, go to **Settings** → **Notifications** +2. Click **Setup Notification**, choose **ntfy** +3. Configure with your ntfy subdomain and credentials + +#### Step 4: Final Verification (Automated) +**The script will automatically verify:** +- ✓ Uptime Kuma credentials in infra_secrets.yml +- ✓ Can connect to Uptime Kuma API +- ✓ ntfy notification is configured +- ✓ All post-deployment steps complete + +If anything is missing, the script will tell you exactly what to do! + +#### Step 5: Subscribe to Notifications on Your Phone (Optional - Manual) +1. Install ntfy app: https://github.com/binwiederhier/ntfy-android +2. Add subscription: + - Server: `https://.` + - Topic: `alerts` (same as configured in Uptime Kuma) + - Username: Your ntfy admin username + - Password: Your ntfy admin password +3. You'll now receive push notifications for all alerts! + +**Pro tip:** Run the script again after completing Steps 1 & 2, and it will automatically configure ntfy and verify everything! + +### Verification: +The script will automatically verify: +- ✓ DNS records are configured correctly (using `dig`) +- ✓ ntfy service running +- ✓ Uptime Kuma container running +- ✓ Caddy configs created for both services + +After post-deployment steps, you can test: +- Visit `https://.` (should load ntfy web UI) +- Visit `https://.` (should load Uptime Kuma) +- Send test notification in Uptime Kuma + +**Note:** DNS validation requires `dig` command. If not available, validation will be skipped (you can continue but SSL may fail). + +### Why This Layer is Critical: +- **All infrastructure monitoring** (Layer 6) depends on Uptime Kuma +- **All alerts** go through ntfy +- Services availability monitoring needs Uptime Kuma +- Without this layer, you won't know when things break! + +--- + +## Layer 5: VPN Infrastructure (Headscale) + +**Goal:** Deploy Headscale for secure mesh networking (like Tailscale, but self-hosted). + +**Script:** `./scripts/setup_layer_5_headscale.sh` + +**This layer is OPTIONAL** - Skip to Layer 6 if you don't need VPN mesh networking. + +### What This Layer Does: + +Deploys Headscale coordination server and optionally joins machines to the mesh: + +#### 5A: Deploy Headscale Server +- Installs Headscale on spacey +- Configures with deny-all ACL policy (you customize later) +- Creates namespace/user for your network +- Sets up Caddy reverse proxy +- Configures embedded DERP server for NAT traversal +- **Deployed to:** spacey + +#### 5B: Join Machines to Mesh (Optional) +- Installs Tailscale client on target machines +- Generates ephemeral pre-auth keys +- Automatically joins machines to your mesh +- Enables Magic DNS +- **Can join:** vipy, watchtower, nodito, lapy, etc. + +### Prerequisites (Complete BEFORE Running): + +**1. Previous layers complete:** +- ✅ Layer 0, 1A, 3 complete (spacey must be set up) +- ✅ Caddy running on spacey (from Layer 3) + +**2. Configure subdomain (in centralized config):** +- ✅ Edit `ansible/services_config.yml` and customize `headscale:` under `subdomains:` section (e.g., `headscale` or `vpn`) + +**3. Create DNS record that matches your configured subdomain:** +- ✅ Create A record: `.` → spacey IP +- ✅ Wait for DNS propagation +- ✅ Verify with: `dig .` should return spacey IP + +**4. Decide on namespace name:** +- ✅ Choose a namespace for your network (default: `counter-net`) +- ✅ This is set in `headscale_vars.yml` as `headscale_namespace` + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_5_headscale.sh +``` + +The script will: +1. Validate DNS configuration +2. Deploy Headscale server +3. Offer to join machines to the mesh + +### Post-Deployment Steps: + +#### Configure ACL Policies (Required for machines to communicate) +1. SSH into spacey: `ssh counterweight@` +2. Edit ACL file: `sudo nano /etc/headscale/acl.json` +3. Configure rules (example - allow all): + ```json + { + "ACLs": [ + {"action": "accept", "src": ["*"], "dst": ["*:*"]} + ] + } + ``` +4. Restart Headscale: `sudo systemctl restart headscale` + +**Default is deny-all for security** - you must configure ACLs for machines to talk! + +#### Join Additional Machines Manually +For machines not in inventory (mobile, desktop): +1. Install Tailscale client on device +2. Generate pre-auth key on spacey: + ```bash + ssh counterweight@ + sudo headscale preauthkeys create --user --reusable + ``` +3. Connect using your Headscale server: + ```bash + tailscale up --login-server https://. --authkey + ``` + +### Automatic Uptime Kuma Monitor: + +**The playbook will automatically create a monitor in Uptime Kuma:** +- ✅ **Headscale** - monitors `https:///health` +- Added to "services" monitor group +- Uses ntfy notification (if configured) +- Check every 60 seconds + +**Prerequisites:** Uptime Kuma credentials must be in `infra_secrets.yml` (from Layer 4) + +### Verification: +The script will automatically verify: +- ✓ DNS records configured correctly +- ✓ Headscale installed and running +- ✓ Namespace created +- ✓ Caddy config created +- ✓ Machines joined (if selected) +- ✓ Monitor created in Uptime Kuma "services" group + +List connected devices: +```bash +ssh counterweight@ +sudo headscale nodes list +``` + +### Why Use Headscale: +- **Secure communication** between all your machines +- **Magic DNS** - access machines by hostname +- **NAT traversal** - works even behind firewalls +- **Self-hosted** - full control of your VPN +- **Mobile support** - use official Tailscale apps + +### Backup: +Optional backup to lapy: +```bash +ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml +``` + +--- + +## Layer 6: Infrastructure Monitoring + +**Goal:** Deploy automated monitoring for disk usage, system health, and CPU temperature. + +**Script:** `./scripts/setup_layer_6_infra_monitoring.sh` + +### What This Layer Does: + +Deploys monitoring scripts that report to Uptime Kuma: + +#### 6A: Disk Usage Monitoring +- Monitors disk usage on specified mount points +- Sends alerts when usage exceeds threshold (default: 80%) +- Creates Uptime Kuma push monitors automatically +- Organizes monitors in host-specific groups +- **Deploys to:** All hosts (selectable) + +#### 6B: System Healthcheck +- Sends regular heartbeat pings to Uptime Kuma +- Alerts if system stops responding +- "No news is good news" monitoring +- **Deploys to:** All hosts (selectable) + +#### 6C: CPU Temperature Monitoring (Nodito only) +- Monitors CPU temperature on Proxmox server +- Alerts when temperature exceeds threshold (default: 80°C) +- **Deploys to:** nodito (if configured) + +### Prerequisites (Complete BEFORE Running): + +**1. Previous layers complete:** +- ✅ Layer 0, 1A/1B, 4 complete +- ✅ Uptime Kuma deployed and configured (Layer 4) +- ✅ **CRITICAL:** `infra_secrets.yml` has Uptime Kuma credentials + +**2. Uptime Kuma API credentials ready:** +- ✅ Must have completed Layer 4 post-deployment steps +- ✅ `ansible/infra_secrets.yml` must contain: + ```yaml + uptime_kuma_username: "your-username" + uptime_kuma_password: "your-password" + ``` + +**3. Python dependencies installed:** +- ✅ `uptime-kuma-api` must be in requirements.txt +- ✅ Should already be installed from Layer 0 +- ✅ Verify: `pip list | grep uptime-kuma-api` + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_6_infra_monitoring.sh +``` + +The script will: +1. Verify Uptime Kuma credentials +2. Offer to deploy disk usage monitoring +3. Offer to deploy system healthchecks +4. Offer to deploy CPU temp monitoring (nodito only) +5. Test monitor creation and alerts + +### What Gets Deployed: + +**For each monitored host:** +- Push monitor in Uptime Kuma (upside-down mode) +- Monitor group named `{hostname} - infra` +- Systemd service for monitoring script +- Systemd timer for periodic execution +- Log file for monitoring history + +**Default settings (customizable):** +- Disk usage threshold: 80% +- Disk check interval: 15 minutes +- Healthcheck interval: 60 seconds +- CPU temp threshold: 80°C +- Monitored mount point: `/` (root) + +### Customization Options: + +Change thresholds and intervals: +```bash +# Disk monitoring with custom settings +ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml \ + -e "disk_usage_threshold_percent=85" \ + -e "disk_check_interval_minutes=10" \ + -e "monitored_mount_point=/home" + +# Healthcheck with custom interval +ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml \ + -e "healthcheck_interval_seconds=30" + +# CPU temp with custom threshold +ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml \ + -e "temp_threshold_celsius=75" +``` + +### Verification: +The script will automatically verify: +- ✓ Uptime Kuma API accessible +- ✓ Monitors created in Uptime Kuma +- ✓ Monitor groups created +- ✓ Systemd services running +- ✓ Can send test alerts + +Check Uptime Kuma web UI: +- Monitors should appear organized by host +- Should receive test pings +- Alerts will show when thresholds exceeded + +### Post-Deployment: + +**Monitor your infrastructure:** +1. Open Uptime Kuma web UI +2. See all monitors organized by host groups +3. Configure notification rules per monitor +4. Set up status pages (optional) + +**Test alerts:** +```bash +# Trigger disk usage alert (fill disk temporarily) +# Trigger healthcheck alert (stop the service) +# Check ntfy for notifications +``` + +### Why This Layer is Important: +- **Proactive monitoring** - Know about issues before users do +- **Disk space alerts** - Prevent services from failing +- **System health** - Detect crashed/frozen machines +- **Temperature monitoring** - Prevent hardware damage +- **Organized** - All monitors grouped by host + +--- + +## Layer 7: Core Services + +**Goal:** Deploy core applications: Vaultwarden, Forgejo, and LNBits. + +**Script:** `./scripts/setup_layer_7_services.sh` + +### What This Layer Does: + +Deploys main services on vipy: + +#### 7A: Vaultwarden (Password Manager) +- Deploys via Docker +- Configures Caddy reverse proxy +- Sets up fail2ban protection +- Enables sign-ups initially (disable after creating first user) +- **Deployed to:** vipy + +#### 7B: Forgejo (Git Server) +- Installs Forgejo binary +- Creates git user and directories +- Configures Caddy reverse proxy +- Enables SSH cloning +- **Deployed to:** vipy + +#### 7C: LNBits (Lightning Wallet) +- Installs system dependencies and uv (Python 3.12 tooling) +- Clones LNBits version v1.3.1 +- Syncs dependencies with uv targeting Python 3.12 +- Configures with FakeWallet backend (for testing) +- Creates systemd service +- Configures Caddy reverse proxy +- **Deployed to:** vipy + +### Prerequisites (Complete BEFORE Running): + +**1. Previous layers complete:** +- ✅ Layer 0, 1A, 2, 3 complete +- ✅ Docker installed on vipy (Layer 2) +- ✅ Caddy running on vipy (Layer 3) + +**2. Configure subdomains (in centralized config):** +- ✅ Edit `ansible/services_config.yml` and customize subdomains under `subdomains:` section: + - Set `vaultwarden:` to your preferred subdomain (e.g., `vault` or `passwords`) + - Set `forgejo:` to your preferred subdomain (e.g., `git` or `code`) + - Set `lnbits:` to your preferred subdomain (e.g., `lnbits` or `wallet`) + +**3. Create DNS records matching your subdomains:** +- ✅ Create A record: `.` → vipy IP +- ✅ Create A record: `.` → vipy IP +- ✅ Create A record: `.` → vipy IP +- ✅ Wait for DNS propagation + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_7_services.sh +``` + +The script will: +1. Validate DNS configuration +2. Offer to deploy each service +3. Configure backups (optional) + +### Post-Deployment Steps: + +#### Vaultwarden: +1. Visit `https://.` +2. Create your first user account +3. **Important:** Disable sign-ups after first user: + ```bash + ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml + ``` +4. Optional: Set up backup to lapy + +#### Forgejo: +1. Visit `https://.` +2. Create admin account on first visit +3. Default: registrations disabled for security +4. SSH cloning works automatically after adding SSH key + +#### LNBits: +1. Visit `https://.` +2. Create superuser on first visit +3. **Important:** Default uses FakeWallet (testing only) +4. Configure real Lightning backend: + - Edit `/opt/lnbits/lnbits/.env` on vipy + - Or use the superuser UI to configure backend +5. Disable new user registration for security +6. Optional: Set up encrypted backup to lapy + +### Backup Configuration: + +After services are stable, set up backups: + +**Vaultwarden backup:** +```bash +ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml +``` + +**LNBits backup (GPG encrypted):** +```bash +ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml +``` + +**Note:** Forgejo backups are not automated - backup manually or set up your own solution. + +### Automatic Uptime Kuma Monitors: + +**The playbooks will automatically create monitors in Uptime Kuma for each service:** +- ✅ **Vaultwarden** - monitors `https:///alive` +- ✅ **Forgejo** - monitors `https:///api/healthz` +- ✅ **LNBits** - monitors `https:///api/v1/health` + +All monitors: +- Added to "services" monitor group +- Use ntfy notification (if configured) +- Check every 60 seconds +- 3 retries before alerting + +**Prerequisites:** Uptime Kuma credentials must be in `infra_secrets.yml` (from Layer 4) + +### Verification: +The script will automatically verify: +- ✓ DNS records configured +- ✓ Services deployed +- ✓ Docker containers running (Vaultwarden) +- ✓ Systemd services running (Forgejo, LNBits) +- ✓ Caddy configs created + +Manual verification: +- Visit each service's subdomain +- Create admin/first user accounts +- Test functionality +- Check Uptime Kuma for new monitors in "services" group + +### Why These Services: +- **Vaultwarden** - Self-hosted password manager (Bitwarden compatible) +- **Forgejo** - Self-hosted Git server (GitHub/GitLab alternative) +- **LNBits** - Lightning Network wallet and accounts system + +--- + +## Layer 8: Secondary Services + +**Goal:** Deploy auxiliary services that depend on the core stack: ntfy-emergency-app and memos. + +**Script:** `./scripts/setup_layer_8_secondary_services.sh` + +### What This Layer Does: +- Deploys the ntfy-emergency-app container on vipy and proxies it through Caddy +- Optionally deploys Memos on `memos-box` (skips automatically if the host is not yet in `inventory.ini`) + +### Prerequisites (Complete BEFORE Running): +- ✅ Layers 0–7 complete (Caddy, ntfy, and Uptime Kuma already online) +- ✅ `ansible/services_config.yml` reviewed so the `ntfy_emergency_app` and `memos` subdomains match your plan +- ✅ `ansible/infra_secrets.yml` contains valid `ntfy_username` and `ntfy_password` +- ✅ DNS A records created for the subdomains (see below) +- ✅ If deploying Memos, ensure `memos-box` exists in `inventory.ini` and is reachable as the `counterweight` user + +### DNS Requirements: +- `.` → vipy IP +- `.` → memos-box IP (skip if memos not yet provisioned) + +The script runs `dig` to validate DNS before deploying and will warn if records are missing or pointing elsewhere. + +### Run the Script: +```bash +source venv/bin/activate +cd /home/counterweight/personal_infra +./scripts/setup_layer_8_secondary_services.sh +``` + +You can deploy each service independently; the script asks for confirmation before running each playbook. + +### Post-Deployment Steps: +- **ntfy-emergency-app:** Visit the emergency subdomain, trigger a test notification, and verify ntfy receives it +- **Memos (if deployed):** Visit the memos subdomain, create the first admin user, and adjust settings from the UI + +### Verification: +- The script checks for the presence of Caddy configs, running containers, and Memos systemd service status +- Review Uptime Kuma or add monitors for these services if you want automatic alerting + +### Optional Follow-Ups: +- Configure backups for any new data stores (e.g., snapshot memos data) +- Add Uptime Kuma monitors for the new services if you want automated alerting + +--- + +## Troubleshooting + +### Common Issues + +#### SSH Connection Fails +- Verify VPS is running and accessible +- Check SSH key is in the correct location +- Ensure SSH key has correct permissions (600) +- Try manual SSH: `ssh -i ~/.ssh/counterganzua root@` + +#### Ansible Not Found +- Make sure you've activated the venv: `source venv/bin/activate` +- Run Layer 0 script again + +#### DNS Not Resolving +- DNS changes can take up to 24-48 hours to propagate +- Use `dig .` to check DNS status +- You can proceed with setup; services will work once DNS propagates + +--- + +## Progress Tracking + +Use this checklist to track your progress: + +- [ ] Layer 0: Foundation Setup +- [ ] Layer 1A: VPS Basic Setup +- [ ] Layer 1B: Nodito (Proxmox) Setup +- [ ] Layer 2: General Infrastructure Tools +- [ ] Layer 3: Reverse Proxy (Caddy) +- [ ] Layer 4: Core Monitoring & Notifications +- [ ] Layer 5: VPN Infrastructure (Headscale) +- [ ] Layer 6: Infrastructure Monitoring +- [ ] Layer 7: Core Services +- [ ] Layer 8: Secondary Services +- [ ] Backups Configured + diff --git a/scripts/setup_layer_0.sh b/scripts/setup_layer_0.sh new file mode 100755 index 0000000..f994f98 --- /dev/null +++ b/scripts/setup_layer_0.sh @@ -0,0 +1,488 @@ +#!/bin/bash + +############################################################################### +# Layer 0: Foundation Setup +# +# This script sets up your laptop (lapy) as the Ansible control node. +# It prepares all the prerequisites needed for the infrastructure deployment. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +prompt_user() { + local prompt="$1" + local default="$2" + local result + + if [ -n "$default" ]; then + read -p "$(echo -e ${BLUE}${prompt}${NC} [${default}]: )" result + result="${result:-$default}" + else + read -p "$(echo -e ${BLUE}${prompt}${NC}: )" result + fi + + echo "$result" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Main Setup Functions +############################################################################### + +check_prerequisites() { + print_header "Checking Prerequisites" + + # Check if we're in the right directory + if [ ! -f "$PROJECT_ROOT/README.md" ] || [ ! -d "$PROJECT_ROOT/ansible" ]; then + print_error "Not in the correct project directory" + echo "Expected: $PROJECT_ROOT" + exit 1 + fi + print_success "Running from correct directory: $PROJECT_ROOT" + + # Check if Python 3 is installed + if ! command -v python3 &> /dev/null; then + print_error "Python 3 is not installed. Please install Python 3 first." + exit 1 + fi + print_success "Python 3 found: $(python3 --version)" + + # Check if git is installed + if ! command -v git &> /dev/null; then + print_warning "Git is not installed. Some features may not work." + else + print_success "Git found: $(git --version | head -n1)" + fi +} + +setup_python_venv() { + print_header "Setting Up Python Virtual Environment" + + cd "$PROJECT_ROOT" + + if [ -d "venv" ]; then + print_info "Virtual environment already exists" + if confirm_action "Recreate virtual environment?"; then + rm -rf venv + python3 -m venv venv + print_success "Virtual environment recreated" + else + print_success "Using existing virtual environment" + fi + else + python3 -m venv venv + print_success "Virtual environment created" + fi + + # Activate venv + source venv/bin/activate + print_success "Virtual environment activated" + + # Upgrade pip + print_info "Upgrading pip..." + pip install --upgrade pip > /dev/null 2>&1 + print_success "pip upgraded" +} + +install_python_requirements() { + print_header "Installing Python Requirements" + + cd "$PROJECT_ROOT" + + if [ ! -f "requirements.txt" ]; then + print_error "requirements.txt not found" + exit 1 + fi + + print_info "Installing packages from requirements.txt..." + pip install -r requirements.txt + print_success "Python requirements installed" + + # Verify Ansible installation + if ! command -v ansible &> /dev/null; then + print_error "Ansible installation failed" + exit 1 + fi + print_success "Ansible installed: $(ansible --version | head -n1)" +} + +install_ansible_collections() { + print_header "Installing Ansible Galaxy Collections" + + cd "$PROJECT_ROOT/ansible" + + if [ ! -f "requirements.yml" ]; then + print_warning "requirements.yml not found, skipping Ansible collections" + return + fi + + print_info "Installing Ansible Galaxy collections..." + ansible-galaxy collection install -r requirements.yml + print_success "Ansible Galaxy collections installed" +} + +setup_inventory_file() { + print_header "Setting Up Inventory File" + + cd "$PROJECT_ROOT/ansible" + + if [ -f "inventory.ini" ]; then + print_info "inventory.ini already exists" + cat inventory.ini + echo "" + if ! confirm_action "Do you want to update it?"; then + print_success "Using existing inventory.ini" + return + fi + fi + + print_info "Let's configure your infrastructure hosts" + echo "" + + # Collect information + echo -e -n "${BLUE}SSH key path${NC} [~/.ssh/counterganzua]: " + read ssh_key + ssh_key="${ssh_key:-~/.ssh/counterganzua}" + + echo "" + echo "Enter the IP addresses for your infrastructure (VMs will be added later):" + echo "" + + echo -e -n "${BLUE}vipy${NC} (main VPS) IP: " + read vipy_ip + echo -e -n "${BLUE}watchtower${NC} (monitoring VPS) IP: " + read watchtower_ip + echo -e -n "${BLUE}spacey${NC} (headscale VPS) IP: " + read spacey_ip + echo -e -n "${BLUE}nodito${NC} (Proxmox server) IP [optional]: " + read nodito_ip + + echo "" + echo -e -n "${BLUE}Your username on lapy${NC} [$(whoami)]: " + read lapy_user + lapy_user="${lapy_user:-$(whoami)}" + + echo -e -n "${BLUE}GPG recipient email${NC} [optional, for encrypted backups]: " + read gpg_email + echo -e -n "${BLUE}GPG key ID${NC} [optional, for encrypted backups]: " + read gpg_key + + # Generate inventory.ini + cat > inventory.ini << EOF +# Ansible Inventory File +# Generated by setup_layer_0.sh + +EOF + + vps_entries="" + if [ -n "$vipy_ip" ]; then + vps_entries+="vipy ansible_host=$vipy_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" + fi + if [ -n "$watchtower_ip" ]; then + vps_entries+="watchtower ansible_host=$watchtower_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" + fi + if [ -n "$spacey_ip" ]; then + vps_entries+="spacey ansible_host=$spacey_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key\n" + fi + + if [ -n "$vps_entries" ]; then + cat >> inventory.ini << EOF +[vps] +${vps_entries} +EOF + fi + + if [ -n "$nodito_ip" ]; then + cat >> inventory.ini << EOF +[nodito_host] +nodito ansible_host=$nodito_ip ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key + +EOF + fi + + # Add nodito_vms placeholder for VMs that will be created later + cat >> inventory.ini << EOF +# Nodito VMs - These don't exist yet and will be created on the Proxmox server +# Add them here once you create VMs on nodito (e.g., memos-box, etc.) +[nodito_vms] +# Example: +# memos_box ansible_host=192.168.1.150 ansible_user=counterweight ansible_port=22 ansible_ssh_private_key_file=$ssh_key + +EOF + + # Add lapy + cat >> inventory.ini << EOF +# Local connection to laptop: this assumes you're running ansible commands from your personal laptop +[lapy] +localhost ansible_connection=local ansible_user=$lapy_user +EOF + + if [ -n "$gpg_email" ] && [ -n "$gpg_key" ]; then + echo " gpg_recipient=$gpg_email gpg_key_id=$gpg_key" >> inventory.ini + fi + + print_success "inventory.ini created" + echo "" + print_info "Review your inventory file:" + cat inventory.ini + echo "" +} + +setup_infra_vars() { + print_header "Setting Up Infrastructure Variables" + + cd "$PROJECT_ROOT/ansible" + + if [ -f "infra_vars.yml" ]; then + print_info "infra_vars.yml already exists" + cat infra_vars.yml + echo "" + if ! confirm_action "Do you want to update it?"; then + print_success "Using existing infra_vars.yml" + return + fi + fi + + echo "" + echo -e -n "${BLUE}Your root domain${NC} (e.g., contrapeso.xyz): " + read domain + + while [ -z "$domain" ]; do + print_warning "Domain cannot be empty" + echo -e -n "${BLUE}Your root domain${NC}: " + read domain + done + + cat > infra_vars.yml << EOF +# Infrastructure Variables +# Generated by setup_layer_0.sh + +new_user: counterweight +ssh_port: 22 +allow_ssh_from: "any" +root_domain: $domain +EOF + + print_success "infra_vars.yml created" + echo "" + print_info "Contents:" + cat infra_vars.yml + echo "" +} + +setup_services_config() { + print_header "Setting Up Services Configuration" + + cd "$PROJECT_ROOT/ansible" + + if [ -f "services_config.yml" ]; then + print_info "services_config.yml already exists" + if ! confirm_action "Do you want to recreate it from template?"; then + print_success "Using existing services_config.yml" + return + fi + fi + + if [ ! -f "services_config.yml.example" ]; then + print_error "services_config.yml.example not found" + return + fi + + cp services_config.yml.example services_config.yml + + print_success "services_config.yml created" + echo "" + print_info "This file centralizes all service subdomains and Caddy settings" + print_info "Customize subdomains in: ansible/services_config.yml" + echo "" +} + +setup_infra_secrets() { + print_header "Setting Up Infrastructure Secrets" + + cd "$PROJECT_ROOT/ansible" + + if [ -f "infra_secrets.yml" ]; then + print_warning "infra_secrets.yml already exists" + if ! confirm_action "Do you want to recreate the template?"; then + print_success "Using existing infra_secrets.yml" + return + fi + fi + + cat > infra_secrets.yml << EOF +# Infrastructure Secrets +# Generated by setup_layer_0.sh +# +# IMPORTANT: This file contains sensitive credentials +# It is already in .gitignore - DO NOT commit it to git +# +# You'll need to fill in the Uptime Kuma credentials after Layer 4 +# when you deploy Uptime Kuma + +# Uptime Kuma Credentials (fill these in after deploying Uptime Kuma in Layer 4) +uptime_kuma_username: "" +uptime_kuma_password: "" +EOF + + print_success "infra_secrets.yml template created" + print_warning "You'll need to fill in Uptime Kuma credentials after Layer 4" + echo "" +} + +validate_ssh_key() { + print_header "Validating SSH Key" + + cd "$PROJECT_ROOT/ansible" + + # Extract SSH key path from inventory + if [ -f "inventory.ini" ]; then + ssh_key=$(grep "ansible_ssh_private_key_file" inventory.ini | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + + # Expand tilde + ssh_key="${ssh_key/#\~/$HOME}" + + if [ -f "$ssh_key" ]; then + print_success "SSH key found: $ssh_key" + + # Check permissions + perms=$(stat -c "%a" "$ssh_key" 2>/dev/null || stat -f "%OLp" "$ssh_key" 2>/dev/null) + if [ "$perms" != "600" ]; then + print_warning "SSH key permissions are $perms (should be 600)" + if confirm_action "Fix permissions?"; then + chmod 600 "$ssh_key" + print_success "Permissions fixed" + fi + else + print_success "SSH key permissions are correct (600)" + fi + else + print_error "SSH key not found: $ssh_key" + print_warning "Make sure to create your SSH key before proceeding to Layer 1" + echo "" + echo "To generate a new SSH key:" + echo " ssh-keygen -t ed25519 -f $ssh_key -C \"your-email@example.com\"" + fi + else + print_warning "inventory.ini not found, skipping SSH key validation" + fi +} + +print_summary() { + print_header "Layer 0 Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "Python virtual environment created and activated" + print_success "Ansible and dependencies installed" + print_success "Ansible Galaxy collections installed" + print_success "inventory.ini configured with your hosts" + print_success "infra_vars.yml configured with your domain" + print_success "services_config.yml created with subdomain settings" + print_success "infra_secrets.yml template created" + echo "" + + print_info "Before proceeding to Layer 1:" + echo " 1. Ensure your SSH key is added to all VPS root users" + echo " 2. Verify you can SSH into each machine manually" + echo " 3. Configure DNS nameservers for your domain (if not done)" + echo "" + + print_info "Note about inventory groups:" + echo " • [nodito_vms] group created as placeholder" + echo " • These VMs will be created later on Proxmox" + echo " • Add their host entries to inventory.ini once created" + echo "" + + print_info "To test SSH access to a host:" + echo " ssh -i ~/.ssh/counterganzua root@" + echo "" + + print_info "Next steps:" + echo " 1. Review the files in ansible/" + echo " 2. Test SSH connections to your hosts" + echo " 3. Proceed to Layer 1: ./scripts/setup_layer_1.sh" + echo "" + + print_warning "Remember to activate the venv before running other commands:" + echo " source venv/bin/activate" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🚀 Layer 0: Foundation Setup" + + echo "This script will set up your laptop (lapy) as the Ansible control node." + echo "It will install all prerequisites and configure basic settings." + echo "" + + if ! confirm_action "Continue with Layer 0 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_prerequisites + setup_python_venv + install_python_requirements + install_ansible_collections + setup_inventory_file + setup_infra_vars + setup_services_config + setup_infra_secrets + validate_ssh_key + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_1a_vps.sh b/scripts/setup_layer_1a_vps.sh new file mode 100755 index 0000000..f60452f --- /dev/null +++ b/scripts/setup_layer_1a_vps.sh @@ -0,0 +1,393 @@ +#!/bin/bash + +############################################################################### +# Layer 1A: VPS Basic Setup +# +# This script configures users, SSH, firewall, and fail2ban on VPS machines. +# Runs independently - can be executed without Nodito setup. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_layer_0_complete() { + print_header "Verifying Layer 0 Prerequisites" + + local errors=0 + + # Check if venv exists + if [ ! -d "$PROJECT_ROOT/venv" ]; then + print_error "Python venv not found. Run Layer 0 first." + ((errors++)) + else + print_success "Python venv exists" + fi + + # Check if we're in a venv + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + # Check if Ansible is installed + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found: $(ansible --version | head -n1)" + fi + + # Check if inventory.ini exists + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + # Check if infra_vars.yml exists + if [ ! -f "$ANSIBLE_DIR/infra_vars.yml" ]; then + print_error "infra_vars.yml not found" + ((errors++)) + else + print_success "infra_vars.yml exists" + fi + + if [ $errors -gt 0 ]; then + print_error "Layer 0 is not complete. Please run ./scripts/setup_layer_0.sh first" + exit 1 + fi + + print_success "Layer 0 prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + + # Parse inventory.ini directly - more reliable than ansible-inventory + if [ -f "$ANSIBLE_DIR/inventory.ini" ]; then + # Look for the group section [target] + local in_section=false + local hosts="" + while IFS= read -r line; do + # Remove comments and whitespace + line=$(echo "$line" | sed 's/#.*$//' | xargs) + [ -z "$line" ] && continue + + # Check if we're entering the target section + if [[ "$line" =~ ^\[$target\]$ ]]; then + in_section=true + continue + fi + + # Check if we're entering a different section + if [[ "$line" =~ ^\[.*\]$ ]]; then + in_section=false + continue + fi + + # If we're in the target section, extract hostname + if [ "$in_section" = true ]; then + local hostname=$(echo "$line" | awk '{print $1}') + if [ -n "$hostname" ]; then + hosts="$hosts $hostname" + fi + fi + done < "$ANSIBLE_DIR/inventory.ini" + echo "$hosts" | xargs + fi +} + +check_vps_configured() { + print_header "Checking VPS Configuration" + + # Get all hosts from the vps group + local vps_hosts=$(get_hosts_from_inventory "vps") + local has_vps=false + + # Check for expected VPS hostnames + for expected_host in vipy watchtower spacey; do + if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then + print_success "$expected_host configured" + has_vps=true + else + print_info "$expected_host not configured (skipping)" + fi + done + + if [ "$has_vps" = false ]; then + print_error "No VPSs configured in inventory.ini" + print_info "Add at least one VPS (vipy, watchtower, or spacey) to the [vps] group to proceed" + exit 1 + fi + + echo "" +} + +check_ssh_connectivity() { + print_header "Testing SSH Connectivity as Root" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + print_info "Using SSH key: $ssh_key" + echo "" + + local all_good=true + + # Get all hosts from the vps group + local vps_hosts=$(get_hosts_from_inventory "vps") + + # Test VPSs (vipy, watchtower, spacey) + for expected_host in vipy watchtower spacey; do + if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then + print_info "Testing SSH to $expected_host as root..." + if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes root@$expected_host "echo 'SSH OK'" &>/dev/null; then + print_success "SSH to $expected_host as root: OK" + else + print_error "Cannot SSH to $expected_host as root" + print_warning "Make sure your SSH key is added to root on $expected_host" + all_good=false + fi + fi + done + + if [ "$all_good" = false ]; then + echo "" + print_error "SSH connectivity test failed" + print_info "To fix this:" + echo " 1. Ensure your VPS provider has added your SSH key to root" + echo " 2. Test manually: ssh -i $ssh_key root@" + echo "" + if ! confirm_action "Continue anyway?"; then + exit 1 + fi + fi + + echo "" + print_success "SSH connectivity verified" +} + +############################################################################### +# VPS Setup Functions +############################################################################### + +setup_vps_users_and_access() { + print_header "Setting Up Users and SSH Access on VPSs" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Create the 'counterweight' user with sudo access" + echo " • Configure SSH key authentication" + echo " • Disable root login (optional, configured in playbook)" + echo "" + print_info "Running: ansible-playbook -i inventory.ini infra/01_user_and_access_setup_playbook.yml" + echo "" + + if ! confirm_action "Proceed with user and access setup?"; then + print_warning "Skipped user and access setup" + return 1 + fi + + # Run the playbook with -e 'ansible_user="root"' to use root for this first run + if ansible-playbook -i inventory.ini infra/01_user_and_access_setup_playbook.yml -e 'ansible_user="root"'; then + print_success "User and access setup complete" + return 0 + else + print_error "User and access setup failed" + return 1 + fi +} + +setup_vps_firewall_and_fail2ban() { + print_header "Setting Up Firewall and Fail2ban on VPSs" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Configure UFW firewall with SSH access" + echo " • Install and configure fail2ban for brute force protection" + echo " • Install and configure auditd for security logging" + echo "" + print_info "Running: ansible-playbook -i inventory.ini infra/02_firewall_and_fail2ban_playbook.yml" + echo "" + + if ! confirm_action "Proceed with firewall and fail2ban setup?"; then + print_warning "Skipped firewall setup" + return 1 + fi + + # Now use the default counterweight user + if ansible-playbook -i inventory.ini infra/02_firewall_and_fail2ban_playbook.yml; then + print_success "Firewall and fail2ban setup complete" + return 0 + else + print_error "Firewall setup failed" + return 1 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_layer_1a() { + print_header "Verifying Layer 1A Completion" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + # Test SSH as counterweight user + print_info "Testing SSH as counterweight user..." + echo "" + + local all_good=true + + # Get all hosts from the vps group + local vps_hosts=$(get_hosts_from_inventory "vps") + + for expected_host in vipy watchtower spacey; do + if echo "$vps_hosts" | grep -q "\b$expected_host\b"; then + if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$expected_host "echo 'SSH OK'" &>/dev/null; then + print_success "SSH to $expected_host as counterweight: OK" + else + print_error "Cannot SSH to $expected_host as counterweight" + all_good=false + fi + fi + done + + echo "" + if [ "$all_good" = true ]; then + print_success "All SSH connectivity verified" + else + print_warning "Some SSH tests failed - manual verification recommended" + print_info "Test manually: ssh -i $ssh_key counterweight@" + fi +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 1A: VPS Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "counterweight user created on all VPSs" + print_success "SSH key authentication configured" + print_success "UFW firewall active and configured" + print_success "fail2ban protecting against brute force attacks" + print_success "auditd logging security events" + echo "" + + print_warning "Important Security Changes:" + echo " • Root SSH login is now disabled (by design)" + echo " • Always use 'counterweight' user for SSH access" + echo " • Firewall is active - only SSH allowed by default" + echo "" + + print_info "Next steps:" + echo " 1. Test SSH access: ssh -i ~/.ssh/counterganzua counterweight@" + echo " 2. (Optional) Set up Nodito: ./scripts/setup_layer_1b_nodito.sh" + echo " 3. Proceed to Layer 2: ./scripts/setup_layer_2.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🔧 Layer 1A: VPS Basic Setup" + + echo "This script will configure users, SSH, firewall, and fail2ban on VPS machines." + echo "" + print_info "Targets: vipy, watchtower, spacey" + echo "" + + if ! confirm_action "Continue with Layer 1A setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_layer_0_complete + check_vps_configured + check_ssh_connectivity + + # VPS Setup + local setup_failed=false + setup_vps_users_and_access || setup_failed=true + setup_vps_firewall_and_fail2ban || setup_failed=true + + verify_layer_1a + + if [ "$setup_failed" = true ]; then + print_warning "Some steps failed - please review errors above" + fi + + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_1b_nodito.sh b/scripts/setup_layer_1b_nodito.sh new file mode 100755 index 0000000..5ebb243 --- /dev/null +++ b/scripts/setup_layer_1b_nodito.sh @@ -0,0 +1,411 @@ +#!/bin/bash + +############################################################################### +# Layer 1B: Nodito (Proxmox) Setup +# +# This script configures the Nodito Proxmox server. +# Runs independently - can be executed without VPS setup. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_layer_0_complete() { + print_header "Verifying Layer 0 Prerequisites" + + local errors=0 + + # Check if venv exists + if [ ! -d "$PROJECT_ROOT/venv" ]; then + print_error "Python venv not found. Run Layer 0 first." + ((errors++)) + else + print_success "Python venv exists" + fi + + # Check if we're in a venv + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + # Check if Ansible is installed + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found: $(ansible --version | head -n1)" + fi + + # Check if inventory.ini exists + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + if [ $errors -gt 0 ]; then + print_error "Layer 0 is not complete. Please run ./scripts/setup_layer_0.sh first" + exit 1 + fi + + print_success "Layer 0 prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +check_nodito_configured() { + print_header "Checking Nodito Configuration" + + local nodito_hosts=$(get_hosts_from_inventory "nodito_host") + + if [ -z "$nodito_hosts" ]; then + print_error "No nodito host configured in inventory.ini" + print_info "Add the nodito host to the [nodito_host] group in inventory.ini to proceed" + exit 1 + fi + + print_success "Nodito configured: $nodito_hosts" + echo "" +} + +############################################################################### +# Nodito Setup Functions +############################################################################### + +setup_nodito_bootstrap() { + print_header "Bootstrapping Nodito (Proxmox Server)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Set up SSH key access for root" + echo " • Create the counterweight user with SSH keys" + echo " • Update and secure the system" + echo " • Disable root login and password authentication" + echo "" + print_info "Running: ansible-playbook -i inventory.ini infra/nodito/30_proxmox_bootstrap_playbook.yml" + print_warning "You will be prompted for the root password" + echo "" + + if ! confirm_action "Proceed with nodito bootstrap?"; then + print_warning "Skipped nodito bootstrap" + return 1 + fi + + # Run with root user and ask for password + if ansible-playbook -i inventory.ini infra/nodito/30_proxmox_bootstrap_playbook.yml -e 'ansible_user=root' --ask-pass; then + print_success "Nodito bootstrap complete" + return 0 + else + print_error "Nodito bootstrap failed" + return 1 + fi +} + +setup_nodito_community_repos() { + print_header "Switching Nodito to Community Repositories" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Remove enterprise repository files" + echo " • Add community repository files" + echo " • Disable subscription nag messages" + echo " • Update Proxmox packages" + echo "" + print_info "Running: ansible-playbook -i inventory.ini infra/nodito/31_proxmox_community_repos_playbook.yml" + echo "" + + if ! confirm_action "Proceed with community repos setup?"; then + print_warning "Skipped community repos setup" + return 1 + fi + + if ansible-playbook -i inventory.ini infra/nodito/31_proxmox_community_repos_playbook.yml; then + print_success "Community repositories configured" + print_warning "Clear browser cache before using Proxmox web UI (Ctrl+Shift+R)" + return 0 + else + print_error "Community repos setup failed" + return 1 + fi +} + +setup_nodito_zfs() { + print_header "Setting Up ZFS Storage Pool on Nodito (Optional)" + + cd "$ANSIBLE_DIR" + + print_warning "⚠️ ZFS setup will DESTROY ALL DATA on the specified disks!" + echo "" + print_info "Before proceeding, you must:" + echo " 1. SSH into nodito: ssh root@" + echo " 2. List disks: ls -la /dev/disk/by-id/ | grep -E '(ata-|scsi-|nvme-)'" + echo " 3. Identify the two disk IDs you want to use for RAID 1" + echo " 4. Edit ansible/infra/nodito/nodito_vars.yml" + echo " 5. Set zfs_disk_1 and zfs_disk_2 to your disk IDs" + echo "" + print_info "Example nodito_vars.yml content:" + echo ' zfs_disk_1: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K1234567"' + echo ' zfs_disk_2: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K7654321"' + echo "" + + if [ ! -f "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" ]; then + print_warning "nodito_vars.yml not found" + if confirm_action "Create nodito_vars.yml template?"; then + cat > "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" << 'EOF' +# Nodito Variables +# Configure these before running ZFS setup + +# ZFS Storage Pool Configuration +# Uncomment and configure these lines after identifying your disk IDs: +# zfs_disk_1: "/dev/disk/by-id/ata-YOUR-DISK-1-ID-HERE" +# zfs_disk_2: "/dev/disk/by-id/ata-YOUR-DISK-2-ID-HERE" +# zfs_pool_name: "proxmox-storage" + +# CPU Temperature Monitoring +monitoring_script_dir: /opt/cpu-temp-monitor +monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" +log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" +temp_threshold_celsius: 80 +EOF + print_success "Created nodito_vars.yml template" + print_info "Edit this file and configure ZFS disks, then re-run this script" + fi + return 1 + fi + + # Check if ZFS disks are configured + if ! grep -q "^zfs_disk_1:" "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" 2>/dev/null; then + print_info "ZFS disks not configured in nodito_vars.yml" + print_info "Edit ansible/infra/nodito/nodito_vars.yml to configure disk IDs" + if ! confirm_action "Skip ZFS setup for now?"; then + print_info "Please configure ZFS disks first" + return 1 + fi + print_warning "Skipped ZFS setup" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/nodito/32_zfs_pool_setup_playbook.yml" + echo "" + + if ! confirm_action "⚠️ Proceed with ZFS setup? (THIS WILL DESTROY DATA ON CONFIGURED DISKS)"; then + print_warning "Skipped ZFS setup" + return 1 + fi + + if ansible-playbook -i inventory.ini infra/nodito/32_zfs_pool_setup_playbook.yml; then + print_success "ZFS storage pool configured" + return 0 + else + print_error "ZFS setup failed" + return 1 + fi +} + +setup_nodito_cloud_template() { + print_header "Creating Debian Cloud Template on Nodito (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Download Debian cloud image" + echo " • Create a VM template (ID 9000)" + echo " • Configure cloud-init for easy VM creation" + echo "" + print_info "Running: ansible-playbook -i inventory.ini infra/nodito/33_proxmox_debian_cloud_template.yml" + echo "" + + if ! confirm_action "Proceed with cloud template creation?"; then + print_warning "Skipped cloud template creation" + return 1 + fi + + if ansible-playbook -i inventory.ini infra/nodito/33_proxmox_debian_cloud_template.yml; then + print_success "Debian cloud template created (VM ID 9000)" + return 0 + else + print_error "Cloud template creation failed" + return 1 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_layer_1b() { + print_header "Verifying Layer 1B Completion" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local nodito_hosts=$(get_hosts_from_inventory "nodito") + + print_info "Testing SSH as counterweight user..." + echo "" + + for host in $nodito_hosts; do + if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then + print_success "SSH to $host as counterweight: OK" + else + print_error "Cannot SSH to $host as counterweight" + print_info "Test manually: ssh -i $ssh_key counterweight@$host" + fi + done + + echo "" +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 1B: Nodito Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "Nodito bootstrapped with SSH keys" + print_success "counterweight user created" + print_success "Community repositories configured" + print_success "Root login and password auth disabled" + + if grep -q "^zfs_disk_1:" "$ANSIBLE_DIR/infra/nodito/nodito_vars.yml" 2>/dev/null; then + print_success "ZFS storage pool configured (if you ran it)" + fi + echo "" + + print_warning "Important Security Changes:" + echo " • Root SSH login is now disabled" + echo " • Always use 'counterweight' user for SSH access" + echo " • Password authentication is disabled" + echo "" + + print_info "Proxmox Web UI:" + local nodito_hosts=$(get_hosts_from_inventory "nodito") + echo " • Access at: https://$nodito_hosts:8006" + echo " • Clear browser cache (Ctrl+Shift+R) to avoid UI issues" + echo "" + + print_info "Next steps:" + echo " 1. Test SSH: ssh -i ~/.ssh/counterganzua counterweight@" + echo " 2. Access Proxmox web UI and verify community repos" + echo " 3. Create VMs on Proxmox (if needed)" + echo " 4. Proceed to Layer 2: ./scripts/setup_layer_2.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🖥️ Layer 1B: Nodito (Proxmox) Setup" + + echo "This script will configure your Nodito Proxmox server." + echo "" + print_info "Target: nodito (Proxmox server)" + echo "" + + if ! confirm_action "Continue with Layer 1B setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_layer_0_complete + check_nodito_configured + + # Nodito Setup + local setup_failed=false + setup_nodito_bootstrap || setup_failed=true + setup_nodito_community_repos || setup_failed=true + setup_nodito_zfs || setup_failed=true + setup_nodito_cloud_template || setup_failed=true + + verify_layer_1b + + if [ "$setup_failed" = true ]; then + print_warning "Some optional steps were skipped - this is normal" + fi + + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_2.sh b/scripts/setup_layer_2.sh new file mode 100755 index 0000000..1f35431 --- /dev/null +++ b/scripts/setup_layer_2.sh @@ -0,0 +1,407 @@ +#!/bin/bash + +############################################################################### +# Layer 2: General Infrastructure Tools +# +# This script installs rsync and docker on the machines that need them. +# Must be run after Layer 1A (VPS) or Layer 1B (Nodito) is complete. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_layer_0_complete() { + print_header "Verifying Layer 0 Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + if [ $errors -gt 0 ]; then + print_error "Layer 0 is not complete" + exit 1 + fi + + print_success "Layer 0 prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +check_ssh_connectivity() { + print_header "Testing SSH Connectivity" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local all_good=true + + for group in vipy watchtower spacey nodito; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + for host in $hosts; do + print_info "Testing SSH to $host as counterweight..." + if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then + print_success "SSH to $host: OK" + else + print_error "Cannot SSH to $host as counterweight" + print_warning "Make sure Layer 1A or 1B is complete for this host" + all_good=false + fi + done + fi + done + + if [ "$all_good" = false ]; then + echo "" + print_error "SSH connectivity test failed" + print_info "Ensure Layer 1A (VPS) or Layer 1B (Nodito) is complete" + echo "" + if ! confirm_action "Continue anyway?"; then + exit 1 + fi + fi + + echo "" + print_success "SSH connectivity verified" +} + +############################################################################### +# rsync Installation +############################################################################### + +install_rsync() { + print_header "Installing rsync" + + cd "$ANSIBLE_DIR" + + print_info "rsync is needed for backup operations" + print_info "Recommended hosts: vipy, watchtower, lapy" + echo "" + + # Show available hosts + echo "Available hosts in inventory:" + for group in vipy watchtower spacey nodito lapy; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + fi + done + echo "" + + print_info "Installation options:" + echo " 1. Install on recommended hosts (vipy, watchtower, lapy)" + echo " 2. Install on all hosts" + echo " 3. Custom selection (specify groups)" + echo " 4. Skip rsync installation" + echo "" + + echo -e -n "${BLUE}Choose option${NC} [1-4]: " + read option + + local limit_hosts="" + case "$option" in + 1) + limit_hosts="vipy,watchtower,lapy" + print_info "Installing rsync on: vipy, watchtower, lapy" + ;; + 2) + limit_hosts="all" + print_info "Installing rsync on: all hosts" + ;; + 3) + echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower,nodito)${NC}: " + read limit_hosts + print_info "Installing rsync on: $limit_hosts" + ;; + 4) + print_warning "Skipping rsync installation" + return 1 + ;; + *) + print_error "Invalid option" + return 1 + ;; + esac + + echo "" + if ! confirm_action "Proceed with rsync installation?"; then + print_warning "Skipped rsync installation" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/900_install_rsync.yml --limit $limit_hosts" + echo "" + + if ansible-playbook -i inventory.ini infra/900_install_rsync.yml --limit "$limit_hosts"; then + print_success "rsync installation complete" + return 0 + else + print_error "rsync installation failed" + return 1 + fi +} + +############################################################################### +# Docker Installation +############################################################################### + +install_docker() { + print_header "Installing Docker and Docker Compose" + + cd "$ANSIBLE_DIR" + + print_info "Docker is needed for containerized services" + print_info "Recommended hosts: vipy, watchtower" + echo "" + + # Show available hosts (exclude lapy - docker on laptop is optional) + echo "Available hosts in inventory:" + for group in vipy watchtower spacey nodito; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + fi + done + echo "" + + print_info "Installation options:" + echo " 1. Install on recommended hosts (vipy, watchtower)" + echo " 2. Install on all hosts" + echo " 3. Custom selection (specify groups)" + echo " 4. Skip docker installation" + echo "" + + echo -e -n "${BLUE}Choose option${NC} [1-4]: " + read option + + local limit_hosts="" + case "$option" in + 1) + limit_hosts="vipy,watchtower" + print_info "Installing Docker on: vipy, watchtower" + ;; + 2) + limit_hosts="all" + print_info "Installing Docker on: all hosts" + ;; + 3) + echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower,nodito)${NC}: " + read limit_hosts + print_info "Installing Docker on: $limit_hosts" + ;; + 4) + print_warning "Skipping Docker installation" + return 1 + ;; + *) + print_error "Invalid option" + return 1 + ;; + esac + + echo "" + if ! confirm_action "Proceed with Docker installation?"; then + print_warning "Skipped Docker installation" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/910_docker_playbook.yml --limit $limit_hosts" + echo "" + + if ansible-playbook -i inventory.ini infra/910_docker_playbook.yml --limit "$limit_hosts"; then + print_success "Docker installation complete" + print_warning "You may need to log out and back in for docker group to take effect" + return 0 + else + print_error "Docker installation failed" + return 1 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_installations() { + print_header "Verifying Installations" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + echo "Checking installed tools on hosts..." + echo "" + + # Check all remote hosts + for group in vipy watchtower spacey nodito; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + for host in $hosts; do + print_info "Checking $host..." + + # Check rsync + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v rsync" &>/dev/null; then + print_success "$host: rsync installed" + else + print_warning "$host: rsync not found (may not be needed)" + fi + + # Check docker + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v docker" &>/dev/null; then + print_success "$host: docker installed" + + # Check docker service + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo systemctl is-active docker" &>/dev/null; then + print_success "$host: docker service running" + else + print_warning "$host: docker service not running" + fi + else + print_warning "$host: docker not found (may not be needed)" + fi + + echo "" + done + fi + done +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 2 Setup Complete! 🎉" + + echo "Summary:" + echo "" + print_success "Infrastructure tools installed on specified hosts" + echo "" + + print_info "What was installed:" + echo " • rsync - for backup operations" + echo " • docker + docker compose - for containerized services" + echo "" + + print_info "Next steps:" + echo " 1. Proceed to Layer 3: ./scripts/setup_layer_3_caddy.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🔧 Layer 2: General Infrastructure Tools" + + echo "This script will install rsync and docker on your infrastructure." + echo "" + + if ! confirm_action "Continue with Layer 2 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_layer_0_complete + check_ssh_connectivity + + # Install tools + install_rsync + echo "" + install_docker + + verify_installations + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_3_caddy.sh b/scripts/setup_layer_3_caddy.sh new file mode 100755 index 0000000..2ce0f6d --- /dev/null +++ b/scripts/setup_layer_3_caddy.sh @@ -0,0 +1,355 @@ +#!/bin/bash + +############################################################################### +# Layer 3: Reverse Proxy (Caddy) +# +# This script deploys Caddy reverse proxy on VPS machines. +# Must be run after Layer 1A (VPS setup) is complete. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_layer_0_complete() { + print_header "Verifying Layer 0 Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + if [ $errors -gt 0 ]; then + print_error "Layer 0 is not complete" + exit 1 + fi + + print_success "Layer 0 prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +check_target_hosts() { + print_header "Checking Target Hosts" + + local has_hosts=false + + print_info "Caddy will be deployed to these hosts:" + echo "" + + for group in vipy watchtower spacey; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + has_hosts=true + else + print_warning "[$group]: not configured (skipping)" + fi + done + + echo "" + + if [ "$has_hosts" = false ]; then + print_error "No target hosts configured for Caddy" + print_info "Caddy needs vipy, watchtower, or spacey in inventory.ini" + exit 1 + fi + + print_success "Target hosts verified" +} + +check_ssh_connectivity() { + print_header "Testing SSH Connectivity" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local all_good=true + + for group in vipy watchtower spacey; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + for host in $hosts; do + print_info "Testing SSH to $host as counterweight..." + if timeout 10 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "echo 'SSH OK'" &>/dev/null; then + print_success "SSH to $host: OK" + else + print_error "Cannot SSH to $host as counterweight" + print_warning "Make sure Layer 1A is complete for this host" + all_good=false + fi + done + fi + done + + if [ "$all_good" = false ]; then + echo "" + print_error "SSH connectivity test failed" + print_info "Ensure Layer 1A (VPS setup) is complete" + echo "" + if ! confirm_action "Continue anyway?"; then + exit 1 + fi + fi + + echo "" + print_success "SSH connectivity verified" +} + +############################################################################### +# Caddy Deployment +############################################################################### + +deploy_caddy() { + print_header "Deploying Caddy" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Install Caddy from official repositories" + echo " • Configure Caddy service" + echo " • Open firewall ports 80/443" + echo " • Create sites-enabled directory structure" + echo " • Enable automatic HTTPS with Let's Encrypt" + echo "" + + print_info "Target hosts: vipy, watchtower, spacey (if configured)" + echo "" + + print_warning "Important:" + echo " • Caddy will start with empty configuration" + echo " • Services will add their own config files in later layers" + echo " • Ports 80/443 must be available on the VPSs" + echo "" + + if ! confirm_action "Proceed with Caddy deployment?"; then + print_warning "Skipped Caddy deployment" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/caddy_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/caddy_playbook.yml; then + print_success "Caddy deployment complete" + return 0 + else + print_error "Caddy deployment failed" + return 1 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_caddy() { + print_header "Verifying Caddy Installation" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + echo "Checking Caddy on each host..." + echo "" + + for group in vipy watchtower spacey; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + for host in $hosts; do + print_info "Checking $host..." + + # Check if caddy is installed + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "command -v caddy" &>/dev/null; then + print_success "$host: Caddy installed" + else + print_error "$host: Caddy not found" + continue + fi + + # Check if caddy service is running + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo systemctl is-active caddy" &>/dev/null; then + print_success "$host: Caddy service running" + else + print_error "$host: Caddy service not running" + fi + + # Check if sites-enabled directory exists + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "test -d /etc/caddy/sites-enabled" &>/dev/null; then + print_success "$host: sites-enabled directory exists" + else + print_warning "$host: sites-enabled directory not found" + fi + + # Check if ports 80/443 are open + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$host "sudo ufw status | grep -E '80|443'" &>/dev/null; then + print_success "$host: Firewall ports 80/443 open" + else + print_warning "$host: Could not verify firewall ports" + fi + + echo "" + done + fi + done +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 3 Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "Caddy installed on VPS hosts" + print_success "Caddy service running" + print_success "Firewall ports 80/443 opened" + print_success "Sites-enabled directory structure created" + echo "" + + print_info "What Caddy provides:" + echo " • Automatic HTTPS with Let's Encrypt" + echo " • Reverse proxy for all web services" + echo " • HTTP/2 support" + echo " • Simple per-service configuration" + echo "" + + print_info "How services use Caddy:" + echo " • Each service adds a config file to /etc/caddy/sites-enabled/" + echo " • Main Caddyfile imports all configs" + echo " • Caddy automatically manages SSL certificates" + echo "" + + print_warning "Important Notes:" + echo " • Caddy is currently running with default/empty config" + echo " • Services deployed in later layers will add their configs" + echo " • DNS must point to your VPS IPs for SSL to work" + echo "" + + print_info "Next steps:" + echo " 1. Verify Caddy is accessible (optional): curl http://" + echo " 2. Proceed to Layer 4: ./scripts/setup_layer_4_monitoring.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🌐 Layer 3: Reverse Proxy (Caddy)" + + echo "This script will deploy Caddy reverse proxy on your VPS machines." + echo "" + print_info "Targets: vipy, watchtower, spacey" + echo "" + + if ! confirm_action "Continue with Layer 3 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_layer_0_complete + check_target_hosts + check_ssh_connectivity + + # Deploy Caddy + if deploy_caddy; then + verify_caddy + print_summary + else + print_error "Caddy deployment failed" + exit 1 + fi +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_4_monitoring.sh b/scripts/setup_layer_4_monitoring.sh new file mode 100755 index 0000000..d82ad41 --- /dev/null +++ b/scripts/setup_layer_4_monitoring.sh @@ -0,0 +1,806 @@ +#!/bin/bash + +############################################################################### +# Layer 4: Core Monitoring & Notifications +# +# This script deploys ntfy and Uptime Kuma on watchtower. +# Must be run after Layers 1A, 2, and 3 are complete. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +get_host_ip() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +hostvars = data.get('_meta', {}).get('hostvars', {}) +if target in hostvars: + print(hostvars[target].get('ansible_host', target)) +else: + hosts = data.get(target, {}).get('hosts', []) + if hosts: + first = hosts[0] + hv = hostvars.get(first, {}) + print(hv.get('ansible_host', first)) +PY +} + +############################################################################### +# Verification Functions +############################################################################### + +check_prerequisites() { + print_header "Verifying Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + # Check if watchtower is configured + if [ -z "$(get_hosts_from_inventory "watchtower")" ]; then + print_error "watchtower not configured in inventory.ini" + print_info "Layer 4 requires watchtower VPS" + ((errors++)) + else + print_success "watchtower configured in inventory" + fi + + if [ $errors -gt 0 ]; then + print_error "Prerequisites not met" + exit 1 + fi + + print_success "Prerequisites verified" +} + +check_vars_files() { + print_header "Checking Configuration Files" + + # Check services_config.yml + if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then + print_error "services_config.yml not found" + print_info "This file should have been created in Layer 0" + exit 1 + fi + + print_success "services_config.yml exists" + + # Show configured subdomains + local ntfy_sub=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") + local uptime_sub=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") + + print_info "Configured subdomains:" + echo " • ntfy: $ntfy_sub" + echo " • uptime_kuma: $uptime_sub" + echo "" +} + +check_dns_configuration() { + print_header "Validating DNS Configuration" + + cd "$ANSIBLE_DIR" + + # Get watchtower IP + local watchtower_ip=$(get_host_ip "watchtower") + + if [ -z "$watchtower_ip" ]; then + print_error "Could not determine watchtower IP from inventory" + return 1 + fi + + print_info "Watchtower IP: $watchtower_ip" + echo "" + + # Get domain from infra_vars.yml + local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) + + if [ -z "$root_domain" ]; then + print_error "Could not determine root_domain from infra_vars.yml" + return 1 + fi + + # Get subdomains from centralized config + local ntfy_subdomain="ntfy" + local uptime_subdomain="uptime" + + if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then + ntfy_subdomain=$(grep "^ ntfy:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "ntfy") + uptime_subdomain=$(grep "^ uptime_kuma:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "uptime") + fi + + local ntfy_fqdn="${ntfy_subdomain}.${root_domain}" + local uptime_fqdn="${uptime_subdomain}.${root_domain}" + + print_info "Checking DNS records..." + echo "" + + local dns_ok=true + + # Check ntfy DNS + print_info "Checking $ntfy_fqdn..." + if command -v dig &> /dev/null; then + local ntfy_resolved=$(dig +short "$ntfy_fqdn" | head -n1) + if [ "$ntfy_resolved" = "$watchtower_ip" ]; then + print_success "$ntfy_fqdn → $ntfy_resolved ✓" + elif [ -n "$ntfy_resolved" ]; then + print_error "$ntfy_fqdn → $ntfy_resolved (expected $watchtower_ip)" + dns_ok=false + else + print_error "$ntfy_fqdn does not resolve" + dns_ok=false + fi + else + print_warning "dig command not found, skipping DNS validation" + print_info "Install dnsutils/bind-tools to enable DNS validation" + return 1 + fi + + # Check Uptime Kuma DNS + print_info "Checking $uptime_fqdn..." + if command -v dig &> /dev/null; then + local uptime_resolved=$(dig +short "$uptime_fqdn" | head -n1) + if [ "$uptime_resolved" = "$watchtower_ip" ]; then + print_success "$uptime_fqdn → $uptime_resolved ✓" + elif [ -n "$uptime_resolved" ]; then + print_error "$uptime_fqdn → $uptime_resolved (expected $watchtower_ip)" + dns_ok=false + else + print_error "$uptime_fqdn does not resolve" + dns_ok=false + fi + fi + + echo "" + + if [ "$dns_ok" = false ]; then + print_error "DNS validation failed" + print_info "Please configure DNS records:" + echo " • $ntfy_fqdn → $watchtower_ip" + echo " • $uptime_fqdn → $watchtower_ip" + echo "" + print_warning "DNS changes can take time to propagate (up to 24-48 hours)" + echo "" + if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then + exit 1 + fi + else + print_success "DNS validation passed" + fi +} + +############################################################################### +# ntfy Deployment +############################################################################### + +deploy_ntfy() { + print_header "Deploying ntfy (Notification Service)" + + cd "$ANSIBLE_DIR" + + print_info "ntfy requires admin credentials for authentication" + echo "" + + # Check if env vars are set + if [ -z "$NTFY_USER" ] || [ -z "$NTFY_PASSWORD" ]; then + print_warning "NTFY_USER and NTFY_PASSWORD environment variables not set" + echo "" + print_info "Please enter credentials for ntfy admin user:" + echo "" + + echo -e -n "${BLUE}ntfy admin username${NC} [admin]: " + read ntfy_user + ntfy_user="${ntfy_user:-admin}" + + echo -e -n "${BLUE}ntfy admin password${NC}: " + read -s ntfy_password + echo "" + + if [ -z "$ntfy_password" ]; then + print_error "Password cannot be empty" + return 1 + fi + + export NTFY_USER="$ntfy_user" + export NTFY_PASSWORD="$ntfy_password" + else + print_success "Using NTFY_USER and NTFY_PASSWORD from environment" + fi + + echo "" + print_info "This will:" + echo " • Install ntfy from official repositories" + echo " • Configure ntfy with authentication (deny-all by default)" + echo " • Create admin user: $NTFY_USER" + echo " • Set up Caddy reverse proxy" + echo "" + + if ! confirm_action "Proceed with ntfy deployment?"; then + print_warning "Skipped ntfy deployment" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/ntfy/deploy_ntfy_playbook.yml; then + print_success "ntfy deployment complete" + echo "" + print_info "ntfy is now available at your configured subdomain" + print_info "Admin user: $NTFY_USER" + return 0 + else + print_error "ntfy deployment failed" + return 1 + fi +} + +############################################################################### +# Uptime Kuma Deployment +############################################################################### + +deploy_uptime_kuma() { + print_header "Deploying Uptime Kuma (Monitoring Platform)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Deploy Uptime Kuma via Docker" + echo " • Configure Caddy reverse proxy" + echo " • Set up data persistence" + echo "" + + if ! confirm_action "Proceed with Uptime Kuma deployment?"; then + print_warning "Skipped Uptime Kuma deployment" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/uptime_kuma/deploy_uptime_kuma_playbook.yml; then + print_success "Uptime Kuma deployment complete" + echo "" + print_warning "IMPORTANT: First-time setup required" + echo " 1. Access Uptime Kuma at your configured subdomain" + echo " 2. Create admin user on first visit" + echo " 3. Update ansible/infra_secrets.yml with credentials" + return 0 + else + print_error "Uptime Kuma deployment failed" + return 1 + fi +} + +############################################################################### +# Backup Configuration +############################################################################### + +setup_uptime_kuma_backup() { + print_header "Setting Up Uptime Kuma Backup (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "This will set up automated backups to lapy" + echo "" + + if ! confirm_action "Set up Uptime Kuma backup to lapy?"; then + print_warning "Skipped backup setup" + return 0 + fi + + # Check if rsync is available + print_info "Verifying rsync is installed on watchtower and lapy..." + if ! ansible watchtower -i inventory.ini -m shell -a "command -v rsync" &>/dev/null; then + print_error "rsync not found on watchtower" + print_info "Run Layer 2 to install rsync" + print_warning "Backup setup skipped - rsync not available" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml" + echo "" + + if ansible-playbook -i inventory.ini services/uptime_kuma/setup_backup_uptime_kuma_to_lapy.yml; then + print_success "Uptime Kuma backup configured" + print_info "Backups will run periodically via cron" + return 0 + else + print_error "Backup setup failed" + return 1 + fi +} + +############################################################################### +# Post-Deployment Configuration +############################################################################### + +setup_ntfy_notification() { + print_header "Setting Up ntfy Notification in Uptime Kuma (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "This will automatically configure ntfy as a notification method in Uptime Kuma" + print_warning "Prerequisites:" + echo " • Uptime Kuma admin account must be created first" + echo " • infra_secrets.yml must have Uptime Kuma credentials" + echo "" + + if ! confirm_action "Set up ntfy notification in Uptime Kuma?"; then + print_warning "Skipped ntfy notification setup" + print_info "You can set this up manually or run this script again later" + return 0 + fi + + # Check if infra_secrets.yml has Uptime Kuma credentials + if ! grep -q "uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null || \ + ! grep -q "uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then + print_error "Uptime Kuma credentials not found in infra_secrets.yml" + print_info "Please complete Step 1 and 2 of post-deployment steps first:" + echo " 1. Create admin user in Uptime Kuma web UI" + echo " 2. Add credentials to ansible/infra_secrets.yml" + print_warning "Skipped - you can run this script again after completing those steps" + return 0 + fi + + # Check credentials are not empty + local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + + if [ -z "$uk_user" ] || [ -z "$uk_pass" ]; then + print_error "Uptime Kuma credentials are empty in infra_secrets.yml" + print_info "Please update ansible/infra_secrets.yml with your credentials" + return 0 + fi + + print_success "Found Uptime Kuma credentials in infra_secrets.yml" + + print_info "Running playbook to configure ntfy notification..." + echo "" + + if ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml; then + print_success "ntfy notification configured in Uptime Kuma" + print_info "You can now use ntfy for all your monitors!" + return 0 + else + print_error "Failed to configure ntfy notification" + print_info "You can set this up manually or run the playbook again later:" + echo " ansible-playbook -i inventory.ini services/ntfy/setup_ntfy_uptime_kuma_notification.yml" + return 0 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_deployments() { + print_header "Verifying Deployments" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local watchtower_host + watchtower_host=$(get_hosts_from_inventory "watchtower") + + if [ -z "$watchtower_host" ]; then + print_error "Could not determine watchtower host" + return + fi + + print_info "Checking services on watchtower ($watchtower_host)..." + echo "" + + # Check ntfy + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "systemctl is-active ntfy" &>/dev/null; then + print_success "ntfy service running" + else + print_warning "ntfy service not running or not installed" + fi + + # Check Uptime Kuma docker container + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "docker ps | grep uptime-kuma" &>/dev/null; then + print_success "Uptime Kuma container running" + else + print_warning "Uptime Kuma container not running" + fi + + # Check Caddy configs + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/ntfy.conf" &>/dev/null; then + print_success "ntfy Caddy config exists" + else + print_warning "ntfy Caddy config not found" + fi + + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$watchtower_host "test -f /etc/caddy/sites-enabled/uptime-kuma.conf" &>/dev/null; then + print_success "Uptime Kuma Caddy config exists" + else + print_warning "Uptime Kuma Caddy config not found" + fi + + echo "" +} + +verify_final_setup() { + print_header "Final Verification - Post-Deployment Steps" + + cd "$ANSIBLE_DIR" + + print_info "Checking if all post-deployment steps were completed..." + echo "" + + local all_ok=true + + # Check 1: infra_secrets.yml has Uptime Kuma credentials + print_info "Checking infra_secrets.yml..." + if grep -q "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null && \ + grep -q "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then + local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + + if [ -n "$uk_user" ] && [ -n "$uk_pass" ] && [ "$uk_user" != '""' ] && [ "$uk_pass" != '""' ]; then + print_success "Uptime Kuma credentials configured in infra_secrets.yml" + else + print_error "Uptime Kuma credentials are empty in infra_secrets.yml" + print_info "Please complete Step 2: Update infra_secrets.yml" + all_ok=false + fi + else + print_error "Uptime Kuma credentials not found in infra_secrets.yml" + print_info "Please complete Step 2: Update infra_secrets.yml" + all_ok=false + fi + + echo "" + + # Check 2: Can connect to Uptime Kuma API + print_info "Checking Uptime Kuma API access..." + + if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then + # Create a test Python script to check API access + local test_script=$(mktemp) + cat > "$test_script" << 'EOFPYTHON' +import sys +import yaml +from uptime_kuma_api import UptimeKumaApi + +try: + # Load config + with open('infra_vars.yml', 'r') as f: + infra_vars = yaml.safe_load(f) + + with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: + uk_vars = yaml.safe_load(f) + + with open('infra_secrets.yml', 'r') as f: + secrets = yaml.safe_load(f) + + root_domain = infra_vars.get('root_domain') + subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') + url = f"https://{subdomain}.{root_domain}" + + username = secrets.get('uptime_kuma_username') + password = secrets.get('uptime_kuma_password') + + # Try to connect + api = UptimeKumaApi(url) + api.login(username, password) + + # Check if we can get monitors + monitors = api.get_monitors() + + print(f"SUCCESS:{len(monitors)}") + api.disconnect() + sys.exit(0) + +except Exception as e: + print(f"ERROR:{str(e)}", file=sys.stderr) + sys.exit(1) +EOFPYTHON + + local result=$(cd "$ANSIBLE_DIR" && python3 "$test_script" 2>&1) + rm -f "$test_script" + + if echo "$result" | grep -q "^SUCCESS:"; then + local monitor_count=$(echo "$result" | grep "^SUCCESS:" | cut -d: -f2) + print_success "Successfully connected to Uptime Kuma API" + print_info "Current monitors: $monitor_count" + else + print_error "Cannot connect to Uptime Kuma API" + print_warning "This usually means:" + echo " • Admin account not created yet (Step 1)" + echo " • Wrong credentials in infra_secrets.yml (Step 2)" + echo " • Uptime Kuma not accessible" + all_ok=false + fi + else + print_warning "Skipping API check - credentials not configured" + all_ok=false + fi + + echo "" + + # Check 3: ntfy notification configured in Uptime Kuma + print_info "Checking ntfy notification configuration..." + + if [ -n "$uk_user" ] && [ -n "$uk_pass" ]; then + local test_notif=$(mktemp) + cat > "$test_notif" << 'EOFPYTHON' +import sys +import yaml +from uptime_kuma_api import UptimeKumaApi + +try: + # Load config + with open('infra_vars.yml', 'r') as f: + infra_vars = yaml.safe_load(f) + + with open('services/uptime_kuma/uptime_kuma_vars.yml', 'r') as f: + uk_vars = yaml.safe_load(f) + + with open('infra_secrets.yml', 'r') as f: + secrets = yaml.safe_load(f) + + root_domain = infra_vars.get('root_domain') + subdomain = uk_vars.get('uptime_kuma_subdomain', 'uptime') + url = f"https://{subdomain}.{root_domain}" + + username = secrets.get('uptime_kuma_username') + password = secrets.get('uptime_kuma_password') + + # Connect + api = UptimeKumaApi(url) + api.login(username, password) + + # Check for ntfy notification + notifications = api.get_notifications() + ntfy_found = any(n.get('type') == 'ntfy' for n in notifications) + + if ntfy_found: + print("SUCCESS:ntfy notification configured") + else: + print("NOTFOUND:No ntfy notification found") + + api.disconnect() + sys.exit(0) + +except Exception as e: + print(f"ERROR:{str(e)}", file=sys.stderr) + sys.exit(1) +EOFPYTHON + + local notif_result=$(cd "$ANSIBLE_DIR" && python3 "$test_notif" 2>&1) + rm -f "$test_notif" + + if echo "$notif_result" | grep -q "^SUCCESS:"; then + print_success "ntfy notification is configured in Uptime Kuma" + elif echo "$notif_result" | grep -q "^NOTFOUND:"; then + print_warning "ntfy notification not yet configured" + print_info "Run the script again and choose 'yes' for ntfy notification setup" + print_info "Or complete Step 3 manually" + all_ok=false + else + print_warning "Could not verify ntfy notification (API access issue)" + fi + else + print_warning "Skipping ntfy check - credentials not configured" + fi + + echo "" + + # Summary + if [ "$all_ok" = true ]; then + print_success "All post-deployment steps completed! ✓" + echo "" + print_info "Layer 4 is fully configured and ready to use" + print_info "You can now proceed to Layer 6 (infrastructure monitoring)" + return 0 + else + print_warning "Some post-deployment steps are incomplete" + echo "" + print_info "Complete these steps:" + echo " 1. Access Uptime Kuma web UI and create admin account" + echo " 2. Update ansible/infra_secrets.yml with credentials" + echo " 3. Run this script again to configure ntfy notification" + echo "" + print_info "You can also complete manually and verify with:" + echo " ./scripts/setup_layer_4_monitoring.sh" + return 1 + fi +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 4 Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "ntfy notification service deployed" + print_success "Uptime Kuma monitoring platform deployed" + print_success "Caddy reverse proxy configured for both services" + echo "" + + print_warning "REQUIRED POST-DEPLOYMENT STEPS:" + echo "" + echo "MANUAL (do these first):" + echo " 1. Access Uptime Kuma Web UI and create admin account" + echo " 2. Update ansible/infra_secrets.yml with credentials" + echo "" + echo "AUTOMATED (script can do these):" + echo " 3. Configure ntfy notification - script will offer to set this up" + echo " 4. Final verification - script will check everything" + echo "" + print_info "After completing steps 1 & 2, the script will:" + echo " • Automatically configure ntfy in Uptime Kuma" + echo " • Verify all post-deployment steps" + echo " • Tell you if anything is missing" + echo "" + print_warning "You MUST complete steps 1 & 2 before proceeding to Layer 6!" + echo "" + + print_info "What these services enable:" + echo " • ntfy: Push notifications to your devices" + echo " • Uptime Kuma: Monitor all services and infrastructure" + echo " • Together: Complete monitoring and alerting solution" + echo "" + + print_info "Next steps:" + echo " 1. Complete the post-deployment steps above" + echo " 2. Test ntfy: Send a test notification" + echo " 3. Test Uptime Kuma: Create a test monitor" + echo " 4. Proceed to Layer 5: ./scripts/setup_layer_5_headscale.sh (optional)" + echo " OR Layer 6: ./scripts/setup_layer_6_infra_monitoring.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "📊 Layer 4: Core Monitoring & Notifications" + + echo "This script will deploy ntfy and Uptime Kuma on watchtower." + echo "" + print_info "Services to deploy:" + echo " • ntfy (notification service)" + echo " • Uptime Kuma (monitoring platform)" + echo "" + + if ! confirm_action "Continue with Layer 4 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_prerequisites + check_vars_files + check_dns_configuration + + # Deploy services (don't fail if skipped) + deploy_ntfy || true + echo "" + deploy_uptime_kuma || true + echo "" + setup_uptime_kuma_backup || true + + echo "" + verify_deployments + + # Always show summary and offer ntfy configuration + print_summary + echo "" + + # Always ask about ntfy notification setup (regardless of deployment status) + print_header "Configure ntfy Notification in Uptime Kuma" + print_info "After creating your Uptime Kuma admin account and updating infra_secrets.yml," + print_info "the script can automatically configure ntfy as a notification method." + echo "" + print_warning "Prerequisites:" + echo " 1. Access Uptime Kuma web UI and create admin account" + echo " 2. Update ansible/infra_secrets.yml with your credentials" + echo "" + + # Always offer to set up ntfy notification + setup_ntfy_notification + + # Final verification + echo "" + verify_final_setup +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_5_headscale.sh b/scripts/setup_layer_5_headscale.sh new file mode 100755 index 0000000..0c89745 --- /dev/null +++ b/scripts/setup_layer_5_headscale.sh @@ -0,0 +1,524 @@ +#!/bin/bash + +############################################################################### +# Layer 5: VPN Infrastructure (Headscale) +# +# This script deploys Headscale and optionally joins machines to the mesh. +# Must be run after Layers 0, 1A, and 3 are complete. +# THIS LAYER IS OPTIONAL - skip to Layer 6 if you don't need VPN. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_prerequisites() { + print_header "Verifying Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + # Check if spacey is configured + if [ -z "$(get_hosts_from_inventory "spacey")" ]; then + print_error "spacey not configured in inventory.ini" + print_info "Layer 5 requires spacey VPS for Headscale server" + ((errors++)) + else + print_success "spacey configured in inventory" + fi + + if [ $errors -gt 0 ]; then + print_error "Prerequisites not met" + exit 1 + fi + + print_success "Prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +get_host_ip() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +hostvars = data.get('_meta', {}).get('hostvars', {}) +if target in hostvars: + print(hostvars[target].get('ansible_host', target)) +else: + hosts = data.get(target, {}).get('hosts', []) + if hosts: + first = hosts[0] + hv = hostvars.get(first, {}) + print(hv.get('ansible_host', first)) +PY +} + +check_vars_files() { + print_header "Checking Configuration Files" + + # Check services_config.yml + if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then + print_error "services_config.yml not found" + print_info "This file should have been created in Layer 0" + exit 1 + fi + + print_success "services_config.yml exists" + + # Show configured subdomain + local hs_sub=$(grep "^ headscale:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "headscale") + print_info "Configured subdomain: headscale: $hs_sub" + echo "" +} + +check_dns_configuration() { + print_header "Validating DNS Configuration" + + cd "$ANSIBLE_DIR" + + # Get spacey IP + local spacey_ip=$(get_host_ip "spacey") + + if [ -z "$spacey_ip" ]; then + print_error "Could not determine spacey IP from inventory" + return 1 + fi + + print_info "Spacey IP: $spacey_ip" + echo "" + + # Get domain from infra_vars.yml + local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) + + if [ -z "$root_domain" ]; then + print_error "Could not determine root_domain from infra_vars.yml" + return 1 + fi + + # Get subdomain from centralized config + local headscale_subdomain="headscale" + + if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then + headscale_subdomain=$(grep "^ headscale:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "headscale") + fi + + local headscale_fqdn="${headscale_subdomain}.${root_domain}" + + print_info "Checking DNS record..." + echo "" + + # Check Headscale DNS + print_info "Checking $headscale_fqdn..." + if command -v dig &> /dev/null; then + local resolved=$(dig +short "$headscale_fqdn" | head -n1) + if [ "$resolved" = "$spacey_ip" ]; then + print_success "$headscale_fqdn → $resolved ✓" + elif [ -n "$resolved" ]; then + print_error "$headscale_fqdn → $resolved (expected $spacey_ip)" + print_warning "DNS changes can take time to propagate (up to 24-48 hours)" + echo "" + if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then + exit 1 + fi + else + print_error "$headscale_fqdn does not resolve" + print_warning "DNS changes can take time to propagate" + echo "" + if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then + exit 1 + fi + fi + else + print_warning "dig command not found, skipping DNS validation" + print_info "Install dnsutils/bind-tools to enable DNS validation" + fi + + echo "" + print_success "DNS validation complete" +} + +############################################################################### +# Headscale Deployment +############################################################################### + +deploy_headscale() { + print_header "Deploying Headscale Server" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Install Headscale on spacey" + echo " • Configure with deny-all ACL policy (you customize later)" + echo " • Create namespace for your network" + echo " • Set up Caddy reverse proxy" + echo " • Configure embedded DERP server" + echo "" + + print_warning "After deployment, you MUST configure ACL policies for machines to communicate" + echo "" + + if ! confirm_action "Proceed with Headscale deployment?"; then + print_warning "Skipped Headscale deployment" + return 1 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/headscale/deploy_headscale_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/headscale/deploy_headscale_playbook.yml; then + print_success "Headscale deployment complete" + return 0 + else + print_error "Headscale deployment failed" + return 1 + fi +} + +############################################################################### +# Join Machines to Mesh +############################################################################### + +join_machines_to_mesh() { + print_header "Join Machines to Mesh (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "This will install Tailscale client and join machines to your Headscale mesh" + echo "" + + # Show available hosts + echo "Available hosts to join:" + for group in vipy watchtower nodito lapy; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + fi + done + echo "" + + print_info "Join options:" + echo " 1. Join recommended machines (vipy, watchtower, nodito)" + echo " 2. Join all machines" + echo " 3. Custom selection (specify groups)" + echo " 4. Skip - join machines later manually" + echo "" + + echo -e -n "${BLUE}Choose option${NC} [1-4]: " + read option + + local limit_hosts="" + case "$option" in + 1) + limit_hosts="vipy,watchtower,nodito" + print_info "Joining: vipy, watchtower, nodito" + ;; + 2) + limit_hosts="all" + print_info "Joining: all hosts" + ;; + 3) + echo -e -n "${BLUE}Enter groups (comma-separated, e.g., vipy,watchtower)${NC}: " + read limit_hosts + print_info "Joining: $limit_hosts" + ;; + 4) + print_warning "Skipping machine join - you can join manually later" + print_info "To join manually:" + echo " ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit " + return 0 + ;; + *) + print_error "Invalid option" + return 0 + ;; + esac + + echo "" + if ! confirm_action "Proceed with joining machines?"; then + print_warning "Skipped joining machines" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit $limit_hosts" + echo "" + + if ansible-playbook -i inventory.ini infra/920_join_headscale_mesh.yml --limit "$limit_hosts"; then + print_success "Machines joined to mesh" + return 0 + else + print_error "Failed to join some machines" + print_info "You can retry or join manually later" + return 0 + fi +} + +############################################################################### +# Backup Configuration +############################################################################### + +setup_headscale_backup() { + print_header "Setting Up Headscale Backup (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "This will set up automated backups to lapy" + echo "" + + if ! confirm_action "Set up Headscale backup to lapy?"; then + print_warning "Skipped backup setup" + return 0 + fi + + # Check if rsync is available + print_info "Verifying rsync is installed on spacey and lapy..." + if ! ansible spacey -i inventory.ini -m shell -a "command -v rsync" &>/dev/null; then + print_error "rsync not found on spacey" + print_info "Run Layer 2 to install rsync" + print_warning "Backup setup skipped - rsync not available" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml" + echo "" + + if ansible-playbook -i inventory.ini services/headscale/setup_backup_headscale_to_lapy.yml; then + print_success "Headscale backup configured" + print_info "Backups will run periodically via cron" + return 0 + else + print_error "Backup setup failed" + return 0 + fi +} + +############################################################################### +# Verification Functions +############################################################################### + +verify_deployment() { + print_header "Verifying Headscale Deployment" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local spacey_host=$(get_hosts_from_inventory "spacey") + + if [ -z "$spacey_host" ]; then + print_error "Could not determine spacey host" + return + fi + + print_info "Checking Headscale on spacey ($spacey_host)..." + echo "" + + # Check Headscale service + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "systemctl is-active headscale" &>/dev/null; then + print_success "Headscale service running" + else + print_warning "Headscale service not running" + fi + + # Check Caddy config + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "test -f /etc/caddy/sites-enabled/headscale.conf" &>/dev/null; then + print_success "Headscale Caddy config exists" + else + print_warning "Headscale Caddy config not found" + fi + + # Check ACL file + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "test -f /etc/headscale/acl.json" &>/dev/null; then + print_success "ACL policy file exists" + else + print_warning "ACL policy file not found" + fi + + # List nodes + print_info "Attempting to list connected nodes..." + local nodes_output=$(timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$spacey_host "sudo headscale nodes list" 2>/dev/null || echo "") + + if [ -n "$nodes_output" ]; then + echo "$nodes_output" + else + print_warning "Could not list nodes (this is normal if no machines joined yet)" + fi + + echo "" +} + +############################################################################### +# Summary Functions +############################################################################### + +print_summary() { + print_header "Layer 5 Setup Complete! 🎉" + + echo "Summary of what was configured:" + echo "" + print_success "Headscale VPN server deployed on spacey" + print_success "Caddy reverse proxy configured" + print_success "Namespace created for your network" + echo "" + + print_warning "CRITICAL POST-DEPLOYMENT STEPS:" + echo "" + echo "1. Configure ACL Policies (REQUIRED for machines to communicate):" + echo " • SSH to spacey: ssh counterweight@" + echo " • Edit ACL: sudo nano /etc/headscale/acl.json" + echo " • Add rules to allow communication" + echo " • Restart: sudo systemctl restart headscale" + echo "" + echo "2. Verify machines joined (if you selected that option):" + echo " • SSH to spacey: ssh counterweight@" + echo " • List nodes: sudo headscale nodes list" + echo "" + echo "3. Join additional machines (mobile, desktop):" + echo " • Generate key: sudo headscale preauthkeys create --user --reusable" + echo " • On device: tailscale up --login-server https:// --authkey " + echo "" + + print_info "What Headscale enables:" + echo " • Secure mesh networking between all machines" + echo " • Magic DNS - access machines by hostname" + echo " • NAT traversal - works behind firewalls" + echo " • Self-hosted Tailscale alternative" + echo "" + + print_info "Next steps:" + echo " 1. Configure ACL policies on spacey" + echo " 2. Verify nodes are connected" + echo " 3. Proceed to Layer 6: ./scripts/setup_layer_6_infra_monitoring.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🔐 Layer 5: VPN Infrastructure (Headscale)" + + echo "This script will deploy Headscale for secure mesh networking." + echo "" + print_warning "THIS LAYER IS OPTIONAL" + print_info "Skip to Layer 6 if you don't need VPN mesh networking" + echo "" + + if ! confirm_action "Continue with Layer 5 setup?"; then + echo "Setup skipped - proceeding to Layer 6 is fine!" + exit 0 + fi + + check_prerequisites + check_vars_files + check_dns_configuration + + # Deploy Headscale + if deploy_headscale; then + echo "" + join_machines_to_mesh + echo "" + setup_headscale_backup + echo "" + verify_deployment + print_summary + else + print_error "Headscale deployment failed" + exit 1 + fi +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_6_infra_monitoring.sh b/scripts/setup_layer_6_infra_monitoring.sh new file mode 100755 index 0000000..7c12780 --- /dev/null +++ b/scripts/setup_layer_6_infra_monitoring.sh @@ -0,0 +1,473 @@ +#!/bin/bash + +############################################################################### +# Layer 6: Infrastructure Monitoring +# +# This script deploys disk usage, healthcheck, and CPU temp monitoring. +# Must be run after Layer 4 (Uptime Kuma) is complete with credentials set. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_prerequisites() { + print_header "Verifying Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + # Check Python uptime-kuma-api + if ! python3 -c "import uptime_kuma_api" 2>/dev/null; then + print_error "uptime-kuma-api Python package not found" + print_info "Install with: pip install -r requirements.txt" + ((errors++)) + else + print_success "uptime-kuma-api package found" + fi + + if [ $errors -gt 0 ]; then + print_error "Prerequisites not met" + exit 1 + fi + + print_success "Prerequisites verified" +} + +check_uptime_kuma_credentials() { + print_header "Verifying Uptime Kuma Configuration" + + cd "$ANSIBLE_DIR" + + # Check if infra_secrets.yml has credentials + if ! grep -q "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null || \ + ! grep -q "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" 2>/dev/null; then + print_error "Uptime Kuma credentials not found in infra_secrets.yml" + print_info "You must complete Layer 4 post-deployment steps first:" + echo " 1. Create admin user in Uptime Kuma web UI" + echo " 2. Add credentials to ansible/infra_secrets.yml" + exit 1 + fi + + local uk_user=$(grep "^uptime_kuma_username:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + local uk_pass=$(grep "^uptime_kuma_password:" "$ANSIBLE_DIR/infra_secrets.yml" | awk '{print $2}' | tr -d '"' | tr -d "'") + + if [ -z "$uk_user" ] || [ -z "$uk_pass" ]; then + print_error "Uptime Kuma credentials are empty in infra_secrets.yml" + exit 1 + fi + + print_success "Uptime Kuma credentials found" + + # Test API connection + print_info "Testing Uptime Kuma API connection..." + + local test_script=$(mktemp) + cat > "$test_script" << 'EOFPYTHON' +import sys +import yaml +from uptime_kuma_api import UptimeKumaApi + +try: + with open('infra_vars.yml', 'r') as f: + infra_vars = yaml.safe_load(f) + + with open('services_config.yml', 'r') as f: + services_config = yaml.safe_load(f) + + with open('infra_secrets.yml', 'r') as f: + secrets = yaml.safe_load(f) + + root_domain = infra_vars.get('root_domain') + subdomain = services_config.get('subdomains', {}).get('uptime_kuma', 'uptime') + url = f"https://{subdomain}.{root_domain}" + + username = secrets.get('uptime_kuma_username') + password = secrets.get('uptime_kuma_password') + + api = UptimeKumaApi(url) + api.login(username, password) + + monitors = api.get_monitors() + print(f"SUCCESS:{len(monitors)}") + api.disconnect() + +except Exception as e: + print(f"ERROR:{str(e)}", file=sys.stderr) + sys.exit(1) +EOFPYTHON + + local result=$(cd "$ANSIBLE_DIR" && python3 "$test_script" 2>&1) + rm -f "$test_script" + + if echo "$result" | grep -q "^SUCCESS:"; then + local monitor_count=$(echo "$result" | grep "^SUCCESS:" | cut -d: -f2) + print_success "Successfully connected to Uptime Kuma API" + print_info "Current monitors: $monitor_count" + else + print_error "Cannot connect to Uptime Kuma API" + print_info "Error: $result" + echo "" + print_info "Make sure:" + echo " • Uptime Kuma is running (Layer 4)" + echo " • Credentials are correct in infra_secrets.yml" + echo " • Uptime Kuma is accessible" + exit 1 + fi + + echo "" + print_success "Uptime Kuma configuration verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +############################################################################### +# Disk Usage Monitoring +############################################################################### + +deploy_disk_usage_monitoring() { + print_header "Deploying Disk Usage Monitoring" + + cd "$ANSIBLE_DIR" + + print_info "This will deploy disk usage monitoring on selected hosts" + print_info "Default settings:" + echo " • Threshold: 80%" + echo " • Check interval: 15 minutes" + echo " • Mount point: /" + echo "" + + # Show available hosts + echo "Available hosts:" + for group in vipy watchtower spacey nodito lapy; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + fi + done + echo "" + + print_info "Deployment options:" + echo " 1. Deploy on all remote hosts (vipy, watchtower, spacey, nodito)" + echo " 2. Deploy on all hosts (including lapy)" + echo " 3. Custom selection (specify groups)" + echo " 4. Skip disk monitoring" + echo "" + + echo -e -n "${BLUE}Choose option${NC} [1-4]: " + read option + + local limit_hosts="" + case "$option" in + 1) + limit_hosts="vipy,watchtower,spacey,nodito" + print_info "Deploying to remote hosts" + ;; + 2) + limit_hosts="all" + print_info "Deploying to all hosts" + ;; + 3) + echo -e -n "${BLUE}Enter groups (comma-separated)${NC}: " + read limit_hosts + print_info "Deploying to: $limit_hosts" + ;; + 4) + print_warning "Skipping disk usage monitoring" + return 0 + ;; + *) + print_error "Invalid option" + return 0 + ;; + esac + + echo "" + if ! confirm_action "Proceed with disk usage monitoring deployment?"; then + print_warning "Skipped" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml --limit $limit_hosts" + echo "" + + if ansible-playbook -i inventory.ini infra/410_disk_usage_alerts.yml --limit "$limit_hosts"; then + print_success "Disk usage monitoring deployed" + return 0 + else + print_error "Deployment failed" + return 0 + fi +} + +############################################################################### +# System Healthcheck Monitoring +############################################################################### + +deploy_system_healthcheck() { + print_header "Deploying System Healthcheck Monitoring" + + cd "$ANSIBLE_DIR" + + print_info "This will deploy system healthcheck monitoring on selected hosts" + print_info "Default settings:" + echo " • Heartbeat interval: 60 seconds" + echo " • Upside-down mode (no news is good news)" + echo "" + + # Show available hosts + echo "Available hosts:" + for group in vipy watchtower spacey nodito lapy; do + local hosts=$(get_hosts_from_inventory "$group") + if [ -n "$hosts" ]; then + echo " [$group]: $hosts" + fi + done + echo "" + + print_info "Deployment options:" + echo " 1. Deploy on all remote hosts (vipy, watchtower, spacey, nodito)" + echo " 2. Deploy on all hosts (including lapy)" + echo " 3. Custom selection (specify groups)" + echo " 4. Skip healthcheck monitoring" + echo "" + + echo -e -n "${BLUE}Choose option${NC} [1-4]: " + read option + + local limit_hosts="" + case "$option" in + 1) + limit_hosts="vipy,watchtower,spacey,nodito" + print_info "Deploying to remote hosts" + ;; + 2) + limit_hosts="all" + print_info "Deploying to all hosts" + ;; + 3) + echo -e -n "${BLUE}Enter groups (comma-separated)${NC}: " + read limit_hosts + print_info "Deploying to: $limit_hosts" + ;; + 4) + print_warning "Skipping healthcheck monitoring" + return 0 + ;; + *) + print_error "Invalid option" + return 0 + ;; + esac + + echo "" + if ! confirm_action "Proceed with healthcheck monitoring deployment?"; then + print_warning "Skipped" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml --limit $limit_hosts" + echo "" + + if ansible-playbook -i inventory.ini infra/420_system_healthcheck.yml --limit "$limit_hosts"; then + print_success "System healthcheck monitoring deployed" + return 0 + else + print_error "Deployment failed" + return 0 + fi +} + +############################################################################### +# CPU Temperature Monitoring (Nodito) +############################################################################### + +deploy_cpu_temp_monitoring() { + print_header "Deploying CPU Temperature Monitoring (Nodito)" + + cd "$ANSIBLE_DIR" + + # Check if nodito is configured + local nodito_hosts=$(get_hosts_from_inventory "nodito") + if [ -z "$nodito_hosts" ]; then + print_info "Nodito not configured in inventory, skipping CPU temp monitoring" + return 0 + fi + + print_info "This will deploy CPU temperature monitoring on nodito (Proxmox)" + print_info "Default settings:" + echo " • Threshold: 80°C" + echo " • Check interval: 60 seconds" + echo "" + + echo "" + if ! confirm_action "Proceed with CPU temp monitoring deployment?"; then + print_warning "Skipped" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml" + echo "" + + if ansible-playbook -i inventory.ini infra/430_cpu_temp_alerts.yml; then + print_success "CPU temperature monitoring deployed" + return 0 + else + print_error "Deployment failed" + return 0 + fi +} + +############################################################################### +# Summary +############################################################################### + +print_summary() { + print_header "Layer 6 Setup Complete! 🎉" + + echo "Summary of what was deployed:" + echo "" + print_success "Infrastructure monitoring configured" + print_success "Monitors created in Uptime Kuma" + print_success "Systemd services and timers running" + echo "" + + print_info "What you have now:" + echo " • Disk usage monitoring on selected hosts" + echo " • System healthcheck monitoring" + echo " • CPU temperature monitoring (if nodito configured)" + echo " • All organized in host-specific groups" + echo "" + + print_info "Verify your monitoring:" + echo " 1. Open Uptime Kuma web UI" + echo " 2. Check monitors organized by host groups" + echo " 3. Verify monitors are receiving data" + echo " 4. Configure notification rules" + echo " 5. Watch for alerts via ntfy" + echo "" + + print_info "Next steps:" + echo " 1. Customize thresholds if needed" + echo " 2. Proceed to Layer 7: Core Services deployment" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "📊 Layer 6: Infrastructure Monitoring" + + echo "This script will deploy automated monitoring for your infrastructure." + echo "" + + if ! confirm_action "Continue with Layer 6 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_prerequisites + check_uptime_kuma_credentials + + # Deploy monitoring + deploy_disk_usage_monitoring + echo "" + deploy_system_healthcheck + echo "" + deploy_cpu_temp_monitoring + + echo "" + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_7_services.sh b/scripts/setup_layer_7_services.sh new file mode 100755 index 0000000..27c3c8d --- /dev/null +++ b/scripts/setup_layer_7_services.sh @@ -0,0 +1,524 @@ +#!/bin/bash + +############################################################################### +# Layer 7: Core Services +# +# This script deploys Vaultwarden, Forgejo, and LNBits on vipy. +# Must be run after Layers 0, 1A, 2, and 3 are complete. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project root directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +############################################################################### +# Helper Functions +############################################################################### + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +############################################################################### +# Verification Functions +############################################################################### + +check_prerequisites() { + print_header "Verifying Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + # Check if vipy is configured + if [ -z "$(get_hosts_from_inventory "vipy")" ]; then + print_error "vipy not configured in inventory.ini" + print_info "Layer 7 requires vipy VPS" + ((errors++)) + else + print_success "vipy configured in inventory" + fi + + if [ $errors -gt 0 ]; then + print_error "Prerequisites not met" + exit 1 + fi + + print_success "Prerequisites verified" +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +get_host_ip() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +hostvars = data.get('_meta', {}).get('hostvars', {}) +if target in hostvars: + print(hostvars[target].get('ansible_host', target)) +else: + hosts = data.get(target, {}).get('hosts', []) + if hosts: + first = hosts[0] + hv = hostvars.get(first, {}) + print(hv.get('ansible_host', first)) +PY +} + +check_dns_configuration() { + print_header "Validating DNS Configuration" + + cd "$ANSIBLE_DIR" + + # Get vipy IP + local vipy_ip=$(get_host_ip "vipy") + + if [ -z "$vipy_ip" ]; then + print_error "Could not determine vipy IP from inventory" + return 1 + fi + + print_info "Vipy IP: $vipy_ip" + echo "" + + # Get domain from infra_vars.yml + local root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) + + if [ -z "$root_domain" ]; then + print_error "Could not determine root_domain from infra_vars.yml" + return 1 + fi + + # Get subdomains from centralized config + local vw_subdomain="vault" + local fg_subdomain="git" + local ln_subdomain="lnbits" + + if [ -f "$ANSIBLE_DIR/services_config.yml" ]; then + vw_subdomain=$(grep "^ vaultwarden:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "vault") + fg_subdomain=$(grep "^ forgejo:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "git") + ln_subdomain=$(grep "^ lnbits:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "lnbits") + fi + + print_info "Checking DNS records..." + echo "" + + local dns_ok=true + + if command -v dig &> /dev/null; then + # Check each subdomain + for service in "vaultwarden:$vw_subdomain" "forgejo:$fg_subdomain" "lnbits:$ln_subdomain"; do + local name=$(echo "$service" | cut -d: -f1) + local subdomain=$(echo "$service" | cut -d: -f2) + local fqdn="${subdomain}.${root_domain}" + + print_info "Checking $fqdn..." + local resolved=$(dig +short "$fqdn" | head -n1) + + if [ "$resolved" = "$vipy_ip" ]; then + print_success "$fqdn → $resolved ✓" + elif [ -n "$resolved" ]; then + print_error "$fqdn → $resolved (expected $vipy_ip)" + dns_ok=false + else + print_error "$fqdn does not resolve" + dns_ok=false + fi + done + else + print_warning "dig command not found, skipping DNS validation" + print_info "Install dnsutils/bind-tools to enable DNS validation" + return 1 + fi + + echo "" + + if [ "$dns_ok" = false ]; then + print_error "DNS validation failed" + print_info "Please configure DNS records for all services" + echo "" + print_warning "DNS changes can take time to propagate" + echo "" + if ! confirm_action "Continue anyway? (SSL certificates will fail without proper DNS)"; then + exit 1 + fi + else + print_success "DNS validation passed" + fi +} + +############################################################################### +# Service Deployment +############################################################################### + +deploy_vaultwarden() { + print_header "Deploying Vaultwarden (Password Manager)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Deploy Vaultwarden via Docker" + echo " • Configure Caddy reverse proxy" + echo " • Set up fail2ban protection" + echo " • Enable sign-ups (disable after first user)" + echo "" + + if ! confirm_action "Proceed with Vaultwarden deployment?"; then + print_warning "Skipped Vaultwarden deployment" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/vaultwarden/deploy_vaultwarden_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/vaultwarden/deploy_vaultwarden_playbook.yml; then + print_success "Vaultwarden deployed" + echo "" + print_warning "POST-DEPLOYMENT:" + echo " 1. Visit your Vaultwarden subdomain" + echo " 2. Create your first user account" + echo " 3. Run: ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml" + return 0 + else + print_error "Vaultwarden deployment failed" + return 0 + fi +} + +deploy_forgejo() { + print_header "Deploying Forgejo (Git Server)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Install Forgejo binary" + echo " • Create git user and directories" + echo " • Configure Caddy reverse proxy" + echo " • Enable SSH cloning on port 22" + echo "" + + if ! confirm_action "Proceed with Forgejo deployment?"; then + print_warning "Skipped Forgejo deployment" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/forgejo/deploy_forgejo_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/forgejo/deploy_forgejo_playbook.yml; then + print_success "Forgejo deployed" + echo "" + print_warning "POST-DEPLOYMENT:" + echo " 1. Visit your Forgejo subdomain" + echo " 2. Create admin account on first visit" + echo " 3. Add your SSH key for git cloning" + return 0 + else + print_error "Forgejo deployment failed" + return 0 + fi +} + +deploy_lnbits() { + print_header "Deploying LNBits (Lightning Wallet)" + + cd "$ANSIBLE_DIR" + + print_info "This will:" + echo " • Install system dependencies and uv (Python 3.12 tooling)" + echo " • Clone LNBits repository (version v1.3.1)" + echo " • Sync dependencies with uv targeting Python 3.12" + echo " • Configure with FakeWallet (testing)" + echo " • Create systemd service" + echo " • Configure Caddy reverse proxy" + echo "" + + if ! confirm_action "Proceed with LNBits deployment?"; then + print_warning "Skipped LNBits deployment" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/lnbits/deploy_lnbits_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/lnbits/deploy_lnbits_playbook.yml; then + print_success "LNBits deployed" + echo "" + print_warning "POST-DEPLOYMENT:" + echo " 1. Visit your LNBits subdomain" + echo " 2. Create superuser on first visit" + echo " 3. Configure real Lightning backend (FakeWallet is for testing only)" + echo " 4. Disable new user registration" + return 0 + else + print_error "LNBits deployment failed" + return 0 + fi +} + +############################################################################### +# Backup Configuration +############################################################################### + +setup_backups() { + print_header "Setting Up Backups (Optional)" + + cd "$ANSIBLE_DIR" + + print_info "Configure automated backups to lapy" + echo "" + + # Vaultwarden backup + if confirm_action "Set up Vaultwarden backup to lapy?"; then + print_info "Running: ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml" + if ansible-playbook -i inventory.ini services/vaultwarden/setup_backup_vaultwarden_to_lapy.yml; then + print_success "Vaultwarden backup configured" + else + print_error "Vaultwarden backup setup failed" + fi + echo "" + fi + + # LNBits backup + if confirm_action "Set up LNBits backup to lapy (GPG encrypted)?"; then + print_info "Running: ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml" + if ansible-playbook -i inventory.ini services/lnbits/setup_backup_lnbits_to_lapy.yml; then + print_success "LNBits backup configured" + else + print_error "LNBits backup setup failed" + fi + echo "" + fi + + print_warning "Forgejo backups are not automated - set up manually if needed" +} + +############################################################################### +# Verification +############################################################################### + +verify_services() { + print_header "Verifying Service Deployments" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local vipy_host=$(get_hosts_from_inventory "vipy") + + if [ -z "$vipy_host" ]; then + print_error "Could not determine vipy host" + return + fi + + print_info "Checking services on vipy ($vipy_host)..." + echo "" + + # Check Vaultwarden + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "docker ps | grep vaultwarden" &>/dev/null; then + print_success "Vaultwarden container running" + else + print_warning "Vaultwarden container not running (may not be deployed)" + fi + + # Check Forgejo + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "systemctl is-active forgejo" &>/dev/null; then + print_success "Forgejo service running" + else + print_warning "Forgejo service not running (may not be deployed)" + fi + + # Check LNBits + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "systemctl is-active lnbits" &>/dev/null; then + print_success "LNBits service running" + else + print_warning "LNBits service not running (may not be deployed)" + fi + + # Check Caddy configs + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "ls /etc/caddy/sites-enabled/*.conf 2>/dev/null" &>/dev/null; then + print_success "Caddy configs exist" + local configs=$(timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "ls /etc/caddy/sites-enabled/*.conf 2>/dev/null" | xargs -n1 basename) + print_info "Configured services:" + echo "$configs" | sed 's/^/ /' + else + print_warning "No Caddy configs found" + fi + + echo "" +} + +############################################################################### +# Summary +############################################################################### + +print_summary() { + print_header "Layer 7 Setup Complete! 🎉" + + echo "Summary of what was deployed:" + echo "" + print_success "Core services deployed on vipy" + echo "" + + print_warning "CRITICAL POST-DEPLOYMENT STEPS:" + echo "" + echo "For each service you deployed, you MUST:" + echo "" + + echo "1. Vaultwarden (if deployed):" + echo " • Visit web UI and create first user" + echo " • Disable sign-ups: ansible-playbook -i inventory.ini services/vaultwarden/disable_vaultwarden_sign_ups_playbook.yml" + echo " • Optional: Set up backup" + echo "" + + echo "2. Forgejo (if deployed):" + echo " • Visit web UI and create admin account" + echo " • Add your SSH public key for git operations" + echo " • Test cloning: git clone git@.:username/repo.git" + echo "" + + echo "3. LNBits (if deployed):" + echo " • Visit web UI and create superuser" + echo " • Configure real Lightning backend (currently FakeWallet)" + echo " • Disable new user registration" + echo " • Optional: Set up encrypted backup" + echo "" + + print_info "Services are now accessible:" + echo " • Vaultwarden: https://." + echo " • Forgejo: https://." + echo " • LNBits: https://." + echo "" + + print_success "Uptime Kuma monitors automatically created:" + echo " • Check Uptime Kuma web UI" + echo " • Look in 'services' monitor group" + echo " • Monitors for Vaultwarden, Forgejo, LNBits should appear" + echo "" + + print_info "Next steps:" + echo " 1. Complete post-deployment steps above" + echo " 2. Test each service" + echo " 3. Check Uptime Kuma monitors are working" + echo " 4. Proceed to Layer 8: ./scripts/setup_layer_8_secondary_services.sh" + echo "" +} + +############################################################################### +# Main Execution +############################################################################### + +main() { + clear + + print_header "🚀 Layer 7: Core Services" + + echo "This script will deploy core services on vipy:" + echo " • Vaultwarden (password manager)" + echo " • Forgejo (git server)" + echo " • LNBits (Lightning wallet)" + echo "" + + if ! confirm_action "Continue with Layer 7 setup?"; then + echo "Setup cancelled." + exit 0 + fi + + check_prerequisites + check_dns_configuration + + # Deploy services + deploy_vaultwarden + echo "" + deploy_forgejo + echo "" + deploy_lnbits + + echo "" + verify_services + + echo "" + setup_backups + + print_summary +} + +# Run main function +main "$@" + diff --git a/scripts/setup_layer_8_secondary_services.sh b/scripts/setup_layer_8_secondary_services.sh new file mode 100755 index 0000000..fccaad8 --- /dev/null +++ b/scripts/setup_layer_8_secondary_services.sh @@ -0,0 +1,384 @@ +#!/bin/bash + +############################################################################### +# Layer 8: Secondary Services +# +# This script deploys the ntfy-emergency-app and memos services. +# Must be run after Layers 0-7 are complete. +############################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Project directories +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ANSIBLE_DIR="$PROJECT_ROOT/ansible" + +declare -a LAYER_SUMMARY=() + +print_header() { + echo -e "\n${BLUE}========================================${NC}" + echo -e "${BLUE}$1${NC}" + echo -e "${BLUE}========================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +confirm_action() { + local prompt="$1" + local response + + read -p "$(echo -e ${YELLOW}${prompt}${NC} [y/N]: )" response + [[ "$response" =~ ^[Yy]$ ]] +} + +record_summary() { + LAYER_SUMMARY+=("$1") +} + +get_hosts_from_inventory() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +if target in data: + print(' '.join(data[target].get('hosts', []))) +else: + hostvars = data.get('_meta', {}).get('hostvars', {}) + if target in hostvars: + print(target) +PY +} + +get_primary_host_ip() { + local target="$1" + cd "$ANSIBLE_DIR" + ansible-inventory -i inventory.ini --list | \ + python3 - "$target" <<'PY' 2>/dev/null || echo "" +import json, sys +data = json.load(sys.stdin) +target = sys.argv[1] +hostvars = data.get('_meta', {}).get('hostvars', {}) +if target in hostvars: + print(hostvars[target].get('ansible_host', target)) +else: + hosts = data.get(target, {}).get('hosts', []) + if hosts: + first = hosts[0] + hv = hostvars.get(first, {}) + print(hv.get('ansible_host', first)) +PY +} + +check_prerequisites() { + print_header "Verifying Prerequisites" + + local errors=0 + + if [ -z "$VIRTUAL_ENV" ]; then + print_error "Virtual environment not activated" + echo "Run: source venv/bin/activate" + ((errors++)) + else + print_success "Virtual environment activated" + fi + + if ! command -v ansible &> /dev/null; then + print_error "Ansible not found" + ((errors++)) + else + print_success "Ansible found" + fi + + if [ ! -f "$ANSIBLE_DIR/inventory.ini" ]; then + print_error "inventory.ini not found" + ((errors++)) + else + print_success "inventory.ini exists" + fi + + if [ ! -f "$ANSIBLE_DIR/infra_vars.yml" ]; then + print_error "infra_vars.yml not found" + ((errors++)) + else + print_success "infra_vars.yml exists" + fi + + if [ ! -f "$ANSIBLE_DIR/services_config.yml" ]; then + print_error "services_config.yml not found" + ((errors++)) + else + print_success "services_config.yml exists" + fi + + if [ -z "$(get_hosts_from_inventory "vipy")" ]; then + print_error "vipy not configured in inventory.ini" + ((errors++)) + else + print_success "vipy configured in inventory" + fi + + if [ -z "$(get_hosts_from_inventory "memos-box")" ]; then + print_warning "memos-box not configured in inventory.ini (memos deployment will be skipped)" + else + print_success "memos-box configured in inventory" + fi + + if [ $errors -gt 0 ]; then + print_error "Prerequisites not met. Resolve the issues above and re-run the script." + exit 1 + fi + + print_success "Prerequisites verified" + + # Display configured subdomains + local emergency_subdomain=$(grep "^ ntfy_emergency_app:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "emergency") + local memos_subdomain=$(grep "^ memos:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "memos") + + print_info "Configured subdomains:" + echo " • ntfy_emergency_app: $emergency_subdomain" + echo " • memos: $memos_subdomain" + echo "" +} + +check_dns_configuration() { + print_header "Validating DNS Configuration" + + if ! command -v dig &> /dev/null; then + print_warning "dig command not found. Skipping DNS validation." + print_info "Install dnsutils/bind-tools to enable DNS validation." + return 0 + fi + + cd "$ANSIBLE_DIR" + + local root_domain + root_domain=$(grep "^root_domain:" "$ANSIBLE_DIR/infra_vars.yml" | awk '{print $2}' 2>/dev/null) + + if [ -z "$root_domain" ]; then + print_error "Could not determine root_domain from infra_vars.yml" + return 1 + fi + + local emergency_subdomain=$(grep "^ ntfy_emergency_app:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "emergency") + local memos_subdomain=$(grep "^ memos:" "$ANSIBLE_DIR/services_config.yml" | awk '{print $2}' 2>/dev/null || echo "memos") + + local vipy_ip + vipy_ip=$(get_primary_host_ip "vipy") + + if [ -z "$vipy_ip" ]; then + print_error "Unable to determine vipy IP from inventory" + return 1 + fi + + local memos_ip="" + local memos_host=$(get_hosts_from_inventory "memos-box") + if [ -n "$memos_host" ]; then + memos_ip=$(get_primary_host_ip "$memos_host") + fi + + local dns_ok=true + + local emergency_fqdn="${emergency_subdomain}.${root_domain}" + local memos_fqdn="${memos_subdomain}.${root_domain}" + + print_info "Expected DNS:" + echo " • $emergency_fqdn → $vipy_ip" + if [ -n "$memos_ip" ]; then + echo " • $memos_fqdn → $memos_ip" + else + echo " • $memos_fqdn → (skipped - memos-box not in inventory)" + fi + echo "" + + local resolved + + print_info "Checking $emergency_fqdn..." + resolved=$(dig +short "$emergency_fqdn" | head -n1) + if [ "$resolved" = "$vipy_ip" ]; then + print_success "$emergency_fqdn resolves to $resolved" + elif [ -n "$resolved" ]; then + print_error "$emergency_fqdn resolves to $resolved (expected $vipy_ip)" + dns_ok=false + else + print_error "$emergency_fqdn does not resolve" + dns_ok=false + fi + + if [ -n "$memos_ip" ]; then + print_info "Checking $memos_fqdn..." + resolved=$(dig +short "$memos_fqdn" | head -n1) + if [ "$resolved" = "$memos_ip" ]; then + print_success "$memos_fqdn resolves to $resolved" + elif [ -n "$resolved" ]; then + print_error "$memos_fqdn resolves to $resolved (expected $memos_ip)" + dns_ok=false + else + print_error "$memos_fqdn does not resolve" + dns_ok=false + fi + fi + + echo "" + + if [ "$dns_ok" = false ]; then + print_error "DNS validation failed." + print_info "Update DNS records as shown above and wait for propagation." + echo "" + if ! confirm_action "Continue anyway? (SSL certificates will fail without correct DNS)"; then + exit 1 + fi + else + print_success "DNS validation passed" + fi +} + +deploy_ntfy_emergency_app() { + print_header "Deploying ntfy-emergency-app" + + cd "$ANSIBLE_DIR" + + print_info "This deploys the emergency notification interface pointing at ntfy." + echo "" + + if ! confirm_action "Deploy / update the ntfy-emergency-app?"; then + print_warning "Skipped ntfy-emergency-app deployment" + record_summary "${YELLOW}• ntfy-emergency-app${NC}: skipped" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/ntfy-emergency-app/deploy_ntfy_emergency_app_playbook.yml; then + print_success "ntfy-emergency-app deployed successfully" + record_summary "${GREEN}• ntfy-emergency-app${NC}: deployed" + else + print_error "ntfy-emergency-app deployment failed" + record_summary "${RED}• ntfy-emergency-app${NC}: failed" + fi +} + +deploy_memos() { + print_header "Deploying Memos" + + if [ -z "$(get_hosts_from_inventory "memos-box")" ]; then + print_warning "memos-box not in inventory. Skipping memos deployment." + record_summary "${YELLOW}• memos${NC}: skipped (memos-box missing)" + return 0 + fi + + cd "$ANSIBLE_DIR" + + if ! confirm_action "Deploy / update memos on memos-box?"; then + print_warning "Skipped memos deployment" + record_summary "${YELLOW}• memos${NC}: skipped" + return 0 + fi + + print_info "Running: ansible-playbook -i inventory.ini services/memos/deploy_memos_playbook.yml" + echo "" + + if ansible-playbook -i inventory.ini services/memos/deploy_memos_playbook.yml; then + print_success "Memos deployed successfully" + record_summary "${GREEN}• memos${NC}: deployed" + else + print_error "Memos deployment failed" + record_summary "${RED}• memos${NC}: failed" + fi +} + +verify_services() { + print_header "Verifying Deployments" + + cd "$ANSIBLE_DIR" + + local ssh_key=$(grep "ansible_ssh_private_key_file" "$ANSIBLE_DIR/inventory.ini" | head -n1 | sed 's/.*ansible_ssh_private_key_file=\([^ ]*\).*/\1/') + ssh_key="${ssh_key/#\~/$HOME}" + + local vipy_host + vipy_host=$(get_hosts_from_inventory "vipy") + + if [ -n "$vipy_host" ]; then + print_info "Checking services on vipy ($vipy_host)..." + + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$vipy_host "docker ps | grep ntfy-emergency-app" &>/dev/null; then + print_success "ntfy-emergency-app container running" + else + print_warning "ntfy-emergency-app container not running" + fi + + echo "" + fi + + local memos_host + memos_host=$(get_hosts_from_inventory "memos-box") + if [ -n "$memos_host" ]; then + print_info "Checking memos on memos-box ($memos_host)..." + if timeout 5 ssh -i "$ssh_key" -o StrictHostKeyChecking=no -o BatchMode=yes counterweight@$memos_host "systemctl is-active memos" &>/dev/null; then + print_success "memos systemd service running" + else + print_warning "memos systemd service not running" + fi + echo "" + fi +} + +print_summary() { + print_header "Layer 8 Summary" + + if [ ${#LAYER_SUMMARY[@]} -eq 0 ]; then + print_info "No actions were performed." + return + fi + + for entry in "${LAYER_SUMMARY[@]}"; do + echo -e "$entry" + done + + echo "" + print_info "Next steps:" + echo " • Visit each service's subdomain to complete any manual setup." + echo " • Configure backups for new services if applicable." + echo " • Update Uptime Kuma monitors if additional endpoints are desired." +} + +main() { + print_header "Layer 8: Secondary Services" + + check_prerequisites + check_dns_configuration + + deploy_ntfy_emergency_app + deploy_memos + + verify_services + print_summary +} + +main "$@" + diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md index 3a0b18f..bb852da 100644 --- a/tofu/nodito/README.md +++ b/tofu/nodito/README.md @@ -45,6 +45,13 @@ vms = { memory_mb = 2048 disk_size_gb = 20 ipconfig0 = "ip=dhcp" # or "ip=192.168.1.50/24,gw=192.168.1.1" + data_disks = [ + { + size_gb = 50 + # storage defaults to var.zfs_storage_name (proxmox-tank-1) + # optional: slot = "scsi2" + } + ] } } ``` diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf index 9123175..4e10a12 100644 --- a/tofu/nodito/main.tf +++ b/tofu/nodito/main.tf @@ -30,7 +30,16 @@ resource "proxmox_vm_qemu" "vm" { lifecycle { prevent_destroy = true - ignore_changes = all + ignore_changes = [ + name, + cpu, + memory, + network, + ipconfig0, + ciuser, + sshkeys, + cicustom, + ] } serial { @@ -64,6 +73,16 @@ resource "proxmox_vm_qemu" "vm" { # optional flags like iothread/ssd/discard differ by provider versions; keep minimal } + dynamic "disk" { + for_each = try(each.value.data_disks, []) + content { + slot = try(disk.value.slot, format("scsi%s", tonumber(disk.key) + 1)) + type = "disk" + storage = try(disk.value.storage, var.zfs_storage_name) + size = "${disk.value.size_gb}G" + } + } + # Cloud-init CD-ROM so ipconfig0/sshkeys apply disk { slot = "ide2" diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example index cc88b3f..c957f35 100644 --- a/tofu/nodito/terraform.tfvars.example +++ b/tofu/nodito/terraform.tfvars.example @@ -20,6 +20,11 @@ vms = { memory_mb = 2048 disk_size_gb = 20 ipconfig0 = "ip=dhcp" + data_disks = [ + { + size_gb = 50 + } + ] } db1 = { diff --git a/tofu/nodito/variables.tf b/tofu/nodito/variables.tf index 30a1418..3f16e75 100644 --- a/tofu/nodito/variables.tf +++ b/tofu/nodito/variables.tf @@ -55,6 +55,11 @@ variable "vms" { disk_size_gb = number vlan_tag = optional(number) ipconfig0 = optional(string) # e.g. "ip=dhcp" or "ip=192.168.1.50/24,gw=192.168.1.1" + data_disks = optional(list(object({ + size_gb = number + storage = optional(string) + slot = optional(string) + })), []) })) default = {} }