From fe321050c1431c51dbcf3acc2231e69092c98eec Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 4 Jan 2026 23:19:19 +0100 Subject: [PATCH 1/2] monitor zfs --- .../nodito/32_zfs_pool_setup_playbook.yml | 496 ++++++++++++++++++ 1 file changed, 496 insertions(+) diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 4ff0ed4..cb72328 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -170,3 +170,499 @@ fail: msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" + +- name: Setup ZFS Pool Health Monitoring and Monthly Scrubs + hosts: nodito + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + + vars: + zfs_check_interval_seconds: 86400 # 24 hours + zfs_check_timeout_seconds: 90000 # ~25 hours (interval + buffer) + zfs_check_retries: 1 + zfs_monitoring_script_dir: /opt/zfs-monitoring + zfs_monitoring_script_path: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.sh" + zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" + zfs_systemd_health_service_name: zfs-health-monitor + zfs_systemd_scrub_service_name: zfs-monthly-scrub + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "zfs-health-{{ host_name.stdout }}" + monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma ZFS health monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_zfs_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + # Get all monitors + monitors = api.get_monitors() + + # Get all notifications and find ntfy notification + notifications = api.get_notifications() + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + # Find or create group + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + group_result = api.add_monitor(type='group', name=group_name) + # Refresh to get the full group object with id + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + # Find or create/update push monitor + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + monitor_result = api.add_monitor(**monitor_data) + # Refresh to get the full monitor object with pushToken + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + # Output result as JSON + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma ZFS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_zfs_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" + "{{ zfs_check_timeout_seconds }}" + "{{ zfs_check_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL and monitor ID as facts + set_fact: + uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" + + - name: Install required packages for ZFS monitoring + package: + name: + - curl + - jq + state: present + + - name: Create monitoring script directory + file: + path: "{{ zfs_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create ZFS health monitoring script + copy: + dest: "{{ zfs_monitoring_script_path }}" + content: | + #!/bin/bash + + # ZFS Pool Health Monitoring Script + # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy + # If any issues detected, does NOT send heartbeat (triggers timeout alert) + + LOG_FILE="{{ zfs_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" + POOL_NAME="{{ zfs_pool_name }}" + HOSTNAME=$(hostname) + + # Function to log messages + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + # Function to check pool health using JSON output + check_pool_health() { + local pool="$1" + local issues_found=0 + + # Get pool status as JSON + local pool_json + pool_json=$(zpool status -j "$pool" 2>&1) + + if [ $? -ne 0 ]; then + log_message "ERROR: Failed to get pool status for $pool" + log_message " -> $pool_json" + return 1 + fi + + # Check 1: Pool state must be ONLINE + local pool_state + pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') + + if [ "$pool_state" != "ONLINE" ]; then + log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" + issues_found=1 + else + log_message "OK: Pool state is ONLINE" + fi + + # Check 2: Check all vdevs and devices for non-ONLINE states + local bad_states + bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.state? and .state != "ONLINE") | + "\(.name // "unknown"): \(.state)" + ' 2>/dev/null) + + if [ -n "$bad_states" ]; then + log_message "ISSUE: Found devices not in ONLINE state:" + echo "$bad_states" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: All devices are ONLINE" + fi + + # Check 3: Check for resilvering in progress + local scan_function scan_state + scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + + if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then + local resilver_progress + resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') + log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" + issues_found=1 + fi + + # Check 4: Check for read/write/checksum errors on all devices + # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison + local devices_with_errors + devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | + "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" + ' 2>/dev/null) + + if [ -n "$devices_with_errors" ]; then + log_message "ISSUE: Found devices with I/O errors:" + echo "$devices_with_errors" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: No read/write/checksum errors detected" + fi + + # Check 5: Check for scan errors (from last scrub/resilver) + local scan_errors + scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') + + if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then + log_message "ISSUE: Last scan reported $scan_errors errors" + issues_found=1 + else + log_message "OK: No scan errors" + fi + + return $issues_found + } + + # Function to get last scrub info for status message + get_scrub_info() { + local pool="$1" + local pool_json + pool_json=$(zpool status -j "$pool" 2>/dev/null) + + local scan_func scan_state scan_start + scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') + + if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then + echo "scrub in progress (started $scan_start)" + elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then + echo "last scrub: $scan_start" + else + echo "no scrub history" + fi + } + + # Function to send heartbeat to Uptime Kuma + send_heartbeat() { + local message="$1" + + log_message "Sending heartbeat to Uptime Kuma: $message" + + # URL encode the message + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent successfully (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + # Main health check logic + main() { + log_message "==========================================" + log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" + + # Run all health checks + if check_pool_health "$POOL_NAME"; then + # All checks passed - send heartbeat + local scrub_info + scrub_info=$(get_scrub_info "$POOL_NAME") + + local message="Pool $POOL_NAME healthy ($scrub_info)" + send_heartbeat "$message" + + log_message "Health check completed: ALL OK" + exit 0 + else + # Issues found - do NOT send heartbeat (will trigger timeout alert) + log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" + log_message "Uptime Kuma will alert after timeout due to missing heartbeat" + exit 1 + fi + } + + # Run main function + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" + content: | + [Unit] + Description=ZFS Pool Health Monitor + After=zfs.target network.target + + [Service] + Type=oneshot + ExecStart={{ zfs_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for daily ZFS health monitoring + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Pool Health Monitor daily + Requires={{ zfs_systemd_health_service_name }}.service + + [Timer] + OnBootSec=5min + OnUnitActiveSec={{ zfs_check_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Create systemd service for ZFS monthly scrub + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" + content: | + [Unit] + Description=ZFS Monthly Scrub for {{ zfs_pool_name }} + After=zfs.target + + [Service] + Type=oneshot + ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for monthly ZFS scrub + copy: + dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" + content: | + [Unit] + Description=Run ZFS Scrub on last day of every month at 4:00 AM + Requires={{ zfs_systemd_scrub_service_name }}.service + + [Timer] + OnCalendar=*-*~01 04:00:00 + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start ZFS health monitoring timer + systemd: + name: "{{ zfs_systemd_health_service_name }}.timer" + enabled: yes + state: started + + - name: Enable and start ZFS monthly scrub timer + systemd: + name: "{{ zfs_systemd_scrub_service_name }}.timer" + enabled: yes + state: started + + - name: Test ZFS health monitoring script + command: "{{ zfs_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "ZFS health monitoring script failed - check pool health" + + - name: Display monitoring configuration + debug: + msg: | + ✓ ZFS Pool Health Monitoring deployed successfully! + + Monitor Name: {{ monitor_friendly_name }} + Monitor Group: {{ uptime_kuma_monitor_group }} + Pool Name: {{ zfs_pool_name }} + + Health Check: + - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) + - Timeout: {{ zfs_check_timeout_seconds }} seconds (~25 hours) + - Script: {{ zfs_monitoring_script_path }} + - Log: {{ zfs_log_file }} + - Service: {{ zfs_systemd_health_service_name }}.service + - Timer: {{ zfs_systemd_health_service_name }}.timer + + Monthly Scrub: + - Schedule: Last day of month at 4:00 AM + - Service: {{ zfs_systemd_scrub_service_name }}.service + - Timer: {{ zfs_systemd_scrub_service_name }}.timer + + Conditions monitored: + - Pool state (must be ONLINE) + - Device states (no DEGRADED/FAULTED/OFFLINE/UNAVAIL) + - Resilver status (alerts if resilvering) + - Read/Write/Checksum errors + - Scrub errors + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_zfs_monitor.py + state: absent + delegate_to: localhost + become: no From 08281ce349ca6dcb46f6071d105ff06ae3e6ba86 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 11 Jan 2026 22:43:27 +0100 Subject: [PATCH 2/2] ups playbook --- .../nodito/34_nut_ups_setup_playbook.yml | 569 ++++++++++++++++++ ansible/infra/nodito/nodito_vars.yml | 9 + 2 files changed, 578 insertions(+) create mode 100644 ansible/infra/nodito/34_nut_ups_setup_playbook.yml diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml new file mode 100644 index 0000000..02468d5 --- /dev/null +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -0,0 +1,569 @@ +- name: Setup NUT (Network UPS Tools) for CyberPower UPS + hosts: nodito_host + become: true + vars_files: + - ../../infra_vars.yml + - nodito_vars.yml + - nodito_secrets.yml + + tasks: + # ------------------------------------------------------------------ + # Installation + # ------------------------------------------------------------------ + - name: Install NUT packages + apt: + name: + - nut + - nut-client + - nut-server + state: present + update_cache: true + + # ------------------------------------------------------------------ + # Verify UPS is detected + # ------------------------------------------------------------------ + - name: Check if UPS is detected via USB + shell: lsusb | grep -i cyber + register: lsusb_output + changed_when: false + failed_when: false + + - name: Display USB detection result + debug: + msg: "{{ lsusb_output.stdout | default('UPS not detected via USB - ensure it is plugged in') }}" + + - name: Fail if UPS not detected + fail: + msg: "CyberPower UPS not detected via USB. Ensure the USB cable is connected." + when: lsusb_output.rc != 0 + + - name: Reload udev rules for USB permissions + shell: | + udevadm control --reload-rules + udevadm trigger --subsystem-match=usb --action=add + changed_when: true + + - name: Verify USB device has nut group permissions + shell: | + BUS_DEV=$(lsusb | grep -i cyber | grep -oP 'Bus \K\d+|Device \K\d+' | tr '\n' '/' | sed 's/\/$//') + if [ -n "$BUS_DEV" ]; then + BUS=$(echo $BUS_DEV | cut -d'/' -f1) + DEV=$(echo $BUS_DEV | cut -d'/' -f2) + ls -la /dev/bus/usb/$BUS/$DEV + else + echo "UPS device not found" + exit 1 + fi + register: usb_permissions + changed_when: false + + - name: Display USB permissions + debug: + msg: "{{ usb_permissions.stdout }} (should show 'root nut', not 'root root')" + + - name: Scan for UPS with nut-scanner + command: nut-scanner -U + register: nut_scanner_output + changed_when: false + failed_when: false + + - name: Display nut-scanner result + debug: + msg: "{{ nut_scanner_output.stdout_lines }}" + + # ------------------------------------------------------------------ + # Configuration files + # ------------------------------------------------------------------ + - name: Configure NUT mode (standalone) + copy: + dest: /etc/nut/nut.conf + content: | + # Managed by Ansible + MODE=standalone + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure UPS device + copy: + dest: /etc/nut/ups.conf + content: | + # Managed by Ansible + [{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsd to listen on localhost + copy: + dest: /etc/nut/upsd.conf + content: | + # Managed by Ansible + LISTEN 127.0.0.1 3493 + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsd users + copy: + dest: /etc/nut/upsd.users + content: | + # Managed by Ansible + [{{ ups_user }}] + password = {{ ups_password }} + upsmon master + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + - name: Configure upsmon + copy: + dest: /etc/nut/upsmon.conf + content: | + # Managed by Ansible + MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + + MINSUPPLIES 1 + SHUTDOWNCMD "/sbin/shutdown -h +0" + POLLFREQ 5 + POLLFREQALERT 5 + HOSTSYNC 15 + DEADTIME 15 + POWERDOWNFLAG /etc/killpower + + # Notifications + NOTIFYMSG ONLINE "UPS %s on line power" + NOTIFYMSG ONBATT "UPS %s on battery" + NOTIFYMSG LOWBATT "UPS %s battery is low" + NOTIFYMSG FSD "UPS %s: forced shutdown in progress" + NOTIFYMSG COMMOK "Communications with UPS %s established" + NOTIFYMSG COMMBAD "Communications with UPS %s lost" + NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" + NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + + # Log all events to syslog + NOTIFYFLAG ONLINE SYSLOG + NOTIFYFLAG ONBATT SYSLOG + NOTIFYFLAG LOWBATT SYSLOG + NOTIFYFLAG FSD SYSLOG + NOTIFYFLAG COMMOK SYSLOG + NOTIFYFLAG COMMBAD SYSLOG + NOTIFYFLAG SHUTDOWN SYSLOG + NOTIFYFLAG REPLBATT SYSLOG + owner: root + group: nut + mode: "0640" + notify: Restart NUT services + + # ------------------------------------------------------------------ + # Verify late-stage shutdown script + # ------------------------------------------------------------------ + - name: Verify nutshutdown script exists + stat: + path: /lib/systemd/system-shutdown/nutshutdown + register: nutshutdown_script + + - name: Warn if nutshutdown script is missing + debug: + msg: "WARNING: /lib/systemd/system-shutdown/nutshutdown not found. UPS may not cut power after shutdown." + when: not nutshutdown_script.stat.exists + + # ------------------------------------------------------------------ + # Services + # ------------------------------------------------------------------ + - name: Enable and start NUT driver enumerator + systemd: + name: nut-driver-enumerator + enabled: true + state: started + + - name: Enable and start NUT server + systemd: + name: nut-server + enabled: true + state: started + + - name: Enable and start NUT monitor + systemd: + name: nut-monitor + enabled: true + state: started + + # ------------------------------------------------------------------ + # Verification + # ------------------------------------------------------------------ + - name: Wait for NUT services to stabilize + pause: + seconds: 3 + + - name: Verify NUT can communicate with UPS + command: upsc {{ ups_name }}@localhost + register: upsc_output + changed_when: false + failed_when: upsc_output.rc != 0 + + - name: Display UPS status + debug: + msg: "{{ upsc_output.stdout_lines }}" + + - name: Get UPS status summary + shell: | + echo "Status: $(upsc {{ ups_name }}@localhost ups.status 2>/dev/null)" + echo "Battery: $(upsc {{ ups_name }}@localhost battery.charge 2>/dev/null)%" + echo "Runtime: $(upsc {{ ups_name }}@localhost battery.runtime 2>/dev/null)s" + echo "Load: $(upsc {{ ups_name }}@localhost ups.load 2>/dev/null)%" + register: ups_summary + changed_when: false + + - name: Display UPS summary + debug: + msg: "{{ ups_summary.stdout_lines }}" + + - name: Verify low battery thresholds + shell: | + echo "Runtime threshold: $(upsc {{ ups_name }}@localhost battery.runtime.low 2>/dev/null)s" + echo "Charge threshold: $(upsc {{ ups_name }}@localhost battery.charge.low 2>/dev/null)%" + register: thresholds + changed_when: false + + - name: Display low battery thresholds + debug: + msg: "{{ thresholds.stdout_lines }}" + + handlers: + - name: Restart NUT services + systemd: + name: "{{ item }}" + state: restarted + loop: + - nut-driver-enumerator + - nut-server + - nut-monitor + + +- name: Setup UPS Heartbeat Monitoring with Uptime Kuma + hosts: nodito + become: true + vars_files: + - ../../infra_vars.yml + - ../../services_config.yml + - ../../infra_secrets.yml + - nodito_vars.yml + - nodito_secrets.yml + + vars: + ups_heartbeat_interval_seconds: 60 + ups_heartbeat_timeout_seconds: 120 + ups_heartbeat_retries: 1 + ups_monitoring_script_dir: /opt/ups-monitoring + ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" + ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" + ups_systemd_service_name: ups-heartbeat + uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" + ntfy_topic: "{{ service_settings.ntfy.topic }}" + + tasks: + - name: Validate Uptime Kuma configuration + assert: + that: + - uptime_kuma_api_url is defined + - uptime_kuma_api_url != "" + - uptime_kuma_username is defined + - uptime_kuma_username != "" + - uptime_kuma_password is defined + - uptime_kuma_password != "" + fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" + + - name: Get hostname for monitor identification + command: hostname + register: host_name + changed_when: false + + - name: Set monitor name and group based on hostname + set_fact: + monitor_name: "ups-{{ host_name.stdout }}" + monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" + uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" + + - name: Create Uptime Kuma UPS monitor setup script + copy: + dest: /tmp/setup_uptime_kuma_ups_monitor.py + content: | + #!/usr/bin/env python3 + import sys + import json + from uptime_kuma_api import UptimeKumaApi + + def main(): + api_url = sys.argv[1] + username = sys.argv[2] + password = sys.argv[3] + group_name = sys.argv[4] + monitor_name = sys.argv[5] + monitor_description = sys.argv[6] + interval = int(sys.argv[7]) + retries = int(sys.argv[8]) + ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" + + api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) + api.login(username, password) + + monitors = api.get_monitors() + notifications = api.get_notifications() + + ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) + notification_id_list = {} + if ntfy_notification: + notification_id_list[ntfy_notification['id']] = True + + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + if not group: + api.add_monitor(type='group', name=group_name) + monitors = api.get_monitors() + group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) + + existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + monitor_data = { + 'type': 'push', + 'name': monitor_name, + 'parent': group['id'], + 'interval': interval, + 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy + 'maxretries': retries, + 'description': monitor_description, + 'notificationIDList': notification_id_list + } + + if existing_monitor: + api.edit_monitor(existing_monitor['id'], **monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + else: + api.add_monitor(**monitor_data) + monitors = api.get_monitors() + monitor = next((m for m in monitors if m.get('name') == monitor_name), None) + + result = { + 'monitor_id': monitor['id'], + 'push_token': monitor['pushToken'], + 'group_name': group_name, + 'group_id': group['id'], + 'monitor_name': monitor_name + } + print(json.dumps(result)) + + api.disconnect() + + if __name__ == '__main__': + main() + mode: '0755' + delegate_to: localhost + become: no + + - name: Run Uptime Kuma UPS monitor setup script + command: > + {{ ansible_playbook_python }} + /tmp/setup_uptime_kuma_ups_monitor.py + "{{ uptime_kuma_api_url }}" + "{{ uptime_kuma_username }}" + "{{ uptime_kuma_password }}" + "{{ uptime_kuma_monitor_group }}" + "{{ monitor_name }}" + "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" + "{{ ups_heartbeat_timeout_seconds }}" + "{{ ups_heartbeat_retries }}" + "{{ ntfy_topic }}" + register: monitor_setup_result + delegate_to: localhost + become: no + changed_when: false + + - name: Parse monitor setup result + set_fact: + monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" + + - name: Set push URL as fact + set_fact: + uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" + + - name: Install required packages for UPS monitoring + package: + name: + - curl + state: present + + - name: Create monitoring script directory + file: + path: "{{ ups_monitoring_script_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Create UPS heartbeat monitoring script + copy: + dest: "{{ ups_monitoring_script_path }}" + content: | + #!/bin/bash + + # UPS Heartbeat Monitoring Script + # Sends heartbeat to Uptime Kuma only when UPS is on mains power + # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) + + LOG_FILE="{{ ups_log_file }}" + UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" + UPS_NAME="{{ ups_name }}" + + log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" + } + + send_heartbeat() { + local message="$1" + + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') + + local response http_code + response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Heartbeat sent: $message (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" + return 1 + fi + } + + main() { + local status charge runtime load + + status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) + + if [ -z "$status" ]; then + log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" + exit 1 + fi + + charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) + runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) + load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) + + if [[ "$status" == *"OL"* ]]; then + local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" + send_heartbeat "$message" + exit 0 + else + log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" + exit 1 + fi + } + + main + owner: root + group: root + mode: '0755' + + - name: Create systemd service for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" + content: | + [Unit] + Description=UPS Heartbeat Monitor + After=network.target nut-monitor.service + + [Service] + Type=oneshot + ExecStart={{ ups_monitoring_script_path }} + User=root + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: '0644' + + - name: Create systemd timer for UPS heartbeat + copy: + dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" + content: | + [Unit] + Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds + Requires={{ ups_systemd_service_name }}.service + + [Timer] + OnBootSec=1min + OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec + Persistent=true + + [Install] + WantedBy=timers.target + owner: root + group: root + mode: '0644' + + - name: Reload systemd daemon + systemd: + daemon_reload: yes + + - name: Enable and start UPS heartbeat timer + systemd: + name: "{{ ups_systemd_service_name }}.timer" + enabled: yes + state: started + + - name: Test UPS heartbeat script + command: "{{ ups_monitoring_script_path }}" + register: script_test + changed_when: false + + - name: Verify script execution + assert: + that: + - script_test.rc == 0 + fail_msg: "UPS heartbeat script failed - check UPS status and communication" + + - name: Display monitoring configuration + debug: + msg: + - "UPS Monitoring configured successfully" + - "" + - "NUT Configuration:" + - " UPS Name: {{ ups_name }}" + - " UPS Description: {{ ups_desc }}" + - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" + - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" + - "" + - "Uptime Kuma Monitoring:" + - " Monitor Name: {{ monitor_friendly_name }}" + - " Monitor Group: {{ uptime_kuma_monitor_group }}" + - " Push URL: {{ uptime_kuma_ups_push_url }}" + - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" + - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" + - "" + - "Scripts and Services:" + - " Script: {{ ups_monitoring_script_path }}" + - " Log: {{ ups_log_file }}" + - " Service: {{ ups_systemd_service_name }}.service" + - " Timer: {{ ups_systemd_service_name }}.timer" + + - name: Clean up temporary Uptime Kuma setup script + file: + path: /tmp/setup_uptime_kuma_ups_monitor.py + state: absent + delegate_to: localhost + become: no diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml index f9e6b0d..c0002f3 100644 --- a/ansible/infra/nodito/nodito_vars.yml +++ b/ansible/infra/nodito/nodito_vars.yml @@ -17,3 +17,12 @@ zfs_pool_name: "proxmox-tank-1" zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" + +# UPS Configuration (CyberPower CP900EPFCLCD via USB) +ups_name: cyberpower +ups_desc: "CyberPower CP900EPFCLCD" +ups_driver: usbhid-ups +ups_port: auto +ups_user: counterweight +ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power +ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power