From 0c34e25502ffa058509f99d9b0bd18ef087eb363 Mon Sep 17 00:00:00 2001 From: counterweight Date: Wed, 29 Oct 2025 00:13:15 +0100 Subject: [PATCH] packages, zfs pool --- 01_infra_setup.md | 41 ++- .../infra/30_proxmox_bootstrap_playbook.yml | 123 ------- .../31_proxmox_community_repos_playbook.yml | 317 ++++++++++++++++++ .../nodito/32_zfs_pool_setup_playbook.yml | 172 ++++++++++ ansible/infra/nodito/nodito_vars.yml | 6 + 5 files changed, 535 insertions(+), 124 deletions(-) delete mode 100644 ansible/infra/30_proxmox_bootstrap_playbook.yml create mode 100644 ansible/infra/nodito/31_proxmox_community_repos_playbook.yml create mode 100644 ansible/infra/nodito/32_zfs_pool_setup_playbook.yml diff --git a/01_infra_setup.md b/01_infra_setup.md index 9474c2f..e4353b2 100644 --- a/01_infra_setup.md +++ b/01_infra_setup.md @@ -63,7 +63,7 @@ Note that, by applying these playbooks, both the root user and the `counterweigh ### Bootstrap SSH Key Access and Create User * Nodito starts with password authentication enabled and no SSH keys configured. We need to bootstrap SSH key access first. -* Run the complete setup with: `ansible-playbook -i inventory.ini infra/00_proxmox_bootstrap_playbook.yml -e 'ansible_user=root'` +* Run the complete setup with: `ansible-playbook -i inventory.ini infra/nodito/30_proxmox_bootstrap_playbook.yml -e 'ansible_user=root'` * This single playbook will: * Set up SSH key access for root * Create the counterweight user with SSH keys @@ -74,6 +74,18 @@ Note that, by applying these playbooks, both the root user and the `counterweigh Note that, by applying these playbooks, both the root user and the `counterweight` user will use the same SSH pubkey for auth, but root login will be disabled. +### Switch to Community Repositories + +* Proxmox VE installations typically come with enterprise repositories enabled, which require a subscription. To avoid subscription warnings and use the community repositories instead: +* Run the repository switch with: `ansible-playbook -i inventory.ini infra/nodito/32_proxmox_community_repos_playbook.yml` +* This playbook will: + * Detect whether your Proxmox installation uses modern deb822 format (Proxmox VE 9) or legacy format (Proxmox VE 8) + * Remove enterprise repository files and create community repository files + * Disable subscription nag messages in both web and mobile interfaces + * Update Proxmox packages from the community repository + * Verify the changes are working correctly +* After running this playbook, clear your browser cache or perform a hard reload (Ctrl+Shift+R) before using the Proxmox VE Web UI to avoid UI display issues. + ### Deploy CPU Temperature Monitoring * The nodito server can be configured with CPU temperature monitoring that sends alerts to Uptime Kuma when temperatures exceed a threshold. @@ -89,6 +101,33 @@ Note that, by applying these playbooks, both the root user and the `counterweigh * Set up a systemd service and timer for automated monitoring * Send alerts to Uptime Kuma when temperature exceeds the threshold (default: 80°C) +### Setup ZFS Storage Pool + +* The nodito server can be configured with a ZFS RAID 1 storage pool for Proxmox VM storage, providing redundancy and data integrity. +* Before running the ZFS pool setup playbook, you need to identify your disk IDs and configure them in the variables file: + * SSH into your nodito server and run: `ls -la /dev/disk/by-id/ | grep -E "(ata-|scsi-|nvme-)"` + * This will show you the persistent disk identifiers for all your disks. Look for the two disks you want to use for the ZFS pool. + * Example output: + ``` + lrwxrwxrwx 1 root root 9 Dec 15 10:30 ata-WDC_WD40EFRX-68N32N0_WD-WCC7K1234567 -> ../../sdb + lrwxrwxrwx 1 root root 9 Dec 15 10:30 ata-WDC_WD40EFRX-68N32N0_WD-WCC7K7654321 -> ../../sdc + ``` + * Update `ansible/infra/nodito/nodito_vars.yml` with your actual disk IDs: + ```yaml + zfs_disk_1: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K1234567" + zfs_disk_2: "/dev/disk/by-id/ata-WDC_WD40EFRX-68N32N0_WD-WCC7K7654321" + ``` +* Run the ZFS pool setup with: `ansible-playbook -i inventory.ini infra/nodito/32_zfs_pool_setup_playbook.yml` +* This will: + * Validate Proxmox VE and ZFS installation + * Install ZFS utilities and kernel modules + * Create a RAID 1 (mirror) ZFS pool named `proxmox-storage` with optimized settings + * Configure ZFS pool properties (ashift=12, compression=lz4, atime=off, etc.) + * Export and re-import the pool for Proxmox compatibility + * Configure Proxmox to use the ZFS pool storage (zfspool type) + * Enable ZFS services for automatic pool import on boot +* **Warning**: This will destroy all data on the specified disks. Make sure you're using the correct disk IDs and that the disks don't contain important data. + ## GPG Keys Some of the backups are stored encrypted for security. To allow this, fill in the gpg variables listed in `example.inventory.ini` under the `lapy` block. diff --git a/ansible/infra/30_proxmox_bootstrap_playbook.yml b/ansible/infra/30_proxmox_bootstrap_playbook.yml deleted file mode 100644 index 4abadc8..0000000 --- a/ansible/infra/30_proxmox_bootstrap_playbook.yml +++ /dev/null @@ -1,123 +0,0 @@ -- name: Bootstrap Nodito SSH Key Access - hosts: nodito - become: true - vars_files: - - ../infra_vars.yml - - tasks: - - name: Ensure SSH directory exists for root - file: - path: /root/.ssh - state: directory - mode: "0700" - owner: root - group: root - - - name: Install SSH public key for root - authorized_key: - user: root - key: "{{ lookup('file', ansible_ssh_private_key_file + '.pub') }}" - state: present - - - name: Ensure SSH key-based authentication is enabled - lineinfile: - path: /etc/ssh/sshd_config - regexp: "^#?PubkeyAuthentication" - line: "PubkeyAuthentication yes" - state: present - backrefs: yes - - - name: Ensure AuthorizedKeysFile is properly configured - lineinfile: - path: /etc/ssh/sshd_config - regexp: "^#?AuthorizedKeysFile" - line: "AuthorizedKeysFile .ssh/authorized_keys" - state: present - backrefs: yes - - - name: Restart SSH service - service: - name: ssh - state: restarted - - - name: Wait for SSH to be ready - wait_for: - port: "{{ ssh_port }}" - host: "{{ ansible_host }}" - delay: 2 - timeout: 30 - - - name: Test SSH key authentication - command: whoami - register: ssh_key_test - changed_when: false - - - name: Verify SSH key authentication works - assert: - that: - - ssh_key_test.stdout == "root" - fail_msg: "SSH key authentication failed - expected 'root', got '{{ ssh_key_test.stdout }}'" - - - name: Create new user - user: - name: "{{ new_user }}" - groups: sudo - shell: /bin/bash - state: present - create_home: yes - - - name: Set up SSH directory for new user - file: - path: "/home/{{ new_user }}/.ssh" - state: directory - mode: "0700" - owner: "{{ new_user }}" - group: "{{ new_user }}" - - - name: Install SSH public key for new user - authorized_key: - user: "{{ new_user }}" - key: "{{ lookup('file', ansible_ssh_private_key_file + '.pub') }}" - state: present - - - name: Allow new user to run sudo without password - copy: - dest: "/etc/sudoers.d/{{ new_user }}" - content: "{{ new_user }} ALL=(ALL) NOPASSWD:ALL" - owner: root - group: root - mode: "0440" - - - name: Disable root login - lineinfile: - path: /etc/ssh/sshd_config - regexp: "^#?PermitRootLogin .*" - line: "PermitRootLogin no" - state: present - backrefs: yes - - - name: Disable password authentication - lineinfile: - path: /etc/ssh/sshd_config - regexp: "^#?PasswordAuthentication .*" - line: "PasswordAuthentication no" - state: present - backrefs: yes - - - name: Restart SSH service - service: - name: ssh - state: restarted - - - name: Wait for SSH to be ready - wait_for: - port: "{{ ssh_port }}" - host: "{{ ansible_host }}" - delay: 2 - timeout: 30 - - - name: Test connection with new user - command: whoami - become_user: "{{ new_user }}" - register: new_user_test - changed_when: false diff --git a/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml new file mode 100644 index 0000000..64d81c2 --- /dev/null +++ b/ansible/infra/nodito/31_proxmox_community_repos_playbook.yml @@ -0,0 +1,317 @@ +- name: Switch Proxmox VE from Enterprise to Community Repositories + hosts: nodito + become: true + vars_files: + - ../infra_vars.yml + + tasks: + - name: Check for deb822 sources format + find: + paths: /etc/apt/sources.list.d/ + patterns: "*.sources" + file_type: file + register: deb822_sources + changed_when: false + + - name: Check for legacy .list files + find: + paths: /etc/apt/sources.list.d/ + patterns: "*.list" + file_type: file + register: legacy_list_files + changed_when: false + + - name: Check main sources.list for Proxmox entries + command: grep -q "proxmox\|trixie" /etc/apt/sources.list + register: main_sources_check + failed_when: false + changed_when: false + + - name: Display current repository status + debug: + msg: | + Repository status: + - deb822 sources files: {{ deb822_sources.matched }} + - legacy .list files: {{ legacy_list_files.matched }} + - Proxmox/Trixie entries in sources.list: {{ main_sources_check.rc == 0 }} + + - name: Check for enterprise repository in deb822 format + shell: | + for file in /etc/apt/sources.list.d/*.sources; do + if grep -q "Components:.*pve-enterprise" "$file" 2>/dev/null; then + echo "$file" + break + fi + done + register: enterprise_deb822_check + failed_when: false + changed_when: false + + - name: Check for enterprise repository in legacy format + shell: | + for file in /etc/apt/sources.list.d/*.list; do + if grep -q "enterprise.proxmox.com" "$file" 2>/dev/null; then + echo "$file" + break + fi + done + register: enterprise_legacy_check + failed_when: false + changed_when: false + + - name: Check for Ceph enterprise repository in deb822 format + shell: | + for file in /etc/apt/sources.list.d/*.sources; do + if grep -q "enterprise.proxmox.com.*ceph" "$file" 2>/dev/null; then + echo "$file" + break + fi + done + register: ceph_enterprise_deb822_check + failed_when: false + changed_when: false + + - name: Check for Ceph enterprise repository in legacy format + shell: | + for file in /etc/apt/sources.list.d/*.list; do + if grep -q "enterprise.proxmox.com.*ceph" "$file" 2>/dev/null; then + echo "$file" + break + fi + done + register: ceph_enterprise_legacy_check + failed_when: false + changed_when: false + + - name: Backup enterprise repository files + copy: + src: "{{ item }}" + dest: "{{ item }}.backup" + remote_src: yes + backup: yes + loop: "{{ (enterprise_deb822_check.stdout_lines + enterprise_legacy_check.stdout_lines + ceph_enterprise_deb822_check.stdout_lines + ceph_enterprise_legacy_check.stdout_lines) | select('string') | list }}" + when: (enterprise_deb822_check.stdout_lines + enterprise_legacy_check.stdout_lines + ceph_enterprise_deb822_check.stdout_lines + ceph_enterprise_legacy_check.stdout_lines) | select('string') | list | length > 0 + + - name: Delete enterprise repository files (deb822 format) + file: + path: "{{ item }}" + state: absent + loop: "{{ enterprise_deb822_check.stdout_lines | select('string') | list }}" + when: enterprise_deb822_check.stdout_lines | select('string') | list | length > 0 + + - name: Delete enterprise repository files (legacy format) + file: + path: "{{ item }}" + state: absent + loop: "{{ enterprise_legacy_check.stdout_lines | select('string') | list }}" + when: enterprise_legacy_check.stdout_lines | select('string') | list | length > 0 + + - name: Delete Ceph enterprise repository files (deb822 format) + file: + path: "{{ item }}" + state: absent + loop: "{{ ceph_enterprise_deb822_check.stdout_lines | select('string') | list }}" + when: ceph_enterprise_deb822_check.stdout_lines | select('string') | list | length > 0 + + - name: Delete Ceph enterprise repository files (legacy format) + file: + path: "{{ item }}" + state: absent + loop: "{{ ceph_enterprise_legacy_check.stdout_lines | select('string') | list }}" + when: ceph_enterprise_legacy_check.stdout_lines | select('string') | list | length > 0 + + - name: Create community repository file (deb822 format) + copy: + dest: /etc/apt/sources.list.d/proxmox.sources + content: | + Types: deb + URIs: http://download.proxmox.com/debian/pve + Suites: trixie + Components: pve-no-subscription + Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg + owner: root + group: root + mode: '0644' + backup: yes + when: deb822_sources.matched > 0 + + - name: Create community repository file (legacy format) + copy: + dest: /etc/apt/sources.list.d/pve-no-subscription.list + content: | + # PVE pve-no-subscription repository provided by proxmox.com, + # NOT recommended for production use + deb http://download.proxmox.com/debian/pve trixie pve-no-subscription + owner: root + group: root + mode: '0644' + backup: yes + when: deb822_sources.matched == 0 + + - name: Create Ceph community repository file (deb822 format) + copy: + dest: /etc/apt/sources.list.d/ceph.sources + content: | + Types: deb + URIs: http://download.proxmox.com/debian/ceph-squid + Suites: trixie + Components: no-subscription + Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg + owner: root + group: root + mode: '0644' + backup: yes + when: deb822_sources.matched > 0 + + - name: Create Ceph community repository file (legacy format) + copy: + dest: /etc/apt/sources.list.d/ceph-no-subscription.list + content: | + # Ceph no-subscription repository provided by proxmox.com, + # NOT recommended for production use + deb http://download.proxmox.com/debian/ceph-squid trixie no-subscription + owner: root + group: root + mode: '0644' + backup: yes + when: deb822_sources.matched == 0 + + - name: Update package cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Verify community repository is working + command: apt-cache policy proxmox-ve + register: community_repo_verify + changed_when: false + + - name: Display community repository verification + debug: + var: community_repo_verify.stdout_lines + + - name: Update Proxmox packages from community repository + apt: + name: proxmox-ve + state: latest + update_cache: yes + + - name: Verify Proxmox VE version + command: pveversion + register: proxmox_version + changed_when: false + + - name: Display Proxmox VE version + debug: + msg: "Proxmox VE version: {{ proxmox_version.stdout }}" + + - name: Check repository status + shell: apt-cache policy | grep -A 5 -B 5 proxmox + register: final_repo_status + changed_when: false + + - name: Display final repository status + debug: + var: final_repo_status.stdout_lines + + - name: Verify no enterprise repository warnings + command: apt update + register: apt_update_result + changed_when: false + + - name: Check for enterprise repository warnings + fail: + msg: "Enterprise repository warnings detected. Check the output above." + when: "'enterprise.proxmox.com' in apt_update_result.stdout" + + - name: Create subscription nag removal script + copy: + dest: /usr/local/bin/pve-remove-nag.sh + content: | + #!/bin/sh + WEB_JS=/usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js + if [ -s "$WEB_JS" ] && ! grep -q NoMoreNagging "$WEB_JS"; then + echo "Patching Web UI nag..." + sed -i -e "/data\.status/ s/!//" -e "/data\.status/ s/active/NoMoreNagging/" "$WEB_JS" + fi + + MOBILE_TPL=/usr/share/pve-yew-mobile-gui/index.html.tpl + MARKER="" + if [ -f "$MOBILE_TPL" ] && ! grep -q "$MARKER" "$MOBILE_TPL"; then + echo "Patching Mobile UI nag..." + printf "%s\n" \ + "$MARKER" \ + "" \ + "" >> "$MOBILE_TPL" + fi + owner: root + group: root + mode: '0755' + + - name: Create APT configuration for nag removal + copy: + dest: /etc/apt/apt.conf.d/no-nag-script + content: | + DPkg::Post-Invoke { "/usr/local/bin/pve-remove-nag.sh"; }; + owner: root + group: root + mode: '0644' + + - name: Run nag removal script immediately + command: /usr/local/bin/pve-remove-nag.sh + changed_when: false + + - name: Reinstall proxmox-widget-toolkit to apply nag removal + apt: + name: proxmox-widget-toolkit + state: present + force: yes + + - name: Clean up backup files + file: + path: "{{ item }}" + state: absent + loop: + - /etc/apt/sources.list.d/ceph.sources.backup + - /etc/apt/sources.list.d/pve-enterprise.sources.backup + ignore_errors: yes + + - name: Success message + debug: + msg: | + Successfully switched from Proxmox Enterprise to Community repositories. + Enterprise repository has been disabled and community repository is now active. + Subscription nag messages have been disabled. + Proxmox VE version: {{ proxmox_version.stdout }} + + IMPORTANT: Clear your browser cache or perform a hard reload (Ctrl+Shift+R) + before using the Proxmox VE Web UI to avoid UI display issues. diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml new file mode 100644 index 0000000..192cd00 --- /dev/null +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -0,0 +1,172 @@ +- name: Setup ZFS RAID 1 Pool for Proxmox Storage + hosts: nodito + become: true + vars_files: + - ../infra_vars.yml + - nodito_vars.yml + + tasks: + - name: Verify Proxmox VE is running + command: pveversion + register: pve_version_check + changed_when: false + failed_when: pve_version_check.rc != 0 + + - name: Update package cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Install ZFS utilities + package: + name: + - zfsutils-linux + - zfs-initramfs + state: present + + - name: Load ZFS kernel module + modprobe: + name: zfs + + - name: Ensure ZFS module loads at boot + lineinfile: + path: /etc/modules + line: zfs + state: present + + - name: Check if ZFS pool already exists + command: zpool list {{ zfs_pool_name }} + register: zfs_pool_exists + failed_when: false + changed_when: false + + - name: Check if disks are in use + shell: | + for disk in {{ zfs_disk_1 }} {{ zfs_disk_2 }}; do + if mount | grep -q "^$disk"; then + echo "ERROR: $disk is mounted" + exit 1 + fi + if lsblk -n -o MOUNTPOINT "$disk" | grep -v "^$" | grep -q .; then + echo "ERROR: $disk has mounted partitions" + exit 1 + fi + done + register: disk_usage_check + failed_when: disk_usage_check.rc != 0 + changed_when: false + + - name: Create ZFS RAID 1 pool with optimized settings + command: > + zpool create {{ zfs_pool_name }} + -o ashift=12 + -O mountpoint=none + mirror {{ zfs_disk_1 }} {{ zfs_disk_2 }} + when: zfs_pool_exists.rc != 0 + register: zfs_pool_create_result + + - name: Check if ZFS dataset already exists + command: zfs list {{ zfs_pool_name }}/vm-storage + register: zfs_dataset_exists + failed_when: false + changed_when: false + + - name: Create ZFS dataset for Proxmox storage + command: zfs create {{ zfs_pool_name }}/vm-storage + when: zfs_dataset_exists.rc != 0 + register: zfs_dataset_create_result + + - name: Set ZFS dataset properties for Proxmox + command: zfs set {{ item.property }}={{ item.value }} {{ zfs_pool_name }}/vm-storage + loop: + - { property: "mountpoint", value: "{{ zfs_pool_mountpoint }}" } + - { property: "compression", value: "lz4" } + - { property: "atime", value: "off" } + - { property: "xattr", value: "sa" } + - { property: "acltype", value: "posixacl" } + - { property: "dnodesize", value: "auto" } + when: zfs_dataset_exists.rc != 0 + + - name: Set ZFS pool properties for Proxmox + command: zpool set autotrim=off {{ zfs_pool_name }} + when: zfs_pool_exists.rc != 0 + + - name: Set ZFS pool mountpoint for Proxmox + command: zfs set mountpoint={{ zfs_pool_mountpoint }} {{ zfs_pool_name }} + when: zfs_pool_exists.rc == 0 + + - name: Export and re-import ZFS pool for Proxmox compatibility + shell: | + zpool export {{ zfs_pool_name }} + zpool import {{ zfs_pool_name }} + when: zfs_pool_exists.rc != 0 + register: zfs_pool_import_result + + + - name: Ensure ZFS services are enabled + systemd: + name: "{{ item }}" + enabled: yes + state: started + loop: + - zfs-import-cache + - zfs-import-scan + - zfs-mount + - zfs-share + - zfs-zed + + - name: Check if ZFS pool storage already exists in Proxmox config + stat: + path: /etc/pve/storage.cfg + register: storage_cfg_file + + - name: Check if storage name exists in Proxmox config + shell: "grep -q '^zfspool: {{ zfs_pool_name }}' /etc/pve/storage.cfg" + register: storage_exists_check + failed_when: false + changed_when: false + when: storage_cfg_file.stat.exists + + - name: Set storage not configured when config file doesn't exist + set_fact: + storage_exists_check: + rc: 1 + when: not storage_cfg_file.stat.exists + + - name: Debug storage configuration status + debug: + msg: | + Config file exists: {{ storage_cfg_file.stat.exists }} + Storage check result: {{ storage_exists_check.rc }} + Pool exists: {{ zfs_pool_exists.rc == 0 }} + Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} + Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} + + - name: Remove existing storage if it exists + command: pvesm remove {{ zfs_pool_name }} + register: pvesm_remove_result + failed_when: false + when: + - zfs_pool_exists.rc == 0 + - storage_exists_check.rc == 0 + + - name: Add ZFS pool storage to Proxmox using pvesm + command: > + pvesm add zfspool {{ zfs_pool_name }} + --pool {{ zfs_pool_name }} + --content rootdir,images + --sparse 1 + when: + - zfs_pool_exists.rc == 0 + - storage_exists_check.rc != 0 + register: pvesm_add_result + + - name: Verify ZFS pool is healthy + command: zpool status {{ zfs_pool_name }} + register: final_zfs_status + changed_when: false + + - name: Fail if ZFS pool is not healthy + fail: + msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" + when: "'ONLINE' not in final_zfs_status.stdout" diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml index c4a8ebd..f9e6b0d 100644 --- a/ansible/infra/nodito/nodito_vars.yml +++ b/ansible/infra/nodito/nodito_vars.yml @@ -11,3 +11,9 @@ log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" # System Configuration systemd_service_name: nodito-cpu-temp-monitor + +# ZFS Pool Configuration +zfs_pool_name: "proxmox-tank-1" +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror +zfs_pool_mountpoint: "/var/lib/vz"