From 102fad268c3c10d611491a602e5d191d7255001b Mon Sep 17 00:00:00 2001 From: counterweight Date: Thu, 30 Oct 2025 23:11:14 +0100 Subject: [PATCH 1/5] qemu agent for vm template --- .../33_proxmox_debian_cloud_template.yml | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index ff69525..6a123d7 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -26,6 +26,9 @@ proxmox_sshkey_path: "/home/{{ new_user }}/.ssh/authorized_keys" # Path to pubkey file for cloud-init injection proxmox_ci_upgrade: true # If true, run package upgrade on first boot + # Auto-install qemu-guest-agent in clones via cloud-init snippet + qemu_agent_snippet_filename: "user-data-qemu-agent.yaml" + # Storage to import disk into; use existing storage like local-lvm or your ZFS pool name proxmox_image_storage: "{{ zfs_pool_name }}" @@ -57,6 +60,34 @@ force: false when: not debian_image_stat.stat.exists + - name: Ensure ZFS storage allows snippets content + command: > + pvesm set {{ proxmox_image_storage }} --content images,rootdir,snippets + + - name: Ensure local storage allows snippets content + command: > + pvesm set local --content images,iso,vztmpl,snippets + failed_when: false + + - name: Ensure snippets directory exists on storage mountpoint + file: + path: "{{ zfs_pool_mountpoint }}/snippets" + state: directory + mode: '0755' + + - name: Write cloud-init user-data snippet to install qemu-guest-agent + copy: + dest: "{{ zfs_pool_mountpoint }}/snippets/{{ qemu_agent_snippet_filename }}" + mode: '0644' + content: | + #cloud-config + package_update: false + package_upgrade: false + packages: + - qemu-guest-agent + runcmd: + - [ systemctl, enable, --now, qemu-guest-agent ] + - name: Check if VMID already exists command: qm config {{ proxmox_template_vmid }} register: vmid_config_check @@ -91,6 +122,21 @@ when: - vmid_config_check.rc != 0 or not vm_already_template + - name: Check if ide2 (cloudinit) drive exists + command: qm config {{ proxmox_template_vmid }} + register: vm_config_check + failed_when: false + changed_when: false + when: + - vmid_config_check.rc == 0 + + - name: Remove existing ide2 (cloudinit) drive if it exists for idempotency + command: > + qm set {{ proxmox_template_vmid }} --delete ide2 + when: + - vmid_config_check.rc == 0 + - "'ide2:' in vm_config_check.stdout" + - name: Build consolidated qm set argument list (simplified) set_fact: qm_set_args: >- @@ -110,15 +156,16 @@ ] + (proxmox_ci_upgrade | bool | ternary(['--ciupgrade 1'], [])) + + ['--cicustom user=local:snippets/' ~ qemu_agent_snippet_filename] }} when: - - vmid_config_check.rc != 0 or not vm_already_template + - vmid_config_check.rc != 0 or not vm_already_template | default(false) - name: Apply consolidated qm set command: > qm set {{ proxmox_template_vmid }} {{ qm_set_args | join(' ') }} when: - - vmid_config_check.rc != 0 or not vm_already_template + - vmid_config_check.rc != 0 or not vm_already_template | default(false) - name: Resize primary disk to requested size command: > From 6f42e43efbc93cd77e7b787232d9988abe881577 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 31 Oct 2025 00:17:42 +0100 Subject: [PATCH 2/5] qemu works --- .../33_proxmox_debian_cloud_template.yml | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index 6a123d7..e0fedcd 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -39,8 +39,6 @@ changed_when: false failed_when: pve_version_check.rc != 0 - - - name: Ensure destination directory exists for cloud image file: path: "{{ debian_cloud_image_dest_dir }}" @@ -75,18 +73,31 @@ state: directory mode: '0755' + - name: Read SSH public key content + slurp: + src: "{{ proxmox_sshkey_path }}" + register: ssh_key_content + + - name: Extract SSH keys from authorized_keys file + set_fact: + ssh_keys_list: "{{ ssh_key_content.content | b64decode | split('\n') | select('match', '^ssh-') | list }}" + - name: Write cloud-init user-data snippet to install qemu-guest-agent copy: dest: "{{ zfs_pool_mountpoint }}/snippets/{{ qemu_agent_snippet_filename }}" mode: '0644' content: | #cloud-config - package_update: false - package_upgrade: false + user: {{ proxmox_ciuser }} + ssh_authorized_keys: + {{ ssh_keys_list | to_nice_yaml | indent(10, first=True) }} + package_update: true + package_upgrade: true packages: - qemu-guest-agent runcmd: - - [ systemctl, enable, --now, qemu-guest-agent ] + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent - name: Check if VMID already exists command: qm config {{ proxmox_template_vmid }} @@ -133,6 +144,7 @@ - name: Remove existing ide2 (cloudinit) drive if it exists for idempotency command: > qm set {{ proxmox_template_vmid }} --delete ide2 + register: ide2_removed when: - vmid_config_check.rc == 0 - "'ide2:' in vm_config_check.stdout" @@ -159,13 +171,13 @@ + ['--cicustom user=local:snippets/' ~ qemu_agent_snippet_filename] }} when: - - vmid_config_check.rc != 0 or not vm_already_template | default(false) + - vmid_config_check.rc != 0 or not vm_already_template | default(false) or ide2_removed.changed | default(false) - name: Apply consolidated qm set command: > qm set {{ proxmox_template_vmid }} {{ qm_set_args | join(' ') }} when: - - vmid_config_check.rc != 0 or not vm_already_template | default(false) + - vmid_config_check.rc != 0 or not vm_already_template | default(false) or ide2_removed.changed | default(false) - name: Resize primary disk to requested size command: > @@ -177,13 +189,3 @@ command: qm template {{ proxmox_template_vmid }} when: - vmid_config_check.rc == 0 and not vm_already_template or vmid_config_check.rc != 0 - - - name: Show resulting template configuration - command: qm config {{ proxmox_template_vmid }} - register: final_template_config - changed_when: false - - - name: Debug final template config - debug: - msg: "{{ final_template_config.stdout }}" - From 4d8e64146684b9115752963f01b701c9387a0150 Mon Sep 17 00:00:00 2001 From: counterweight Date: Fri, 31 Oct 2025 08:54:18 +0100 Subject: [PATCH 3/5] tf defined vms --- .gitignore | 11 +++++ tofu/nodito/README.md | 66 +++++++++++++++++++++++++++ tofu/nodito/main.tf | 67 ++++++++++++++++++++++++++++ tofu/nodito/provider.tf | 8 ++++ tofu/nodito/terraform.tfvars.example | 35 +++++++++++++++ tofu/nodito/variables.tf | 62 +++++++++++++++++++++++++ tofu/nodito/versions.tf | 12 +++++ 7 files changed, 261 insertions(+) create mode 100644 tofu/nodito/README.md create mode 100644 tofu/nodito/main.tf create mode 100644 tofu/nodito/provider.tf create mode 100644 tofu/nodito/terraform.tfvars.example create mode 100644 tofu/nodito/variables.tf create mode 100644 tofu/nodito/versions.tf diff --git a/.gitignore b/.gitignore index 312de7e..852f36f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,14 @@ +# OpenTofu / Terraform +.terraform/ +.tofu/ +.terraform.lock.hcl +.tofu.lock.hcl +terraform.tfstate +terraform.tfstate.* +crash.log +*.tfvars +*.tfvars.json + inventory.ini venv/* .env diff --git a/tofu/nodito/README.md b/tofu/nodito/README.md new file mode 100644 index 0000000..a6762a5 --- /dev/null +++ b/tofu/nodito/README.md @@ -0,0 +1,66 @@ +## Nodito VMs with OpenTofu (Proxmox) + +This directory lets you declare VMs on the `nodito` Proxmox node and apply with OpenTofu. It clones the Ansible-built template `debian-13-cloud-init` and places disks on the ZFS pool `proxmox-tank-1`. + +### Prereqs +- Proxmox API token with VM privileges. Example: user `root@pam`, token name `tofu`. +- OpenTofu installed. + ``` + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://get.opentofu.org/opentofu.gpg | sudo tee /etc/apt/keyrings/opentofu.gpg >/dev/null + curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | sudo gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu-repo.gpg >/dev/null + sudo chmod a+r /etc/apt/keyrings/opentofu.gpg /etc/apt/keyrings/opentofu-repo.gpg + + echo \ + "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main + deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" | \ + sudo tee /etc/apt/sources.list.d/opentofu.list > /dev/null + sudo chmod a+r /etc/apt/sources.list.d/opentofu.list + + sudo apt-get update + sudo apt-get install -y tofu + tofu version + ``` +- The Ansible template exists: `debian-13-cloud-init` (VMID 9001 by default). + +### Provider Auth +Create a `terraform.tfvars` (copy from `terraform.tfvars.example`) and set: +- `proxmox_api_url` (e.g. `https://nodito:8006/api2/json`) +- `proxmox_api_token_id` (e.g. `root@pam!tofu`) +- `proxmox_api_token_secret` +- `ssh_authorized_keys` (your public key content) + +Alternatively, you can export env vars and reference them in a tfvars file. + +### Declare VMs +Edit `terraform.tfvars` and fill the `vms` map. Example entry: +``` +vms = { + web1 = { + name = "web1" + cores = 2 + memory_mb = 2048 + disk_size_gb = 20 + ipconfig0 = "ip=dhcp" # or "ip=192.168.1.50/24,gw=192.168.1.1" + } +} +``` + +All VM disks are created on `zfs_storage_name` (defaults to `proxmox-tank-1`). Network attaches to `vmbr0`. VLAN can be set per-VM with `vlan_tag`. + +### Usage +``` +tofu init +tofu plan -var-file=terraform.tfvars +tofu apply -var-file=terraform.tfvars +``` + +### Notes +- Clones are full clones by default (`full_clone = true`). +- Cloud-init injects `cloud_init_user` and `ssh_authorized_keys`. +- Disks use `scsi0` on ZFS with `discard` enabled. + + diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf new file mode 100644 index 0000000..a4ce792 --- /dev/null +++ b/tofu/nodito/main.tf @@ -0,0 +1,67 @@ +locals { + default_ipconfig0 = "ip=dhcp" +} + +resource "proxmox_vm_qemu" "vm" { + for_each = var.vms + + name = each.value.name + target_node = var.proxmox_node + vmid = try(each.value.vmid, null) + + onboot = true + agent = 1 + clone = var.template_name + full_clone = true + vga { + type = "serial0" + } + + cpu { + sockets = 1 + cores = each.value.cores + type = "host" + } + memory = each.value.memory_mb + + scsihw = "virtio-scsi-pci" + boot = "c" + bootdisk = "scsi0" + + serial { + id = 0 + type = "socket" + } + + # Network: bridge vmbr0, optional VLAN tag + network { + id = 0 + model = "virtio" + bridge = "vmbr0" + tag = try(each.value.vlan_tag, 0) + } + + # Cloud-init: user, ssh keys, IP, and custom snippet for qemu-guest-agent + ciuser = var.cloud_init_user + sshkeys = var.ssh_authorized_keys + ipconfig0 = try(each.value.ipconfig0, local.default_ipconfig0) + cicustom = "user=local:snippets/user-data-qemu-agent.yaml" + + # Disk on ZFS storage + disk { + slot = "scsi0" + type = "disk" + storage = var.zfs_storage_name + size = "${each.value.disk_size_gb}G" + # optional flags like iothread/ssd/discard differ by provider versions; keep minimal + } + + # Cloud-init CD-ROM so ipconfig0/sshkeys apply + disk { + slot = "ide2" + type = "cloudinit" + storage = var.zfs_storage_name + } +} + + diff --git a/tofu/nodito/provider.tf b/tofu/nodito/provider.tf new file mode 100644 index 0000000..f907fb9 --- /dev/null +++ b/tofu/nodito/provider.tf @@ -0,0 +1,8 @@ +provider "proxmox" { + pm_api_url = var.proxmox_api_url + pm_api_token_id = var.proxmox_api_token_id + pm_api_token_secret = var.proxmox_api_token_secret + pm_tls_insecure = true +} + + diff --git a/tofu/nodito/terraform.tfvars.example b/tofu/nodito/terraform.tfvars.example new file mode 100644 index 0000000..cc88b3f --- /dev/null +++ b/tofu/nodito/terraform.tfvars.example @@ -0,0 +1,35 @@ +proxmox_api_url = "https://nodito:8006/api2/json" +proxmox_api_token_id = "root@pam!tofu" +proxmox_api_token_secret = "REPLACE_ME" + +proxmox_node = "nodito" +zfs_storage_name = "proxmox-tank-1" +template_name = "debian-13-cloud-init" +cloud_init_user = "counterweight" + +# paste your ~/.ssh/id_ed25519.pub or similar +ssh_authorized_keys = < Date: Sun, 2 Nov 2025 01:24:39 +0100 Subject: [PATCH 4/5] fix cloud init template --- ansible/infra/nodito/33_proxmox_debian_cloud_template.yml | 6 +----- tofu/nodito/main.tf | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index e0fedcd..061a2a3 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -58,11 +58,7 @@ force: false when: not debian_image_stat.stat.exists - - name: Ensure ZFS storage allows snippets content - command: > - pvesm set {{ proxmox_image_storage }} --content images,rootdir,snippets - - - name: Ensure local storage allows snippets content + - name: Ensure local storage allows snippets content (used for cloud-init snippets) command: > pvesm set local --content images,iso,vztmpl,snippets failed_when: false diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf index a4ce792..1fba9a5 100644 --- a/tofu/nodito/main.tf +++ b/tofu/nodito/main.tf @@ -42,6 +42,7 @@ resource "proxmox_vm_qemu" "vm" { } # Cloud-init: user, ssh keys, IP, and custom snippet for qemu-guest-agent + # Note: Using 'local' storage for snippets (not ZFS) as ZFS storage doesn't properly support snippet paths ciuser = var.cloud_init_user sshkeys = var.ssh_authorized_keys ipconfig0 = try(each.value.ipconfig0, local.default_ipconfig0) From 9d43c191892cf368dfb3e3d868b43038ac9d59a8 Mon Sep 17 00:00:00 2001 From: counterweight Date: Sun, 2 Nov 2025 01:48:07 +0100 Subject: [PATCH 5/5] hostname works --- .../33_proxmox_debian_cloud_template.yml | 9 +++---- tofu/nodito/main.tf | 26 ++++++++++--------- tofu/nodito/variables.tf | 14 +++++----- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index 061a2a3..40cf26e 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -78,15 +78,14 @@ set_fact: ssh_keys_list: "{{ ssh_key_content.content | b64decode | split('\n') | select('match', '^ssh-') | list }}" - - name: Write cloud-init user-data snippet to install qemu-guest-agent + - name: Write cloud-init vendor-data snippet to install qemu-guest-agent copy: dest: "{{ zfs_pool_mountpoint }}/snippets/{{ qemu_agent_snippet_filename }}" mode: '0644' content: | #cloud-config - user: {{ proxmox_ciuser }} - ssh_authorized_keys: - {{ ssh_keys_list | to_nice_yaml | indent(10, first=True) }} + # Vendor-data snippet: Proxmox will automatically set hostname from VM name when using vendor-data + # User info (ciuser/sshkeys) is set separately via Terraform/Proxmox parameters package_update: true package_upgrade: true packages: @@ -164,7 +163,7 @@ ] + (proxmox_ci_upgrade | bool | ternary(['--ciupgrade 1'], [])) - + ['--cicustom user=local:snippets/' ~ qemu_agent_snippet_filename] + + ['--cicustom vendor=local:snippets/' ~ qemu_agent_snippet_filename] }} when: - vmid_config_check.rc != 0 or not vm_already_template | default(false) or ide2_removed.changed | default(false) diff --git a/tofu/nodito/main.tf b/tofu/nodito/main.tf index 1fba9a5..cc7a75d 100644 --- a/tofu/nodito/main.tf +++ b/tofu/nodito/main.tf @@ -9,9 +9,9 @@ resource "proxmox_vm_qemu" "vm" { target_node = var.proxmox_node vmid = try(each.value.vmid, null) - onboot = true - agent = 1 - clone = var.template_name + onboot = true + agent = 1 + clone = var.template_name full_clone = true vga { type = "serial0" @@ -22,7 +22,7 @@ resource "proxmox_vm_qemu" "vm" { cores = each.value.cores type = "host" } - memory = each.value.memory_mb + memory = each.value.memory_mb scsihw = "virtio-scsi-pci" boot = "c" @@ -42,18 +42,20 @@ resource "proxmox_vm_qemu" "vm" { } # Cloud-init: user, ssh keys, IP, and custom snippet for qemu-guest-agent - # Note: Using 'local' storage for snippets (not ZFS) as ZFS storage doesn't properly support snippet paths - ciuser = var.cloud_init_user - sshkeys = var.ssh_authorized_keys + # Note: Using vendor-data snippet (instead of user-data) allows Proxmox to automatically + # set the hostname from the VM name. User info is set separately via ciuser/sshkeys. + # Using 'local' storage for snippets (not ZFS) as ZFS storage doesn't properly support snippet paths + ciuser = var.cloud_init_user + sshkeys = var.ssh_authorized_keys ipconfig0 = try(each.value.ipconfig0, local.default_ipconfig0) - cicustom = "user=local:snippets/user-data-qemu-agent.yaml" + cicustom = "vendor=local:snippets/user-data-qemu-agent.yaml" # Disk on ZFS storage disk { - slot = "scsi0" - type = "disk" - storage = var.zfs_storage_name - size = "${each.value.disk_size_gb}G" + slot = "scsi0" + type = "disk" + storage = var.zfs_storage_name + size = "${each.value.disk_size_gb}G" # optional flags like iothread/ssd/discard differ by provider versions; keep minimal } diff --git a/tofu/nodito/variables.tf b/tofu/nodito/variables.tf index 95108db..30a1418 100644 --- a/tofu/nodito/variables.tf +++ b/tofu/nodito/variables.tf @@ -48,13 +48,13 @@ variable "ssh_authorized_keys" { variable "vms" { description = "Map of VMs to create" type = map(object({ - name = string - vmid = optional(number) - cores = number - memory_mb = number - disk_size_gb = number - vlan_tag = optional(number) - ipconfig0 = optional(string) # e.g. "ip=dhcp" or "ip=192.168.1.50/24,gw=192.168.1.1" + name = string + vmid = optional(number) + cores = number + memory_mb = number + disk_size_gb = number + vlan_tag = optional(number) + ipconfig0 = optional(string) # e.g. "ip=dhcp" or "ip=192.168.1.50/24,gw=192.168.1.1" })) default = {} }