Add NVMe storage auto-setup, sops secrets, fix SSH permissions

- setup-node-storage service auto-partitions NVMe for containerd/longhorn
- Root password encrypted with sops/age, decrypted during build
- Fix SSH host key permissions (0600) so sshd actually starts
- Disable SSH socket activation for reliable boot
- Add OPERATIONS.md with runbook
- Makefile tracks source dependencies

build-image.sh

@@ -14,6 +14,22 @@ VERSION=$(date +%Y%m%d-%H%M)
 
 echo "Building netboot image version $VERSION"
 
+# Decrypt secrets from phoenix (requires SSH access as the invoking user, not root)
+echo "Decrypting secrets from phoenix..."
+SECRETS_FILE="$SCRIPT_DIR/secrets/netboot.sops.yaml"
+SUDO_USER_HOME=$(getent passwd "${SUDO_USER:-$USER}" | cut -d: -f6)
+if [ -f "$SECRETS_FILE" ]; then
+    # Run SSH as the original user (not root) to use their SSH keys
+    ROOT_PW_HASH=$(sudo -u "${SUDO_USER:-$USER}" bash -c "cat '$SECRETS_FILE' | ssh phoenix 'sops -d --input-type yaml --output-type yaml /dev/stdin'" | grep root_password_hash | cut -d' ' -f2)
+    if [ -z "$ROOT_PW_HASH" ]; then
+        echo "WARNING: Failed to decrypt root password, console login will be disabled"
+        ROOT_PW_HASH="*"
+    fi
+else
+    echo "WARNING: No secrets file found at $SECRETS_FILE, console login will be disabled"
+    ROOT_PW_HASH="*"
+fi
+
 # Clean previous build - unmount any stray mounts first
 if [ -d "$BUILD_DIR/rootfs" ]; then
     echo "Cleaning up previous build mounts..."
@@ -40,6 +56,17 @@ debootstrap --arch=amd64 --variant=minbase --components=main,universe,multiverse
     noble $BUILD_DIR/rootfs \
     http://archive.ubuntu.com/ubuntu
 
+# Write root password hash to temp file for chroot to read
+# Use /root/ not /tmp/ because systemd installation may mount tmpfs over /tmp
+mkdir -p "$BUILD_DIR/rootfs/root"
+if [ -n "$ROOT_PW_HASH" ] && [ "$ROOT_PW_HASH" != "*" ]; then
+    echo "$ROOT_PW_HASH" > "$BUILD_DIR/rootfs/root/.pw_hash"
+    echo "Root password hash written to rootfs"
+else
+    echo "*" > "$BUILD_DIR/rootfs/root/.pw_hash"
+    echo "WARNING: No valid password hash, console login will be disabled"
+fi
+
 # Chroot and configure
 cat << 'CHROOT_SCRIPT' > $BUILD_DIR/rootfs/setup.sh
 #!/bin/bash
@@ -118,7 +145,10 @@ apt-get install -y \
     less \
     rsync \
     git \
-    squashfs-tools
+    squashfs-tools \
+    parted \
+    fdisk \
+    gdisk
 
 # Clean up
 apt-get clean
@@ -148,11 +178,19 @@ EOF
 systemctl enable systemd-networkd
 systemctl enable systemd-resolved
 
-# Configure SSH
+# Configure SSH - disable socket activation, use traditional daemon
 sed -i 's/#PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
 sed -i 's/#PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+# Disable socket activation (Ubuntu 24.04 default) and use traditional sshd
+systemctl disable ssh.socket 2>/dev/null || true
+rm -f /etc/systemd/system/ssh.service.requires/ssh.socket 2>/dev/null || true
+rm -f /etc/systemd/system/sockets.target.wants/ssh.socket 2>/dev/null || true
 systemctl enable ssh
 
+# Fix SSH host key permissions (must be 0600 for private keys, sshd refuses otherwise)
+chmod 600 /etc/ssh/ssh_host_*_key
+chmod 644 /etc/ssh/ssh_host_*_key.pub
+
 # Create SSH directory for root
 mkdir -p /root/.ssh
 chmod 700 /root/.ssh
@@ -165,8 +203,10 @@ SSHKEY
 
 chmod 600 /root/.ssh/authorized_keys
 
-# Disable password authentication completely
-echo "root:*" | chpasswd -e
+# Set root password from decrypted hash (for console login only)
+ROOT_PW_HASH=$(cat /root/.pw_hash)
+echo "root:$ROOT_PW_HASH" | chpasswd -e
+rm -f /root/.pw_hash
 
 # Configure tmpfs mounts for ephemeral data
 cat >> /etc/fstab <<FSTAB
@@ -233,6 +273,17 @@ cp "$INITRAMFS_CONFIG/modules" "$BUILD_DIR/rootfs/etc/initramfs-tools/"
 cp -r "$INITRAMFS_CONFIG/hooks/"* "$BUILD_DIR/rootfs/usr/share/initramfs-tools/hooks/"
 cp -r "$INITRAMFS_CONFIG/scripts/"* "$BUILD_DIR/rootfs/usr/share/initramfs-tools/scripts/"
 
+# Install node storage setup service
+echo "Installing node storage setup service..."
+FILES_DIR="$SCRIPT_DIR/files"
+cp "$FILES_DIR/setup-node-storage" "$BUILD_DIR/rootfs/usr/local/bin/"
+chmod +x "$BUILD_DIR/rootfs/usr/local/bin/setup-node-storage"
+cp "$FILES_DIR/setup-node-storage.service" "$BUILD_DIR/rootfs/etc/systemd/system/"
+# Enable the service (create symlink manually since we can't run systemctl)
+mkdir -p "$BUILD_DIR/rootfs/etc/systemd/system/multi-user.target.wants"
+ln -sf /etc/systemd/system/setup-node-storage.service \
+    "$BUILD_DIR/rootfs/etc/systemd/system/multi-user.target.wants/setup-node-storage.service"
+
 # Build initramfs while /proc/sys/dev are still mounted
 echo "Building custom netboot initramfs..."
 KERNEL_VERSION=$(ls -1 $BUILD_DIR/rootfs/boot/vmlinuz-* | sed 's|.*/vmlinuz-||' | head -1)