VoiceStudio on WSL2 with an AMD GPU (ROCm, no Docker)¶

G:\WSL\
  images\vs-rocm-base.vhdx       # layer 1
  images\vs-0.5.7-ready.vhdx     # layer 2
  VoiceStudio\                   # the live distro's disk (a copy)
  scripts\provision-base.sh      # builds layer 1
  scripts\provision-voicestudio.sh  # builds layer 2
  scripts\verify-gpu.sh          # GPU check
  scripts\vs-wsl.ps1             # reset / start / save / check

#!/usr/bin/env bash
# Layer 1: Ubuntu 24.04 + ROCm 7.2.4 runtime + librocdxg 1.2.0 (AMD ROCDXG path for WSL2).
# Run as root inside a fresh Ubuntu-24.04 WSL distro.
set -euxo pipefail
export DEBIAN_FRONTEND=noninteractive

ROCM_VER=7.2.4
ROCM_DEB=amdgpu-install_7.2.4.70204-1_all.deb
# SHA-256 of the file used in the tested build (repo.radeon.com publishes no digest for it).
ROCM_DEB_SHA256=53c7ec5cace439f1ea97efebd59a3b32aa288c5c0bf76f57ab3cd7159dd98fad
ROCDXG_VER=1.2.0
# GitHub's published digest for the release asset; a mismatch stops before dpkg runs.
ROCDXG_SHA256=3ed9526719290cd8f590150dad8ea0f234fa779bea6a4c9a8449d7ae6b8cfb6e
USER_NAME=vs

apt-get update
apt-get -y upgrade
apt-get install -y --no-install-recommends \
  ca-certificates curl wget git git-lfs build-essential pkg-config unzip xz-utils \
  python3 python3-venv python3-dev python3-setuptools python3-wheel \
  ffmpeg libsndfile1 espeak-ng sox libsox-fmt-all portaudio19-dev libportaudio2 \
  sudo locales

locale-gen en_US.UTF-8

# Non-root user with passwordless sudo, default for the distro.
id "$USER_NAME" >/dev/null 2>&1 || useradd -m -s /bin/bash "$USER_NAME"
echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-$USER_NAME
chmod 0440 /etc/sudoers.d/90-$USER_NAME
cat > /etc/wsl.conf <<EOF
[user]
default=$USER_NAME
[boot]
systemd=false
[interop]
appendWindowsPath=false
EOF

# ROCm apt repo (no DKMS: WSL uses the Windows driver through /dev/dxg).
cd /tmp
wget -q "https://repo.radeon.com/amdgpu-install/${ROCM_VER}/ubuntu/noble/${ROCM_DEB}"
echo "${ROCM_DEB_SHA256}  ${ROCM_DEB}" | sha256sum -c -
apt-get install -y "./${ROCM_DEB}"
apt-get update
# Ubuntu's own old ROCm runtime packages shadow /opt/rocm if present.
apt-get remove -y libhsa-runtime64-1 libhsakmt1 2>/dev/null || true
apt-get install -y --no-install-recommends rocm-hip-runtime rocminfo
# AMD's repo.radeon.com torch wheels link these from /opt/rocm instead of bundling them.
apt-get install -y --no-install-recommends \
  rocblas rocsolver hipblas hipblaslt hipfft hiprand hipsolver hipsparse hipsparselt \
  miopen-hip rccl roctracer rocm-smi-lib libsox-dev
groupadd -f render; groupadd -f video
usermod -a -G render,video "$USER_NAME"

# librocdxg: the WSL bridge between ROCr and the Windows driver.
wget -q "https://github.com/ROCm/librocdxg/releases/download/v${ROCDXG_VER}/rocdxg-roct_${ROCDXG_VER}_amd64.deb"
echo "${ROCDXG_SHA256}  rocdxg-roct_${ROCDXG_VER}_amd64.deb" | sha256sum -c -
dpkg -i "rocdxg-roct_${ROCDXG_VER}_amd64.deb"

# Make the ROCm libraries (incl. librocdxg) visible to every process.
echo /opt/rocm/lib > /etc/ld.so.conf.d/rocm.conf
ldconfig
cat > /etc/profile.d/rocm-wsl.sh <<'EOF'
# ROCm 7.2.x on WSL2 needs DXG detection switched on (dropped from ROCm 7.13).
export HSA_ENABLE_DXG_DETECTION=1
export PATH=/opt/rocm/bin:$PATH
EOF

apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/*.deb
echo "BASE PROVISION OK"

New-Item -ItemType Directory -Force G:\WSL\images, G:\WSL\scripts, G:\WSL\build | Out-Null
wsl --install Ubuntu-24.04 --name VSBase --location G:\WSL\build\VSBase --no-launch
# Scripts saved from Windows have CRLF line endings; strip them on the way in.
wsl -d VSBase -u root -- bash -c "sed 's/\r$//' /mnt/g/WSL/scripts/provision-base.sh | bash"
wsl --terminate VSBase
wsl -d VSBase -- bash -lc "rocminfo | grep -E 'Marketing Name|Name: +gfx'"

wsl --manage VSBase --set-sparse true --allow-unsafe
wsl -d VSBase -u root -- fstrim -av
wsl --shutdown
wsl --export VSBase G:\WSL\images\vs-rocm-base.vhdx --format vhd
if ($LASTEXITCODE -eq 0) { wsl --unregister VSBase; Remove-Item -Recurse G:\WSL\build }

#!/usr/bin/env bash
# Layer 2: VoiceStudio v0.5.7 from source + PyTorch 2.8.0 for ROCm 7.2.4.
# Run as the default user (vs) inside a distro imported from the base image.
set -euxo pipefail
source /etc/profile.d/rocm-wsl.sh

VS_TAG=v0.5.7
VS_DIR="$HOME/VoiceStudio"
ROCM_WHEELS=https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.4/

# Toolchain: uv (Python) and bun (JS), both per-user.
command -v uv  >/dev/null || curl -LsSf https://astral.sh/uv/install.sh | sh
command -v bun >/dev/null || curl -fsSL https://bun.sh/install | bash
export PATH="$HOME/.local/bin:$HOME/.bun/bin:$PATH"

[ -d "$VS_DIR/.git" ] || git clone --branch "$VS_TAG" --depth 1 https://github.com/debpalash/VoiceStudio.git "$VS_DIR"
cd "$VS_DIR"
git -c advice.detachedHead=false checkout "$VS_TAG"

bun install --frozen-lockfile
bun run setup:api   # uv sync (locked CUDA torch) + scripts/setup.py

# Swap the locked CUDA torch for AMD's ROCm 7.2.4 build of the same versions.
uv pip install \
  --reinstall-package torch --reinstall-package torchaudio --reinstall-package torchvision \
  "torch==2.8.0+rocm7.2.4.lw.gitd618db82" "torchaudio==2.8.0+rocm7.2.4.git6e1c7fe9" \
  "torchvision==0.23.0+rocm7.2.4.git824e8c87" \
  --find-links "$ROCM_WHEELS"
# Drop the CUDA runtime wheels the locked build pulled in (several GB, unused on ROCm).
uv pip uninstall $(uv pip list --format freeze | grep -oE '^nvidia-[a-z0-9-]+-cu12' || true) 2>/dev/null || true
uv run --no-sync python -c "import torch; print('torch', torch.__version__, 'hip', torch.version.hip)"

# Launch helper: backend without re-sync (keeps the ROCm torch) + web UI.
cat > "$HOME/start-voicestudio.sh" <<'EOF'
#!/usr/bin/env bash
source /etc/profile.d/rocm-wsl.sh
export PATH="$HOME/.local/bin:$HOME/.bun/bin:$PATH"
export OMNIVOICE_TORCH_VARIANT=rocm
cd "$HOME/VoiceStudio"
bun run dev:api &
API=$!
trap 'kill $API 2>/dev/null' EXIT
# With networkingMode=Mirrored, Windows reaches WSL on 127.0.0.1 only (::1 hangs).
bun run wait:api && echo "Backend ready: http://127.0.0.1:3900  —  UI: http://127.0.0.1:3901"
bun run dev:frontend
EOF
chmod +x "$HOME/start-voicestudio.sh"
echo "VOICESTUDIO PROVISION OK"

#!/usr/bin/env bash
# GPU sanity check for the VoiceStudio venv: device, arch list, and a real matmul.
source /etc/profile.d/rocm-wsl.sh
export PATH="$HOME/.local/bin:$PATH"
cd ~/VoiceStudio
uv pip show nvidia-ml-py >/dev/null 2>&1 || uv pip install -q nvidia-ml-py
uv run --no-sync python - <<'EOF'
import time, torch
print("torch", torch.__version__, "hip", torch.version.hip)
ok = torch.cuda.is_available()
print("gpu available:", ok)
if ok:
    print("device:", torch.cuda.get_device_name(0), "| archs:", torch.cuda.get_arch_list())
    a = torch.randn(4096, 4096, device="cuda", dtype=torch.float16)
    torch.cuda.synchronize(); t = time.time()
    for _ in range(20): a @ a
    torch.cuda.synchronize()
    print(f"matmul x20 fp16 4096: {time.time()-t:.3f}s  (~{20*2*4096**3/(time.time()-t)/1e12:.1f} TFLOPS)")
raise SystemExit(0 if ok else 1)  # a missing GPU fails the check
EOF

wsl --import VoiceStudio G:\WSL\VoiceStudio G:\WSL\images\vs-rocm-base.vhdx --vhd
wsl --manage VoiceStudio --set-sparse true --allow-unsafe
wsl -d VoiceStudio -- bash -lc "sed 's/\r$//' /mnt/g/WSL/scripts/provision-voicestudio.sh | bash"
wsl -d VoiceStudio -- bash -lc "sed 's/\r$//' /mnt/g/WSL/scripts/verify-gpu.sh | bash"

torch 2.8.0+rocm7.2.4.gitd618db82 hip 7.2.53211-97f5574fe2
gpu available: True
device: AMD Radeon RX 7800 XT | archs: [..., 'gfx1100', 'gfx1101', ...]

wsl -d VoiceStudio -- bash -lc 'export PATH=$HOME/.local/bin:$PATH; uv cache clean; rm -rf ~/.bun/install/cache; sudo fstrim -av'
wsl --shutdown
wsl --export VoiceStudio G:\WSL\images\vs-0.5.7-ready.vhdx --format vhd

# VoiceStudio-on-WSL2 image manager (ROCm 7.2.4 + librocdxg).
#
#   .\vs-wsl.ps1 reset            # wipe "VoiceStudio" and re-import the ready image (fast)
#   .\vs-wsl.ps1 reset -From base # re-import the ROCm-only base and reinstall VoiceStudio v0.5.7
#   .\vs-wsl.ps1 start            # launch VoiceStudio, then open http://127.0.0.1:3901
#   .\vs-wsl.ps1 save             # snapshot the current "VoiceStudio" as the new ready image
#   .\vs-wsl.ps1 check            # GPU check (rocminfo + torch)
param(
    [Parameter(Mandatory)][ValidateSet('reset', 'start', 'save', 'check')][string]$Action,
    [ValidateSet('ready', 'base')][string]$From = 'ready'
)
$ErrorActionPreference = 'Stop'
$Root    = Split-Path $PSScriptRoot          # the folder above scripts\, e.g. G:\WSL
$Distro  = 'VoiceStudio'
$Images  = @{ base = "$Root\images\vs-rocm-base.vhdx"; ready = "$Root\images\vs-0.5.7-ready.vhdx" }
$Install = "$Root\$Distro"
# Windows path of scripts\ as seen from WSL, e.g. G:\WSL\scripts -> /mnt/g/WSL/scripts
$Scripts = '/mnt/' + $PSScriptRoot.Substring(0, 1).ToLower() + ($PSScriptRoot.Substring(2) -replace '\\', '/')

function Stop-Distro($name) {
    wsl --terminate $name | Out-Null
    # The VM keeps the VHDX open for a few seconds after terminate.
    for ($i = 0; $i -lt 30; $i++) {
        $row = (wsl -l -v) -replace "`0", '' | Where-Object { $_ -match "\s$name\s" }
        if ($row -match 'Stopped') { Start-Sleep 3; return }
        Start-Sleep 1
    }
    throw "$name did not stop"
}

function Export-Image($name, $dest) {
    Stop-Distro $name
    $tmp = $dest -replace '\.vhdx$', '.partial.vhdx'   # wsl insists on a .vhdx extension
    for ($try = 1; $try -le 3; $try++) {
        wsl --export $name $tmp --format vhd
        if ($LASTEXITCODE -eq 0 -and (Test-Path $tmp)) { Move-Item -Force $tmp $dest; return }
        # The utility VM can hold the disk even with the distro stopped; stop the VM.
        wsl --shutdown; Start-Sleep 5
    }
    throw "export of $name failed; nothing was removed"
}

switch ($Action) {
    'reset' {
        if (-not (Test-Path $Images[$From])) { throw "missing image $($Images[$From])" }
        if (((wsl -l -q) -replace "`0", '') -contains $Distro) { wsl --unregister $Distro }
        wsl --import $Distro $Install $Images[$From] --vhd   # copies; the image stays untouched
        if ($LASTEXITCODE -ne 0) { throw "import of $($Images[$From]) failed" }
        # Sparse VHDX gives freed space back to the host drive (run `sudo fstrim -av` inside after big deletes).
        # WSL calls sparse mode unsafe (possible disk corruption): the images stay the known-good copy.
        wsl --manage $Distro --set-sparse true --allow-unsafe
        if ($LASTEXITCODE -ne 0) { throw "enabling sparse mode on $Distro failed" }
        if ($From -eq 'base') {
            wsl -d $Distro -- bash -lc "sed 's/\r$//' '$Scripts/provision-voicestudio.sh' | bash"
        }
    }
    'start' { wsl -d $Distro -- bash -lc '~/start-voicestudio.sh' }
    'save'  { Export-Image $Distro $Images.ready }
    'check' { wsl -d $Distro -- bash -lc "sed 's/\r$//' '$Scripts/verify-gpu.sh' | bash" }
}

powershell -ExecutionPolicy Bypass -File G:\WSL\scripts\vs-wsl.ps1 start

The process cannot access the file because it is being used by another process.
Error code: Wsl/Service/ERROR_SHARING_VIOLATION

The specified file must have the .vhdx file extension.
Error code: Wsl/Service/WSL_E_EXPORT_FAILED

ImportError: libroctx64.so.4: cannot open shared object file: No such file or directory

cd ~/VoiceStudio/.venv/lib/python3.11/site-packages
for f in torch/lib/*.so torchaudio/lib/*.so torchvision/*.so; do ldd "$f" | grep 'not found'; done | sort -u

There is not enough space on the disk.  Error code: Wsl/Service/0x80070070

curl -X PUT http://127.0.0.1:3900/api/settings/perf/torch-compile-disabled \
  -H 'Content-Type: application/json' -d '{"enabled":true}'