name: Dev Build (all platforms)

# Staging pipeline for the `dev` branch.
#
# Every push to `dev` builds every supported backend and republishes the
# rolling `dev-latest` prerelease with all archives, so a build can be
# grabbed and smoke-tested on real hardware without a local checkout.
# Pull requests into `dev` build everything but publish nothing.
#
# Stable releases are cut separately from `master` (see release-turboquant.yml).

on:
  workflow_dispatch:
  push:
    branches:
      - dev
    paths:
      - '.github/workflows/dev-build.yml'
      - '.github/actions/windows-setup-cuda/**'
      - '.github/actions/windows-code-sign/**'
      - '**/CMakeLists.txt'
      - '**/*.h'
      - '**/*.hpp'
      - '**/*.c'
      - '**/*.cpp'
      - '**/*.cu'
      - '**/*.cuh'
      - '**/*.comp'
      - '**/*.glsl'
      - '**/*.metal'
  # No paths filter: these jobs are required status checks on dev, and a
  # required check that never reports leaves the PR stuck in "Expected"
  # forever (e.g. a docs-only PR). Runs on PRs into master too so the
  # dev -> master promotion PR reports the same checks.
  pull_request:
    branches:
      - dev
      - master

# Group by ref so consecutive pushes to dev serialize: a newer push cancels
# the older in-flight build instead of racing it for the dev-latest release.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  LLAMA_LOG_COLORS: 1
  LLAMA_LOG_PREFIX: 1
  LLAMA_LOG_TIMESTAMPS: 1
  # Keep in sync with the upstream `windows` release job.
  VULKAN_VERSION: 1.4.313.2

jobs:
  linux-x64-vulkan:
    runs-on: ubuntu-22.04

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-x64-vulkan
          evict-old-files: 1d

      - name: Dependencies
        run: |
          wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
          sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
          sudo apt-get update -y
          sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk

      - name: Build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_VULKAN=ON \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ./build/bin/llama-server --help 2>&1 | grep -A2 "cache-type-k" || true
          ldd build/bin/llama-server || true

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cd release
          zip -ry ../llama-turboquant-linux-x64-vulkan.zip .
          tar -czf ../llama-turboquant-linux-x64-vulkan.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-x64-vulkan
          path: |
            llama-turboquant-linux-x64-vulkan.zip
            llama-turboquant-linux-x64-vulkan.tar.gz
          retention-days: 14

  linux-x64-cuda-13-3:
    name: linux-x64-cuda-13.3
    runs-on: ubuntu-22.04

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-x64-cuda-13.3
          evict-old-files: 1d

      - name: Install CUDA Toolkit
        run: |
          wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
          sudo dpkg -i cuda-keyring_1.1-1_all.deb
          sudo apt-get update -y
          sudo apt-get install -y build-essential \
            cuda-nvcc-13-3 cuda-cccl-13-3 cuda-cudart-dev-13-3 libcublas-dev-13-3
          echo "/usr/local/cuda/bin" >> "$GITHUB_PATH"

      - name: Build
        # Consumer SASS: RTX 30 (86), RTX 40 (89), RTX 50 / Blackwell (120).
        # Two PTX floors: 75-virtual for Turing, 80-virtual so the server cards
        # (A100 80, H100 90, B200 100) JIT Ampere-class code instead of Turing
        # code -- cp.async and the Ampere MMA path are gated on __CUDA_ARCH__
        # >= 800, so a compute_75 PTX fallback quietly cost them both. Runner
        # has no GPU: build only, backend is a dlopen'd libggml-cuda.so.
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_CUB_3DOT2=ON \
            -DCMAKE_CUDA_ARCHITECTURES="75-virtual;80-virtual;86-real;89-real;120-real" \
            -DCMAKE_CUDA_FLAGS=-compress-mode=size \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ls -l build/bin/libggml-cuda.so

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          # CUDA runtime is not a given on user systems -- bundle it like the
          # Windows job bundles cudart/cublas DLLs.
          cp -P /usr/local/cuda/lib64/libcudart.so* release/build/bin/
          cp -P /usr/local/cuda/lib64/libcublas.so* release/build/bin/
          cp -P /usr/local/cuda/lib64/libcublasLt.so* release/build/bin/
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cd release
          zip -ry ../llama-turboquant-linux-x64-cuda-13.3.zip .
          tar -czf ../llama-turboquant-linux-x64-cuda-13.3.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-x64-cuda-13.3
          path: |
            llama-turboquant-linux-x64-cuda-13.3.zip
            llama-turboquant-linux-x64-cuda-13.3.tar.gz
          retention-days: 14

  # NVIDIA DGX Spark (GB10) and other arm64 + NVIDIA Linux boxes. Built on
  # the free GitHub arm64 runner -- no cross-compilation involved.
  linux-arm64-cuda-13-3:
    name: linux-arm64-cuda-13.3
    runs-on: ubuntu-24.04-arm

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-arm64-cuda-13.3
          evict-old-files: 1d

      # GCC 14: GGML_CPU_ALL_VARIANTS builds armv9.2+sme CPU variants, which
      # the 24.04 default GCC 13 cannot assemble. Same workaround as the
      # upstream arm64 job in build-cpu.yml.
      - name: Toolchain
        run: |
          sudo apt-get update -y
          sudo apt-get install -y build-essential gcc-14 g++-14
          echo "CC=gcc-14" >> "$GITHUB_ENV"
          echo "CXX=g++-14" >> "$GITHUB_ENV"

      - name: Install CUDA Toolkit
        # arm64-SBSA repo -- the one NVIDIA ships for DGX Spark / GH200-class
        # machines. The `arm64` repo of the same distro is Jetson/L4T and has
        # no CUDA 13 packages; `cccl` lost its `cuda-` prefix in 13.3.
        run: |
          wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb
          sudo dpkg -i cuda-keyring_1.1-1_all.deb
          sudo apt-get update -y
          sudo apt-get install -y \
            cuda-nvcc-13-3 cccl-13-3 cuda-cudart-dev-13-3 libcublas-dev-13-3 \
            cuda-cuobjdump-13-3
          echo "/usr/local/cuda/bin" >> "$GITHUB_PATH"

      - name: Build
        # SASS for GB10 / DGX Spark only (sm_121). PTX floor at Hopper
        # (90-virtual) so the other arm64 CUDA machines -- GH200 (90),
        # GB200 (100), Jetson Thor (110) -- JIT at first run instead of
        # doubling build time on a 4-core runner. Runner has no GPU: build
        # only, the backend is a dlopen'd libggml-cuda.so.
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_CUB_3DOT2=ON \
            -DCMAKE_CUDA_ARCHITECTURES="90-virtual;121-real" \
            -DCMAKE_CUDA_FLAGS=-compress-mode=size \
            -DCMAKE_CUDA_HOST_COMPILER=g++-14 \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ls -l build/bin/libggml-cuda.so
          # Catch a silently-empty arch list: the archive is worthless without
          # sm_121 SASS in it.
          cuobjdump --list-elf build/bin/libggml-cuda.so | grep -q 'sm_121'

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          # CUDA runtime is not a given on user systems -- bundle it like the
          # x64 job does. On aarch64 the libs live under targets/sbsa-linux.
          CUDA_LIB=/usr/local/cuda/lib64
          [ -d "$CUDA_LIB" ] || CUDA_LIB=/usr/local/cuda/targets/sbsa-linux/lib
          cp -P "$CUDA_LIB"/libcudart.so* release/build/bin/
          cp -P "$CUDA_LIB"/libcublas.so* release/build/bin/
          cp -P "$CUDA_LIB"/libcublasLt.so* release/build/bin/
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cd release
          zip -ry ../llama-turboquant-linux-arm64-cuda-13.3.zip .
          tar -czf ../llama-turboquant-linux-arm64-cuda-13.3.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-arm64-cuda-13.3
          path: |
            llama-turboquant-linux-arm64-cuda-13.3.zip
            llama-turboquant-linux-arm64-cuda-13.3.tar.gz
          retention-days: 14

  linux-x64-cpu:
    runs-on: ubuntu-22.04

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-x64-cpu
          evict-old-files: 1d

      - name: Dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y build-essential

      - name: Build
        # Pure CPU, no GPU backend. GGML_CPU_ALL_VARIANTS picks the best SIMD
        # level (SSE..AVX512) at runtime, so one binary runs on any x64 CPU
        # with zero GPU runtime dependencies.
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cd release
          zip -ry ../llama-turboquant-linux-x64-cpu.zip .
          tar -czf ../llama-turboquant-linux-x64-cpu.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-x64-cpu
          path: |
            llama-turboquant-linux-x64-cpu.zip
            llama-turboquant-linux-x64-cpu.tar.gz
          retention-days: 14

  linux-x64-cuda-12-4:
    name: linux-x64-cuda-12.4
    runs-on: ubuntu-22.04

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-x64-cuda-12.4
          evict-old-files: 1d

      - name: Install CUDA Toolkit
        run: |
          wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
          sudo dpkg -i cuda-keyring_1.1-1_all.deb
          sudo apt-get update -y
          sudo apt-get install -y build-essential \
            cuda-nvcc-12-4 cuda-cccl-12-4 cuda-cudart-dev-12-4 libcublas-dev-12-4
          echo "/usr/local/cuda/bin" >> "$GITHUB_PATH"

      - name: Build
        # CUDA 12.4 for consumer cards on OLDER drivers (12.4 works where 13.x
        # needs a newer driver -- the most-downloaded variant on Windows).
        # Wide old-card net: SASS for Turing/Ampere/Ada (75/86/89), PTX floor
        # at Pascal (61) for JIT. No sm_120 -- CUDA 12.4 predates Blackwell,
        # and RTX 50 needs the newer driver + the 13.3 build anyway.
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_CUB_3DOT2=ON \
            -DCMAKE_CUDA_ARCHITECTURES="61-virtual;75-real;86-real;89-real" \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ls -l build/bin/libggml-cuda.so

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          cp -P /usr/local/cuda/lib64/libcudart.so* release/build/bin/
          cp -P /usr/local/cuda/lib64/libcublas.so* release/build/bin/
          cp -P /usr/local/cuda/lib64/libcublasLt.so* release/build/bin/
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cd release
          zip -ry ../llama-turboquant-linux-x64-cuda-12.4.zip .
          tar -czf ../llama-turboquant-linux-x64-cuda-12.4.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-x64-cuda-12.4
          path: |
            llama-turboquant-linux-x64-cuda-12.4.zip
            llama-turboquant-linux-x64-cuda-12.4.tar.gz
          retention-days: 14

  linux-x64-rocm:
    runs-on: ubuntu-22.04
    env:
      ROCM_VERSION: "7.2.1"

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      # ROCm SDK + build artifacts overrun the default runner disk.
      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-linux-x64-rocm
          evict-old-files: 1d

      - name: Install ROCm
        run: |
          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
            gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
          sudo tee /etc/apt/sources.list.d/rocm.list << EOF
          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main
          EOF
          sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
          Package: *
          Pin: release o=repo.radeon.com
          Pin-Priority: 600
          EOF
          sudo apt-get update -y
          sudo apt-get install -y build-essential cmake libssl-dev rocm-hip-sdk

      - name: Build
        # RDNA2-RDNA4: gfx1030, gfx1100/1101/1102, gfx1151 (Strix Halo),
        # gfx1200/1201. CDNA: gfx90a, gfx942 (MI200/MI300).
        #
        # Both families ship in one archive. --offload-compress makes that
        # affordable: measured on gfx942, libggml-hip.so is 439 MiB for RDNA
        # alone without it, 48 MiB for RDNA with it, and 65 MiB for RDNA+CDNA
        # with it. Without the flag the combined build is ~2 GB, which is why
        # CDNA used to be excluded.
        #
        # Runner has no AMD GPU: build only. libggml-hip.so is dlopen'd
        # thanks to GGML_BACKEND_DL.
        run: |
          export ROCM_PATH=/opt/rocm
          export PATH=$PATH:$ROCM_PATH/bin
          cmake -B build \
            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_HIP=ON \
            -DHIP_PLATFORM=amd \
            -DGPU_TARGETS="gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1200;gfx1201;gfx90a;gfx942" \
            -DCMAKE_HIP_FLAGS="--offload-compress" \
            -DGGML_HIP_ROCWMMA_FATTN=ON \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(nproc)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ls -l build/bin/libggml-hip.so

      - name: Prepare archive
        # Lean archive: binaries + libggml-*.so only. Do NOT bundle the ROCm
        # runtime — rocBLAS/hipBLASLt ship a Tensile kernel database for every
        # gfx arch that balloons the archive past 9 GB (and GitHub's 2 GB asset
        # limit). AMD users install the ROCm runtime system-wide, exactly like
        # upstream's ROCm builds; libggml-hip.so dlopen-links it at runtime.
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          find build/bin -name "*.so*" -exec cp -P {} release/build/bin/ \; 2>/dev/null || true
          cp LICENSE release/build/bin/ 2>/dev/null || true
          cat > release/build/bin/README-ROCm.txt << 'EOF'
          This build needs the AMD ROCm runtime installed on the system
          (https://rocm.docs.amd.com). Targets AMD RDNA2-RDNA4
          (gfx1030/1100/1101/1102/1151/1200/1201) and CDNA (gfx90a, gfx942 --
          MI200/MI300). Older GCN GPUs: use the Vulkan build.
          EOF
          cd release
          zip -ry ../llama-turboquant-linux-x64-rocm.zip .
          tar -czf ../llama-turboquant-linux-x64-rocm.tar.gz .

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-linux-x64-rocm
          path: |
            llama-turboquant-linux-x64-rocm.zip
            llama-turboquant-linux-x64-rocm.tar.gz
          retention-days: 14

  windows-x64:
    runs-on: windows-2022

    strategy:
      fail-fast: false
      matrix:
        include:
          - variant: cpu
            cmake_flags: ''
          - variant: vulkan
            cmake_flags: '-DGGML_VULKAN=ON'
          # Do not add flags here -- see the CUDA_EXTRA note in the Build step.
          - variant: cuda-12.4
            cuda: '12.4'
            cmake_flags: '-DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON'
          - variant: cuda-13.3
            cuda: '13.3'
            cmake_flags: '-DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON'

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-windows-x64-${{ matrix.variant }}
          # 500M (the action default) cannot hold a CUDA build, so the cache
          # thrashed and every run was effectively cold.
          max-size: ${{ startsWith(matrix.variant, 'cuda') && '2G' || '500M' }}
          evict-old-files: 7d

      - name: Install CUDA Toolkit
        if: ${{ startsWith(matrix.variant, 'cuda') }}
        uses: ./.github/actions/windows-setup-cuda
        with:
          cuda_version: ${{ matrix.cuda }}

      - name: Install Vulkan SDK
        if: ${{ matrix.variant == 'vulkan' }}
        shell: pwsh
        run: |
          curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
          & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
          Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
          Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"

      - name: Install Ninja
        run: choco install ninja -y

      # Defender scans every object nvcc writes, and the CUDA variants write
      # tens of thousands of them. Best-effort: never fail the build over it.
      - name: Exclude build tree from Defender
        shell: pwsh
        run: |
          try {
            Add-MpPreference -ExclusionPath "${{ github.workspace }}", "$env:RUNNER_TEMP"
            Add-MpPreference -ExclusionProcess "nvcc.exe", "cl.exe", "ninja.exe", "cicc.exe", "ptxas.exe", "cudafe++.exe"
          } catch {
            Write-Host "Defender exclusions unavailable: $_"
          }

      - name: Build
        shell: cmd
        # CUDA_EXTRA carries the pinned arch lists (the ggml default builds 8
        # targets on 12.4 and 7 on 13.3, including DGX Spark SASS on Windows)
        # and -compress-mode=size, which needs CTK >= 12.8 so 12.4 cannot have
        # it. These live here rather than in the matrix because a matrix value
        # becomes part of the job's display name, and those names are master's
        # required status checks -- renaming one leaves the required check
        # permanently "Expected" and the PR unmergeable.
        # Nothing that runs today loses support: only 90-virtual on 12.4
        # (Hopper still JITs from 80-virtual) and 121a-real on 13.3 are dropped.
        # NOTE: GGML_CUDA_CUB_3DOT2 can be dropped once CCCL 3.2 ships in the CTK used here.
        run: |
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          set CUDA_EXTRA=
          if "${{ matrix.variant }}"=="cuda-12.4" set CUDA_EXTRA=-DCMAKE_CUDA_ARCHITECTURES="50-virtual;61-virtual;70-virtual;75-virtual;80-virtual;86-real;89-real"
          if "${{ matrix.variant }}"=="cuda-13.3" set CUDA_EXTRA=-DCMAKE_CUDA_ARCHITECTURES="75-virtual;80-virtual;86-real;89-real;90-virtual;120a-real" -DCMAKE_CUDA_FLAGS=-compress-mode=size
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -DGGML_NATIVE=OFF ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_CPU_ALL_VARIANTS=ON ^
            -DGGML_RPC=OFF ^
            -DLLAMA_CURL=OFF ^
            -DLLAMA_OPENSSL=OFF ^
            -DLLAMA_BUILD_SERVER=ON ^
            -DLLAMA_BUILD_TOOLS=ON ^
            -DLLAMA_BUILD_TESTS=OFF ^
            -DLLAMA_BUILD_EXAMPLES=OFF ^
            ${{ matrix.cmake_flags }} %CUDA_EXTRA%
          set NINJA_JOBS=%NUMBER_OF_PROCESSORS%
          cmake --build build --config Release -j %NINJA_JOBS% -t ggml
          cmake --build build --config Release -j %NINJA_JOBS%

      - name: Verify build
        shell: pwsh
        run: |
          .\build\bin\Release\llama-server.exe --version 2>&1 | Write-Output
          .\build\bin\Release\llama-server.exe --help 2>&1 | Select-String -Pattern "cache-type-k" -Context 0,2

      - name: Bundle CUDA runtime DLLs
        if: ${{ startsWith(matrix.variant, 'cuda') }}
        shell: pwsh
        run: |
          $dst = ".\build\bin\Release"
          Get-ChildItem "$env:CUDA_PATH\bin" -Filter *.dll |
            Where-Object { $_.Name -match '^(cudart64|cublas64|cublasLt64)_.*\.dll$' } |
            ForEach-Object {
              Write-Output "Bundling $($_.Name)"
              Copy-Item $_.FullName -Destination $dst -Force
            }

      # Signed before staging, so the archive carries the signatures. Signing
      # runs only on push: PR runs may lack the secrets, and dev binaries are
      # for internal testing anyway.
      - name: Sign Windows binaries
        if: ${{ github.event_name != 'pull_request' }}
        uses: ./.github/actions/windows-code-sign
        with:
          path: build\bin\Release
          sm-api-key: ${{ secrets.SM_API_KEY }}
          sm-client-cert-b64: ${{ secrets.SM_CLIENT_CERT_FILE_B64 }}
          sm-client-cert-password: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}

      - name: Prepare archive
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Force -Path release\build\bin | Out-Null
          Copy-Item .\build\bin\Release\* release\build\bin\ -Recurse -Force
          Copy-Item .\LICENSE release\build\bin\ -ErrorAction SilentlyContinue
          $zip = "llama-turboquant-windows-x64-${{ matrix.variant }}.zip"
          # Compress-Archive is single-threaded and needs double-digit minutes
          # on the CUDA archives. 7-Zip ships with the runner image.
          if (Get-Command 7z -ErrorAction SilentlyContinue) {
            Push-Location release
            7z a -tzip -mx=5 -mmt=on "..\$zip" build | Out-Null
            Pop-Location
          } else {
            Compress-Archive -Path release\build -DestinationPath $zip -Force
          }

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-windows-x64-${{ matrix.variant }}
          path: llama-turboquant-windows-x64-${{ matrix.variant }}.zip
          retention-days: 14

  macos-arm64:
    runs-on: macos-latest

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: turboquant-macos-arm64
          evict-old-files: 1d

      # Signing runs only on push: PR runs may lack the secrets, and dev
      # binaries are for internal testing anyway.
      - name: Import code signing certificate
        if: ${{ github.event_name != 'pull_request' }}
        env:
          MACOS_CERTIFICATE_P12: ${{ secrets.MACOS_CERTIFICATE_P12 }}
          MACOS_CERTIFICATE_PASSWORD: ${{ secrets.MACOS_CERTIFICATE_PASSWORD }}
          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
        run: |
          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db

          echo -n "$MACOS_CERTIFICATE_P12" | base64 --decode -o $CERTIFICATE_PATH

          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

          security import $CERTIFICATE_PATH -P "$MACOS_CERTIFICATE_PASSWORD" \
            -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
          security set-key-partition-list -S apple-tool:,apple: \
            -k "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security list-keychain -d user -s $KEYCHAIN_PATH

          echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> "$GITHUB_ENV"

          IDENTITY=$(security find-identity -v -p codesigning $KEYCHAIN_PATH | head -1 | grep -o '".*"' | tr -d '"')
          echo "CODESIGN_IDENTITY=$IDENTITY" >> "$GITHUB_ENV"

      - name: Build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_SHARED_LIBS=OFF \
            -DLLAMA_CURL=OFF \
            -DLLAMA_OPENSSL=OFF \
            -DGGML_METAL=ON \
            -DGGML_METAL_USE_BF16=ON \
            -DGGML_METAL_EMBED_LIBRARY=ON \
            -DLLAMA_BUILD_SERVER=ON \
            -DLLAMA_BUILD_TOOLS=ON \
            -DLLAMA_BUILD_TESTS=OFF \
            -DLLAMA_BUILD_EXAMPLES=OFF
          cmake --build build --config Release -j $(sysctl -n hw.ncpu)

      - name: Verify build
        run: |
          ./build/bin/llama-server --version 2>&1 || true
          ./build/bin/llama-server --help 2>&1 | grep -A2 "cache-type-k" || true
          if otool -L build/bin/llama-server | grep -vE '/usr/lib|/System|llama-server' | grep '\.dylib'; then
            echo "ERROR: Found non-system dynamic dependency!"
            exit 1
          fi

      - name: Sign binaries
        if: ${{ github.event_name != 'pull_request' }}
        run: |
          for bin in build/bin/llama-server build/bin/llama-cli build/bin/llama-bench build/bin/llama-perplexity build/bin/llama-quantize; do
            if [ -f "$bin" ]; then
              codesign --force --options runtime --timestamp \
                --entitlements .github/entitlements.plist \
                --sign "$CODESIGN_IDENTITY" "$bin"
              codesign --verify --verbose "$bin"
            fi
          done

      # Dev builds are signed but NOT notarized (saves ~10 min of Apple API
      # per push). Testers: xattr -dr com.apple.quarantine <dir> after unzip.
      # The stable release workflow does full notarization.

      - name: Prepare archive
        run: |
          mkdir -p release/build/bin
          cp build/bin/llama-server release/build/bin/
          cp build/bin/llama-cli release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-bench release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-perplexity release/build/bin/ 2>/dev/null || true
          cp build/bin/llama-quantize release/build/bin/ 2>/dev/null || true
          cd release
          zip -r ../llama-turboquant-macos-arm64.zip .
          tar -czf ../llama-turboquant-macos-arm64.tar.gz .

      - name: Clean up keychain
        if: ${{ always() && github.event_name != 'pull_request' }}
        run: |
          security delete-keychain $KEYCHAIN_PATH 2>/dev/null || true

      - name: Upload archive
        uses: actions/upload-artifact@v4
        with:
          name: archive-macos-arm64
          path: |
            llama-turboquant-macos-arm64.zip
            llama-turboquant-macos-arm64.tar.gz
          retention-days: 14

  # Republishes the rolling `dev-latest` prerelease with whatever archives the
  # build jobs produced. Runs even when some jobs fail (if: always()) so a
  # broken backend never blocks the others from shipping to testers — the
  # release notes call out what is missing.
  publish-dev-latest:
    needs: [linux-x64-cpu, linux-x64-vulkan, linux-x64-cuda-12-4, linux-x64-cuda-13-3, linux-arm64-cuda-13-3, linux-x64-rocm, windows-x64, macos-arm64]
    if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/dev' }}
    runs-on: ubuntu-22.04
    permissions:
      contents: write

    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Download archives
        uses: actions/download-artifact@v4
        with:
          pattern: archive-*
          path: archives
          merge-multiple: true

      - name: Recreate dev-latest release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          SHORT_SHA=$(git rev-parse --short HEAD)
          DATE=$(date -u +%Y-%m-%d)
          ls -lh archives/

          EXPECTED="linux-x64-cpu linux-x64-vulkan linux-x64-cuda-12.4 linux-x64-cuda-13.3 linux-arm64-cuda-13.3 linux-x64-rocm windows-x64-cpu windows-x64-vulkan windows-x64-cuda-12.4 windows-x64-cuda-13.3 macos-arm64"
          MISSING=""
          for b in $EXPECTED; do
            ls archives/ | grep -q "llama-turboquant-${b}\." || MISSING="$MISSING $b"
          done

          NOTES="Rolling dev build from \`dev\` at commit \`${SHORT_SHA}\` (${DATE}).

          **Staging channel — not for production.** Every push to \`dev\` overwrites this release.

          macOS binaries are signed but not notarized; after unpacking run \`xattr -dr com.apple.quarantine build/\`. Stable releases are fully notarized."
          if [ -n "$MISSING" ]; then
            NOTES="$NOTES

          ⚠️ **Missing backends in this build:**${MISSING} — see the failed jobs of run ${{ github.run_id }}."
          fi

          gh release delete dev-latest --cleanup-tag --yes || true
          sleep 5
          gh release create dev-latest \
            --prerelease \
            --target "${{ github.sha }}" \
            --title "Dev latest (${SHORT_SHA}, ${DATE})" \
            --notes "$NOTES" \
            archives/*