Spaces:
Build error
Fix HF Spaces build timeout: limit mamba_ssm to sm_89 only
Browse filesPrevious fix (MAX_JOBS=1) prevented OOMKill but turned an already long
build into one exceeding HF Spaces' 60-minute build cap, because
mamba_ssm's setup.py hardcodes 8+ GPU architectures (sm_53..sm_90) and
serial compilation across all of them is too slow.
Real fix: patch mamba_ssm's setup.py at build time to compile only for
sm_89 (L40S / Ada / RTX 40-series, the actual target). With only one
architecture to build, MAX_JOBS=2 + NVCC_THREADS=2 is safe again.
Estimated build time drops from >60 min (timeout) to ~10-12 min.
- scripts/patch_mamba_ssm.py: line-based, idempotent setup.py patcher
with sentinel marker. Replaces cc_flag.append() calls with `pass` so
enclosing `if:` blocks remain syntactically valid.
- Dockerfile: pip download mamba_ssm sdist, run patcher, then install
from the patched source tree with --no-build-isolation.
Bump MAX_JOBS=2, NVCC_THREADS=2 (safe with single-arch build).
Also includes drift cleanup:
- app_gradio_baseline.py: add HfFolder shim + Gradio 4.44.x schema-bug
fix (mirrors app.py); bind to 127.0.0.1 by default on Windows so the
V1 screenshot can be reproduced locally without share=True.
- assets/gradio_v1_{reject,accept_png}.png: paper figure source images
showing gr.Image rejecting .nii.gz vs. accepting a normal PNG.
- .gitignore: ignore local venv directories (.v1/, .venv/, venv/).
- .gitignore +3 -0
- Dockerfile +20 -7
- app_gradio_baseline.py +60 -1
- assets/gradio_v1_accept_png.png +0 -0
- assets/gradio_v1_reject.png +0 -0
- scripts/patch_mamba_ssm.py +104 -0
|
@@ -12,6 +12,9 @@ v3_pred.nii.gz
|
|
| 12 |
__pycache__/
|
| 13 |
*.pyc
|
| 14 |
*.pyo
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
# Research paper / LaTeX build artifacts (kept local only)
|
| 17 |
Paper.tex
|
|
|
|
| 12 |
__pycache__/
|
| 13 |
*.pyc
|
| 14 |
*.pyo
|
| 15 |
+
.v1/
|
| 16 |
+
.venv/
|
| 17 |
+
venv/
|
| 18 |
|
| 19 |
# Research paper / LaTeX build artifacts (kept local only)
|
| 20 |
Paper.tex
|
|
@@ -12,12 +12,13 @@ ENV TORCH_CUDA_ARCH_LIST=8.9
|
|
| 12 |
ENV CUDA_HOME=/usr/local/cuda
|
| 13 |
ENV MPLCONFIGDIR=/tmp/matplotlib
|
| 14 |
|
| 15 |
-
# Cap CUDA-extension build parallelism
|
| 16 |
-
#
|
| 17 |
-
#
|
| 18 |
-
#
|
| 19 |
-
|
| 20 |
-
ENV
|
|
|
|
| 21 |
|
| 22 |
RUN apt-get update && apt-get install -y \
|
| 23 |
python3.10 \
|
|
@@ -48,7 +49,19 @@ RUN pip install --no-cache-dir -r requirements.txt
|
|
| 48 |
|
| 49 |
COPY . .
|
| 50 |
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
RUN python - <<'PY'
|
| 54 |
import torch, sys
|
|
|
|
| 12 |
ENV CUDA_HOME=/usr/local/cuda
|
| 13 |
ENV MPLCONFIGDIR=/tmp/matplotlib
|
| 14 |
|
| 15 |
+
# Cap CUDA-extension build parallelism. mamba_ssm hardcodes 8+ GPU arches
|
| 16 |
+
# in setup.py (sm_53..sm_90) which makes the build both memory-heavy AND
|
| 17 |
+
# wall-clock heavy on HF Spaces (~16 GB RAM, ~60 min build cap).
|
| 18 |
+
# We further patch mamba_ssm below to compile for only sm_89 (L40S target),
|
| 19 |
+
# so MAX_JOBS=2 is now safe and ~6x faster than the all-arch serial build.
|
| 20 |
+
ENV MAX_JOBS=2
|
| 21 |
+
ENV NVCC_THREADS=2
|
| 22 |
|
| 23 |
RUN apt-get update && apt-get install -y \
|
| 24 |
python3.10 \
|
|
|
|
| 49 |
|
| 50 |
COPY . .
|
| 51 |
|
| 52 |
+
# Download mamba-ssm source, patch its hardcoded cc_flag list to only build
|
| 53 |
+
# for sm_89 (L40S / Ada / RTX 40-series), then install. This avoids HF
|
| 54 |
+
# Spaces 60-minute build timeouts when compiling for all 8+ archs serially.
|
| 55 |
+
COPY scripts/patch_mamba_ssm.py /tmp/patch_mamba_ssm.py
|
| 56 |
+
RUN set -eux; \
|
| 57 |
+
pip download "mamba-ssm>=2.2.2" --no-deps --no-binary=:all: -d /tmp/mamba-src; \
|
| 58 |
+
cd /tmp/mamba-src; \
|
| 59 |
+
SDIST=$(ls mamba_ssm*.tar.gz mamba-ssm*.tar.gz 2>/dev/null | head -1); \
|
| 60 |
+
tar -xzf "$SDIST"; \
|
| 61 |
+
SRCDIR=$(tar -tzf "$SDIST" | head -1 | sed 's:/.*::'); \
|
| 62 |
+
cd "$SRCDIR"; \
|
| 63 |
+
python /tmp/patch_mamba_ssm.py setup.py; \
|
| 64 |
+
pip install . --no-build-isolation -v
|
| 65 |
|
| 66 |
RUN python - <<'PY'
|
| 67 |
import torch, sys
|
|
@@ -47,9 +47,64 @@ from pathlib import Path
|
|
| 47 |
import numpy as np
|
| 48 |
import torch
|
| 49 |
import nibabel as nib
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
import gradio as gr
|
| 51 |
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
_model_cache: dict = {}
|
| 54 |
|
| 55 |
|
|
@@ -356,8 +411,12 @@ def main() -> None:
|
|
| 356 |
else:
|
| 357 |
demo = build_v3_demo()
|
| 358 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
demo.launch(
|
| 360 |
-
server_name=
|
| 361 |
server_port=args.port,
|
| 362 |
share=args.share,
|
| 363 |
)
|
|
|
|
| 47 |
import numpy as np
|
| 48 |
import torch
|
| 49 |
import nibabel as nib
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# ---------------------------------------------------------------------------
|
| 53 |
+
# Compatibility shims (mirror app.py).
|
| 54 |
+
# Must run BEFORE `import gradio as gr` for the HfFolder shim, and AFTER for
|
| 55 |
+
# the schema bug fix.
|
| 56 |
+
# ---------------------------------------------------------------------------
|
| 57 |
+
def _patch_hf_folder():
|
| 58 |
+
"""Gradio 4.44.x imports HfFolder from huggingface_hub, which was removed
|
| 59 |
+
in huggingface_hub 1.0+. Inject a minimal shim."""
|
| 60 |
+
import huggingface_hub as hh
|
| 61 |
+
if getattr(hh, "HfFolder", None) is not None:
|
| 62 |
+
return
|
| 63 |
+
try:
|
| 64 |
+
from huggingface_hub import get_token
|
| 65 |
+
except ImportError:
|
| 66 |
+
get_token = lambda: None # noqa: E731
|
| 67 |
+
|
| 68 |
+
class HfFolder:
|
| 69 |
+
@staticmethod
|
| 70 |
+
def get_token():
|
| 71 |
+
return get_token()
|
| 72 |
+
|
| 73 |
+
hh.HfFolder = HfFolder
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
_patch_hf_folder()
|
| 77 |
+
|
| 78 |
import gradio as gr
|
| 79 |
|
| 80 |
|
| 81 |
+
def _fix_gradio_schema_bug():
|
| 82 |
+
"""Patch gradio_client.utils.get_type to handle boolean schemas
|
| 83 |
+
(Gradio 4.44.x crashes on additionalProperties: True)."""
|
| 84 |
+
try:
|
| 85 |
+
import gradio_client.utils as gu
|
| 86 |
+
if not hasattr(gu, "get_type"):
|
| 87 |
+
return
|
| 88 |
+
original = gu.get_type
|
| 89 |
+
|
| 90 |
+
def patched(schema):
|
| 91 |
+
if isinstance(schema, bool):
|
| 92 |
+
return "Any"
|
| 93 |
+
if isinstance(schema, dict):
|
| 94 |
+
if schema.get("additionalProperties") is True:
|
| 95 |
+
schema["additionalProperties"] = {}
|
| 96 |
+
elif schema.get("additionalProperties") is False:
|
| 97 |
+
schema.pop("additionalProperties", None)
|
| 98 |
+
return original(schema)
|
| 99 |
+
|
| 100 |
+
gu.get_type = patched
|
| 101 |
+
except Exception:
|
| 102 |
+
pass
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
_fix_gradio_schema_bug()
|
| 106 |
+
|
| 107 |
+
|
| 108 |
_model_cache: dict = {}
|
| 109 |
|
| 110 |
|
|
|
|
| 411 |
else:
|
| 412 |
demo = build_v3_demo()
|
| 413 |
|
| 414 |
+
server_name = os.environ.get(
|
| 415 |
+
"GRADIO_SERVER_NAME",
|
| 416 |
+
"127.0.0.1" if os.name == "nt" else "0.0.0.0",
|
| 417 |
+
)
|
| 418 |
demo.launch(
|
| 419 |
+
server_name=server_name,
|
| 420 |
server_port=args.port,
|
| 421 |
share=args.share,
|
| 422 |
)
|
|
|
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Patch mamba-ssm's setup.py to compile CUDA kernels for only sm_89.
|
| 2 |
+
|
| 3 |
+
Mamba-ssm's upstream setup.py builds for 8+ GPU architectures (sm_53,
|
| 4 |
+
sm_60, sm_70, sm_75, sm_80, sm_86, sm_89, sm_90). On the HF Spaces
|
| 5 |
+
build VM (~16 GB RAM, ~60 min wall-clock cap) compiling all of them
|
| 6 |
+
either OOMKills or hits the build timeout. This patch reduces the list
|
| 7 |
+
to only sm_89 (the L40S target the production VoxPixel and bench
|
| 8 |
+
Spaces both run on).
|
| 9 |
+
|
| 10 |
+
Strategy (line-based, robust to upstream formatting changes):
|
| 11 |
+
|
| 12 |
+
1. Find the line ``cc_flag = []``.
|
| 13 |
+
2. Replace it with a list literal pinned to sm_89.
|
| 14 |
+
3. Comment out every subsequent line containing
|
| 15 |
+
``cc_flag.append(`` (these would otherwise re-inject the other
|
| 16 |
+
architectures, including via conditional ``if`` blocks).
|
| 17 |
+
|
| 18 |
+
Idempotent: the patch leaves a sentinel marker and is a no-op on
|
| 19 |
+
already-patched files.
|
| 20 |
+
|
| 21 |
+
Usage:
|
| 22 |
+
python scripts/patch_mamba_ssm.py path/to/setup.py
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import re
|
| 26 |
+
import sys
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
SENTINEL = "# PATCHED: cc_flag limited to sm_89 by patch_mamba_ssm.py"
|
| 31 |
+
TARGET_ARCH = "89"
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _leading_indent(line: str) -> str:
|
| 35 |
+
return line[: len(line) - len(line.lstrip())]
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def patch(path: Path) -> bool:
|
| 39 |
+
src = path.read_text()
|
| 40 |
+
if SENTINEL in src:
|
| 41 |
+
print(f"{path}: already patched (sentinel present), skipping.")
|
| 42 |
+
return False
|
| 43 |
+
if "cc_flag" not in src:
|
| 44 |
+
raise SystemExit(
|
| 45 |
+
f"ERROR: 'cc_flag' not found in {path}. "
|
| 46 |
+
"Upstream mamba-ssm setup.py may have changed."
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
init_pattern = re.compile(r"^cc_flag\s*=\s*\[\]\s*$")
|
| 50 |
+
out_lines: list[str] = []
|
| 51 |
+
init_patched = False
|
| 52 |
+
appends_neutralized = 0
|
| 53 |
+
|
| 54 |
+
for line in src.splitlines(keepends=True):
|
| 55 |
+
stripped = line.strip()
|
| 56 |
+
|
| 57 |
+
if not init_patched and init_pattern.match(stripped):
|
| 58 |
+
indent = _leading_indent(line)
|
| 59 |
+
out_lines.append(f"{indent}{SENTINEL}\n")
|
| 60 |
+
out_lines.append(
|
| 61 |
+
f'{indent}cc_flag = ["-gencode", '
|
| 62 |
+
f'"arch=compute_{TARGET_ARCH},code=sm_{TARGET_ARCH}"]\n'
|
| 63 |
+
)
|
| 64 |
+
init_patched = True
|
| 65 |
+
continue
|
| 66 |
+
|
| 67 |
+
if init_patched and "cc_flag.append(" in stripped:
|
| 68 |
+
# Replace with `pass` (not a comment) so any enclosing
|
| 69 |
+
# `if:` / `else:` block keeps a syntactically valid body.
|
| 70 |
+
indent = _leading_indent(line)
|
| 71 |
+
out_lines.append(
|
| 72 |
+
f"{indent}pass # (was: {stripped})\n"
|
| 73 |
+
)
|
| 74 |
+
appends_neutralized += 1
|
| 75 |
+
continue
|
| 76 |
+
|
| 77 |
+
out_lines.append(line)
|
| 78 |
+
|
| 79 |
+
if not init_patched:
|
| 80 |
+
raise SystemExit(
|
| 81 |
+
f"ERROR: line 'cc_flag = []' not found in {path}. "
|
| 82 |
+
"Upstream mamba-ssm setup.py format may have changed."
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
path.write_text("".join(out_lines))
|
| 86 |
+
print(
|
| 87 |
+
f"{path}: patched. cc_flag pinned to sm_{TARGET_ARCH}; "
|
| 88 |
+
f"{appends_neutralized} subsequent cc_flag.append calls neutralized."
|
| 89 |
+
)
|
| 90 |
+
return True
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def main() -> None:
|
| 94 |
+
if len(sys.argv) != 2:
|
| 95 |
+
print("usage: patch_mamba_ssm.py <path/to/setup.py>", file=sys.stderr)
|
| 96 |
+
sys.exit(2)
|
| 97 |
+
path = Path(sys.argv[1])
|
| 98 |
+
if not path.is_file():
|
| 99 |
+
raise SystemExit(f"ERROR: {path} not found.")
|
| 100 |
+
patch(path)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
if __name__ == "__main__":
|
| 104 |
+
main()
|