"""alephllm-chat — talk to Beatrix. Serves EVERY shipped Beatrix craft the training repo holds, each with its own config, its own byte format, and its own arm library: mini-beatrix-1 112.5M · plain-text chat tags · the arm library (chat, turn-end, and behavior arms). An arm is NEVER attached to a core it was not trained on (metadata-verified per selection), a checkpoint with no chat arm NEVER defaults to one, and adapter shapes are read from each anchor's own tensors rather than assumed. mini-beatrix-2s 237.1M · FULL SPLAT (a governed multi-constellation aleph read in every block) · the 0.8.0 SPECIALS format: structural tokens live on the invalid-UTF-8 bytes, so the frame is unforgeable by content. Arms are read from the repo for both crafts — 2s has none yet. Crafts are discovered from the repo at boot: a craft with no manifest or no checkpoint is skipped, and one that starts shipping appears on its own. Off is bit-exact: the untouched base model. Standard ZeroGPU Space shape: models loaded at import, @spaces.GPU on the generator, demo.launch() at the end. The one framework patch is the gradio.utils lock/event shim below (documented at its site). Code: https://github.com/AbstractEyes/alephllm """ import asyncio import json import os import re import sys import time import uuid from datetime import datetime, timezone from pathlib import Path import gradio as gr import gradio.utils import numpy as np import spaces import torch # gradio.utils.safe_get_lock / safe_get_stop_event call get_running_loop(), # which always raises during setup (nothing is running yet), so each call # builds a throwaway event loop just to construct a Lock/Event and drops # the previous one — measured on 6.23.1: 9 loops created, 7 orphaned per # boot. Each orphan produces one # ValueError: Invalid file descriptor: -1 # when the collector finalizes it (the socketpair can be freed before the # loop object, so __del__ calls fileno() on a closed socket). # On Python 3.10+ Lock()/Event() bind lazily to whichever loop first awaits # them, so no loop is needed at construction. This REMOVES loop creation — # it starts no loop, thread, process or coroutine of its own. Measured # after: 1 loop, 0 orphans. gradio.queueing binds the NAMES rather than the # module, so every holder is swept. _orig_lock = gradio.utils.safe_get_lock _orig_stop = gradio.utils.safe_get_stop_event for _mod in list(sys.modules.values()): if getattr(_mod, "safe_get_lock", None) is _orig_lock: _mod.safe_get_lock = lambda: asyncio.Lock() if getattr(_mod, "safe_get_stop_event", None) is _orig_stop: _mod.safe_get_stop_event = lambda: asyncio.Event() from huggingface_hub import HfApi, hf_hub_download from safetensors.torch import load_file import amoe from amoe.core.adapter import AdapterSpec, RelayPatchwork from amoe.io.checkpoint import load_anchor from amoe.runtime.attach import _per_block_state from geolip.alephllm.presets import AlephLMConfig from geolip.alephllm.model.alephlm import AlephLM from geolip.alephllm.amoe_bridge import CHAT_HEADER, USER_TAG, ASSISTANT_TAG from geolip.alephllm.data import special_tokens as SP REPO = "AbstractPhil/alephllm-mini-beatrix-training" CORE_ONLY = "" # arm-picker value for "no arm attached" CORE_LABEL = "Core only — raw base model (bit-exact)" MAX_NEW_CAP = 2048 # ZeroGPU kills the call at 120s. Generation stops itself at this mark # so the reply survives (and says why it stopped) instead of the request # being killed mid-stream; the rest is headroom for prefill and the # checkpoint/arm switch that may precede it. GEN_BUDGET_S = 95.0 # Craft table. `fmt` selects the byte format the prompts are built in: # "v1tags" plain-text transcript tags (CHAT_HEADER/User:/Beatrix:) — # the shipped v1 arms' training frame, unchanged. # "specials" the 0.8.0 control plane: SYS/USER/MODEL/END on the # invalid-UTF-8 bytes. Encoded text can never contain one, # so the frame cannot be spoofed by anything typed here. # The DEFAULT craft is 2s: its mission is complete (16.101B bytes, # 61,422 steps) and its anneal taught the conversation frame, so the # core chats with no arm attached — the caveats this file used to carry # for it are gated on manifest state (frame_trained) and switched # themselves off when anneal_mix went done. v1 stays one click away as # the detach exhibit: a core that does NOT chat plus a 3.2M arm that # makes it chat, removable bit-for-bit. Both crafts READ the arm index: # 2s has no arms today, and the day one is trained on a 2s core it # appears in the picker with no code change. CRAFTS = [ {"name": "mini-beatrix-1", "fmt": "v1tags", "wants_arms": True}, {"name": "mini-beatrix-2s", "fmt": "specials", "wants_arms": True}, ] DEFAULT_CRAFT = "mini-beatrix-2s" # the completed mission leads # --------------------------------------------------------------- logging # The mounted bucket IS the store. Rows are appended to a file under # /data — no token, no HfApi, no scheduler, no uploader, no thread. The # only writes this app makes are plain appends inside the mount. DATA_DIR = Path(os.environ.get("BEATRIX_DATA", "/data")) HISTORY_DIR = DATA_DIR / "chat-history" _boot_id = uuid.uuid4().hex[:8] _log_on = False if DATA_DIR.is_dir() and os.environ.get("BEATRIX_NO_LOG") != "1": # The bucket is MOUNTED by the platform — never try to create /data # itself (the app runs as a non-root user, so that raises # PermissionError and reads like a failure when the real state is # simply "no bucket attached"). Only the subdirectory is ours. try: HISTORY_DIR.mkdir(parents=True, exist_ok=True) _log_on = True except OSError as e: print(f"[boot] {DATA_DIR} mounted but read-only ({type(e).__name__})" " — conversation logging off", flush=True) else: print(f"[boot] no bucket mounted at {DATA_DIR} — conversation logging off", flush=True) LOG_NOTE = ( "🌐 **Public research log — read before typing.** Everything entered " "here (prompts, chats, sampling settings, and Beatrix's replies) is " "written as JSON into this Space's persistent storage, tagged with " "an anonymous per-visit session id and the checkpoint that actually " "answered, and is **periodically published** to the public " "[alephllm-chat-history](https://huggingface.co/datasets/AbstractPhil/" "alephllm-chat-history) dataset. No accounts, names, IP addresses or " "device data are collected — but anything you type is intended to " "become public research data, so **do not enter personal " "information**." if _log_on else "📴 Conversation logging is **off** — nothing you type here is recorded.") def _log(kind, session, payload, prov): """One append into the mounted bucket. Called from inside the gradio event handlers, so it runs in gradio's own worker — no thread, no network, nothing to sync.""" if not _log_on: return try: row = dict(payload, kind=kind, session=session, ts=datetime.now(timezone.utc).isoformat(), **prov) day = datetime.now(timezone.utc).strftime("%Y-%m-%d") with open(HISTORY_DIR / f"sessions-{day}-{_boot_id}.jsonl", "a", encoding="utf-8", errors="replace") as f: f.write(json.dumps(row, ensure_ascii=False) + "\n") except Exception: # logging must never affect a conversation pass def _session(request): import hashlib try: raw = str(request.session_hash) except Exception: raw = uuid.uuid4().hex return hashlib.sha256(raw.encode()).hexdigest()[:16] def _text(content): """gradio hands chat content back as a list of content blocks.""" if isinstance(content, str): return content if isinstance(content, list): return "".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text") return str(content) def _clean_turn(s): """A user message is data, not transcript syntax.""" s = " ".join(str(s).split()) return s.replace(USER_TAG.strip(), "User -").replace(ASSISTANT_TAG, "Beatrix -") def _finish_reply(text, stops): """Trim a trailing partial stop tag (budget ran out mid-tag) and any replacement char from a truncated multi-byte tail.""" text = text.rstrip("�") for s in stops: for k in range(len(s), 0, -1): if text.endswith(s[:k]): text = text[:-k] break return text.strip() # ----------------------------------------------------------- model load def _step_of(path, pat): m = re.search(pat, path) return int(m.group(1)) if m else None # The v1 default (Phil's pick, 2026-08-27): the NEWEST core — 88,508 / # 26.10B bytes — with the turn-end arm that gives it a turn boundary. # That core has no CHAT arm, so the default arm is named explicitly here # rather than guessed; the "no chat arm" law still holds everywhere # (nothing is silently promoted to stand in for one), and the panel says # what is attached. # # The older pre-anneal exhibit pair stays one click away and is still the # cleanest demonstration of the detach claim: the 2026-08-15 anneal mix # carried chat-formatted SODA + identity texture, so 58,664-and-later # cores chat BARE and their "core only" mode demonstrates nothing, while # step 51,882 + its chat arm is the honest contrast (a core that does not # chat, and a 3.2M arm that makes it chat, bit-exact removable). EXHIBIT_STEP = {"mini-beatrix-1": 88508} # always OFFERED (never defaulted): checkpoints that are worth reaching # even when they fall outside the newest-N window. 2s ends its mission # as a BEFORE/AFTER pair across the chat anneal — 57,607 continues # documents, 61,422 answers — and that contrast is the clearest thing # the run produced, so the earlier half stays one click away. ALWAYS_OFFER = {"mini-beatrix-2s": [57607]} # default arm for the craft's DEFAULT step only — scoped so it can never # displace another checkpoint's own chat arm. day1/poly is a BEHAVIOR arm # on a single-turn Q&A frame: Chat therefore sends one question at a time # (no history), and auto-decode drops it to greedy on attach, which is # the setting its own metadata asks for. Both facts are stated in the # status panel rather than left for the visitor to discover. DEFAULT_ARM_NAME = {"mini-beatrix-1": "poly"} N_LISTED = 12 # newest N checkpoints offered in the dropdown def list_cores(files, craft): """{step: core_path} for every shipped checkpoint of one craft. Numeric step compare — substring matching would let '@step2000' match '@step20000'. Archived runs live under /archive/… and are excluded by the prefix, as are the fp8 exports.""" return {s: f for f in files if f.startswith(f"{craft}/checkpoints/step_") and f.endswith(".safetensors") and "/fp8/" not in f for s in [_step_of(f, r"step_(\d+)\.safetensors$")] if s} def load_arm_index(files, craft): """Every arm the repo holds for one craft, grouped by the core it was trained on. Reads arms/index.json (name, behavior, adapter spec, size — built from each anchor's own metadata and tensor shapes). Without it the Space would have to download 29 anchors to learn what they are; if it is missing we still LIST the arms from their filenames and read each spec when it is actually selected.""" idx_path = f"{craft}/arms/index.json" rows = [] if idx_path in files: try: rows = json.load(open(hf_hub_download(REPO, idx_path), encoding="utf-8"))["arms"] except Exception as e: print(f"[boot] arm index unreadable ({type(e).__name__})" " — falling back to filenames", flush=True) rows = [] if not rows: for f in files: if not (f.startswith(f"{craft}/arms/") and f.endswith(".anchor.pt")): continue s = _step_of(f, r"@step(\d+)\.anchor\.pt$") if not s: continue rel = f[len(f"{craft}/arms/"):] stem = rel.split("/")[-1].split("@")[0] rows.append({"path": f, "step": s, "name": stem, "group": rel.split("/")[0] if "/" in rel else "", "kind": "chat" if stem.startswith("chat") else ("turn-end" if stem.startswith("stop") else "behavior"), "behavior": "", "spec": None, "sites": None, "params": None}) by_step = {} for r in rows: by_step.setdefault(int(r["step"]), []).append(r) for s in by_step: # chat first, then turn-end order = {"chat": 0, "turn-end": 1, "behavior": 2} by_step[s].sort(key=lambda r: (order.get(r["kind"], 3), r.get("group", ""), r["name"])) return by_step CHAT_STOPS = ("\nUser:", "\nBeatrix:", "User:", "Beatrix:") FRAME_TAG = {"chat": "chat transcript", "qa": "single-turn Q&A", "raw": "raw text (no frame)"} def _tmpl(arm): """An arm's TEMPLATE is its tokenizer on a byte-native model: the frame its prompt was built with and the bytes that end a turn. They differ per family (transcript vs single-turn Q&A vs raw lines; "User:" vs newline-pair vs NUL), so serving every arm through one hardcoded chat format silently mis-runs most of them.""" t = dict((arm or {}).get("template") or {}) t.setdefault("frame", "chat") t.setdefault("stops", list(CHAT_STOPS)) t.setdefault("chat_ok", True) t.setdefault("turn_end", "next User: tag") return t def _arm_title(a): who = f"{a['group']}/{a['name']}" if a.get("group") else a["name"] t = _tmpl(a) bits = [who, FRAME_TAG.get(t["frame"], t["frame"])] if t.get("status") == "refuted": bits.append("⚠ refuted control") if a.get("behavior"): b = a["behavior"] bits.append(b if len(b) <= 46 else b[:43] + "…") if a.get("params"): bits.append(f"{a['params']/1e6:.1f}M") return " · ".join(bits) def _verify_anchor(local_arm, craft, step): """amoe's strict= check is inert for AlephLM (no model.config), so provenance is asserted here from the anchor's own metadata.""" try: got = load_anchor(local_arm).meta.get("base_model_id") except Exception: return False want = f"alephllm/{craft}@step{step}" if got and got != want: print(f"[boot] refusing arm: trained on {got}, core is {want}", flush=True) return False return True def _spec_of(ck): """Read the adapter geometry from the anchor's OWN tensors. The arm library holds more than one shape (n_slots 16 and 32, hidden 178 and 256), and amoe builds its modules from a spec before loading state — assuming the default would raise on every wide arm.""" cb, proj = ck.adapters.get("0.addr.codebook"), ck.adapters.get( "0.proj.weight") cons = ck.adapters.get("0.consume.0.weight") if cb is None or proj is None or cons is None: return AdapterSpec() d = int(cb.shape[1]) return AdapterSpec(n_slots=int(proj.shape[0] // d), K=int(cb.shape[0]), D=d, hidden=int(cons.shape[0])) class Craft: """One served lineage: its config, its checkpoints, its arms, its model, and the byte format its prompts are built in.""" def __init__(self, spec, files): self.name = spec["name"] self.fmt = spec["fmt"] self.cores = list_cores(files, self.name) if not self.cores: raise FileNotFoundError(f"{self.name}: no shipped checkpoints") manifest = json.load(open( hf_hub_download(REPO, f"{self.name}/manifest.json"), encoding="utf-8")) self.cfg = AlephLMConfig.from_dict(manifest["model_config"]) self.ckpt_meta = {c["step"]: c for c in manifest.get("checkpoints", []) if c.get("kind") == "safetensors"} # val_bpb is CARRIED FORWARD in the manifest: a checkpoint saved # between evals records the last eval's number as its own. Map # each value back to the first step that reported it, so the # panel can say which step actually measured it instead of # attributing someone else's eval to this checkpoint. self.bpb_at = {} first_seen = {} for s in sorted(self.ckpt_meta): v = self.ckpt_meta[s].get("val_bpb") if v is None: continue self.bpb_at[s] = first_seen.setdefault(round(float(v), 12), s) self.phases = manifest.get("phases", []) self.tokens_seen = manifest.get("tokens_seen", 0) # Has the craft been through the phase that TEACHES its chat # frame? Read from the manifest, never assumed — so this flips # by itself the day the anneal phase completes. # WHICH PHASE a checkpoint belongs to, derived from the # manifest's cumulative tokens_done — the checkpoint records # carry no phase field, but every phase reports what it # consumed, so the boundaries are exact arithmetic rather than # a hardcoded step. This is what separates the two final cores: # 2s ends anneal_nochat at 15,101,329,408 bytes (step 57,607) # and then spends 1,000,079,360 more on anneal_mix (61,422). # The pre-chat core continues documents; only the post-chat one # has ever seen a conversation frame. # NOT a cumulative sum in manifest order — the phase list is not # chronological (the curriculum phases are listed after the # anneals but ran before them), and summing it labelled every # checkpoint "chat-taught". The anneals are the LAST two phases # of the mission, so count BACK from the total: subtract # anneal_mix to find where the chat anneal began, then # anneal_nochat for the anneal pair's start. Self-checking — # the derived boundary lands exactly on the 57,607 checkpoint. _ph = {p.get("name"): p for p in manifest.get("phases", [])} def _done(name): p = _ph.get(name) or {} return int(p.get("tokens_done") or 0) if \ p.get("status") == "done" else 0 _mix, _nochat = _done("anneal_mix"), _done("anneal_nochat") self._chat_from = (self.tokens_seen - _mix) if _mix else None self._anneal_from = ((self._chat_from - _nochat) if self._chat_from is not None and _nochat else None) self.frame_trained = any( ph.get("name") == "anneal_mix" and ph.get("status") == "done" for ph in self.phases) self.active_phase = next((ph.get("name") for ph in self.phases if ph.get("status") == "active"), None) self.arms = load_arm_index(files, self.name) if spec["wants_arms"] \ else {} self._chat_ok = { # per CHECKPOINT, not craft s: bool(self.frame_trained and self._chat_from is not None and int(m.get("tokens") or 0) > self._chat_from) for s, m in self.ckpt_meta.items()} self.arm_by_path = {a["path"]: a for arms in self.arms.values() for a in arms} all_steps = sorted(self.cores, reverse=True) self.steps = all_steps[:N_LISTED] for s in sorted(self.arms, reverse=True): # any core with arms if s in self.cores and s not in self.steps: # stays reachable self.steps.append(s) exhibit = EXHIBIT_STEP.get(self.name) if exhibit in self.cores and exhibit not in self.steps: self.steps.append(exhibit) # always offered for s in ALWAYS_OFFER.get(self.name, []): # the before/after if s in self.cores and s not in self.steps: # pair stays self.steps.append(s) # reachable self.steps.sort(reverse=True) self.step = exhibit if exhibit in self.cores else self.steps[0] # ONE read of the core: this used to be deserialized twice (once # for load_state_dict, once to enumerate keys for the keymap # below) — 450MB for v1 and 948MB for 2s, per craft, per cold # boot, on a Space with a history of hanging in APP_STARTING. sd = load_file(hf_hub_download(REPO, self.cores[self.step])) self.model = AlephLM(self.cfg) self.model.load_state_dict(sd) self.model.eval() self.core_params = self.model.param_count() # attach ONCE so the adapter sites exist and ZeroGPU packs them; # the template is THIS checkpoint's own default arm when it has # one, so the common path needs no rebuild. Selecting an arm of # another geometry rebuilds the site modules for that request. self.wrappers, spec_now = [], None tmpl = self.default_arm(self.step) or next(iter(self.arm_by_path), None) if tmpl: local = hf_hub_download(REPO, tmpl) if _verify_anchor(local, self.name, self.arm_by_path[tmpl]["step"]): ck = load_anchor(local) spec_now = _spec_of(ck) amoe.attach(self.model, ck, spec=spec_now, strict=False) self.wrappers = [b for b in self.model.blocks if hasattr(b, "adapter")] print(f"[boot] {self.name}: adapter sites " f"{len(self.wrappers)} (template " f"{tmpl.split('/')[-1]}, n_slots={spec_now.n_slots}, " f"hidden={spec_now.hidden})", flush=True) # core-key -> live-key map: attach wraps blocks, so a core # checkpoint's "blocks.N.*" lands at "blocks.N.block.*". self.live = dict(self.model.named_parameters()) self.live.update(dict(self.model.named_buffers())) self.keymap = {} for k in sd: if k in self.live: self.keymap[k] = k else: alt = re.sub(r"^blocks\.(\d+)\.", r"blocks.\1.block.", k) if alt in self.live: self.keymap[k] = alt self.model.to("cuda") # ZeroGPU packs here (standard pattern) self.loaded = {"step": self.step, "arm_path": tmpl if self.wrappers else None, "spec": spec_now} print(f"[boot] {self.name}: {len(self.cores)} cores, " f"{len(self.arm_by_path)} arms, {self.core_params/1e6:.1f}M " f"core, ctx {self.cfg.context}, fmt {self.fmt}", flush=True) # ---------------------------------------------------------- helpers def arm_choices(self, step): """Core first — a checkpoint is always runnable bare.""" return ([(CORE_LABEL, CORE_ONLY)] + [(_arm_title(a), a["path"]) for a in self.arms.get(int(step), [])]) def default_arm(self, step): """The chat arm for THIS core if one exists — never another arm standing in for it, and never a chat default on a checkpoint that has no chat arm (those open bare, the honest state). One exception, and only on the craft's own default step: a named arm may be preferred there (DEFAULT_ARM_NAME). It is still a real arm on that exact core, listed with its own kind and frame, so nothing is misrepresented as a chat arm.""" step = int(step) arms = self.arms.get(step, []) want = DEFAULT_ARM_NAME.get(self.name) if want and step == getattr(self, "step", None): for a in arms: if a["name"] == want: return a["path"] for a in arms: if a["kind"] == "chat": return a["path"] return CORE_ONLY def chats(self, step): """Has THIS core been through the chat anneal? A craft-level flag was wrong: the two final checkpoints straddle that phase, so the pre-chat core would have claimed a conversation frame it has never seen (measured: 57,607 continues documents, 61,422 answers as Beatrix).""" if self.fmt != "specials": return True # v1 cores chat via plain-text tags return self._chat_ok.get(int(step), self.frame_trained) def phase_of(self, step): """Where a checkpoint sits relative to the closing anneals. Only the anneal boundaries are derivable from the manifest, so only those are named: everything earlier is reported as the pretraining body rather than guessed at a phase it may not belong to.""" if self.fmt != "specials" or self._chat_from is None: return None tok = int((self.ckpt_meta.get(int(step)) or {}).get("tokens") or 0) if tok > self._chat_from: return "anneal_mix — the chat anneal" if self._anneal_from is not None and tok > self._anneal_from: return "anneal_nochat — the pre-chat anneal" return "pretraining body (before the anneals)" def label(self, step): arms = self.arms.get(step, []) meta = self.ckpt_meta.get(step, {}) bits = [f"step {step:,}"] if meta.get("tokens"): bits.append(f"{meta['tokens']/1e9:.2f}B") if meta.get("val_bpb") and self.bpb_at.get(step) == step: bits.append(f"bpb {meta['val_bpb']:.3f}") # measured HERE # the two final cores are a BEFORE/AFTER pair across the chat # anneal and look identical in a step-and-bytes label; say which # side of it each one sits on if self.fmt == "specials" and self._chat_from is not None: bits.append("chat-taught" if self.chats(step) else "pre-chat (documents only)") n_chat = sum(a["kind"] == "chat" for a in arms) if not arms: bits.append("core only") elif n_chat: bits.append(f"chat arm ✓ · {len(arms)} arm" f"{'s' if len(arms) != 1 else ''}") else: bits.append(f"{len(arms)} arm{'s' if len(arms) != 1 else ''}" " · no chat arm") return " · ".join(bits) def title(self): # .1f, not .0f: every other surface on the page says 112.5M n = f"{self.core_params/1e6:.1f}M" arms = (f"{len(self.arm_by_path)} arms" if self.arm_by_path else "no arms yet") if self.fmt == "specials": state = ("in training" if self.active_phase else "training complete") return (f"{self.name} · {n} · full splat · specials format " f"· {arms} · {state}") return f"{self.name} · {n} · {arms} · plain-text tags" # The models are built ONCE and their tensors are packed by ZeroGPU at # import (the documented pattern). Switching checkpoints therefore copies # new VALUES into those same tensors rather than building a second model # — the packed storage is preserved and no lazy .to('cuda') is needed. files = HfApi().list_repo_files(REPO) STATE, CRAFT_NAMES = {}, [] for _spec in CRAFTS: try: STATE[_spec["name"]] = Craft(_spec, files) CRAFT_NAMES.append(_spec["name"]) except Exception as e: # a craft that has not shipped yet, or a print(f"[boot] skipping {_spec['name']}: " # repo hiccup, must f"{type(e).__name__}: {e}", flush=True) # not sink the app if not STATE: raise SystemExit("[boot] no craft could be loaded from " + REPO) CRAFT = DEFAULT_CRAFT if DEFAULT_CRAFT in STATE else CRAFT_NAMES[0] STEP = STATE[CRAFT].step def S(craft): """The Craft record for a (possibly stale) dropdown value.""" return STATE.get(craft) or STATE[CRAFT] def _step(c, v): """Coerce a step that came from the UI. Both dropdowns are allow_custom_value (their choice lists are rebuilt per craft, and gradio validates submissions against BOOT-TIME choices), which in 6.23.1 also makes them filterable — i.e. free-text comboboxes. A typed value therefore arrives verbatim, and a bare int() would raise inside every handler, with the crash re-raised out of the logging finally: block on top. Anything unparseable or unknown falls back to the craft's own default step.""" try: s = int(v) except (TypeError, ValueError): return c.step return s if s in c.cores else c.step def _resolve_arm(c, arm_path, step): """The ONE place an arm path becomes an arm. An arm belongs to exactly one core, so a path left over from another checkpoint (or another craft) resolves to None — and every consumer must agree, or the prompt gets built in the frame of an arm that was refused.""" arm = c.arm_by_path.get(arm_path or "") return arm if (arm and arm["step"] == int(step)) else None def _select(craft, step, arm_path): """Make the live model BE (craft, step, arm). arm_path is "" for core-only. Runs inside the ZeroGPU worker, where each call starts from the parent's packed state, so a non-default selection re-applies per request (weights come from the local hub cache — no re-download). Returns a provenance dict.""" c = S(craft) step = _step(c, step) # stale/typed values fall back t0 = time.time() if c.loaded["step"] != step: sd = load_file(hf_hub_download(REPO, c.cores[step])) with torch.no_grad(): for k, v in sd.items(): tgt = c.live.get(c.keymap.get(k, k)) if tgt is not None and tgt.shape == v.shape: tgt.copy_(v.to(tgt.device, tgt.dtype)) c.loaded["step"] = step # an arm belongs to exactly one core: a stale pick from another # checkpoint (or another craft) is dropped rather than force-fitted arm = _resolve_arm(c, arm_path, step) if arm is None: arm_path = CORE_ONLY arm_on = bool(c.wrappers) and bool(arm) if arm_on and c.loaded.get("arm_path") != arm_path: local = hf_hub_download(REPO, arm_path) if _verify_anchor(local, c.name, step): # provenance, every switch ck = load_anchor(local) spec = _spec_of(ck) dev = next(c.model.parameters()).device # INVALIDATE FIRST, COMMIT LAST. This state survives the # request (a ZeroGPU worker is reused, not forked per call), # so a partial load must never be recorded as a complete # one: if the loop below raises, the wrappers hold a mix of # two arms, and a loaded[arm_path] still naming the old arm # would make the next request skip the reload and serve the # chimera under the wrong provenance label. prev_spec = c.loaded["spec"] c.loaded["arm_path"] = None if (spec.n_slots, spec.K, spec.D, spec.hidden) != ( prev_spec.n_slots, prev_spec.K, prev_spec.D, prev_spec.hidden): # different adapter geometry: rebuild the site modules # for this request rather than forcing mismatched state d_hidden = int(ck.adapters["0.proj.weight"].shape[1]) for w in c.wrappers: w.adapter = RelayPatchwork(d_hidden, spec).to(dev) c.loaded["spec"] = spec with torch.no_grad(): for i, w in enumerate(c.wrappers): st = _per_block_state(ck.adapters, i) if st: w.adapter.load_state_dict( {k: v.to(next(w.adapter.parameters()).device) for k, v in st.items()}) c.loaded["arm_path"] = arm_path # commit only now else: arm_on, arm = False, None for w in c.wrappers: w.enabled = arm_on return {"craft": c.name, "checkpoint_step": step, "arm": (arm_path.split("/")[-1] if arm_on else None), "arm_kind": (arm["kind"] if arm_on else "core-only"), "switch_s": round(time.time() - t0, 2)} def status_md(craft, step, arm_path=CORE_ONLY): c = S(craft) step = _step(c, step) arms = c.arms.get(step, []) meta = c.ckpt_meta.get(step, {}) newest = max(c.cores) arm = _resolve_arm(c, arm_path, step) # compact by design: this panel sits above the tabs, and the Space # is embedded in a non-scrolling iframe where every extra line # pushes the controls further out of reach if c.fmt == "specials" and not arm: arm_line = ("**Format** specials — `⟦DOC⟧` ends a document, " "`⟦SYS⟧ ⟦USER⟧ ⟦MODEL⟧ ⟦END⟧` frame a conversation. " "They live on bytes UTF-8 can never produce, so " "nothing you type can forge them.") elif not c.wrappers: arm_line = "**Arm:** adapter sites unavailable — core only" elif arm: sp = arm.get("spec") or {} t = _tmpl(arm) dt, dp, _ = _decode_of(arm) arm_line = ( f"**Arm** `{arm['name']}`" + (f" ({arm['group']})" if arm.get("group") else "") + (f" · {arm['params']/1e6:.1f}M" if arm.get("params") else "") + (f" · {arm['sites']} sites" if arm.get("sites") else "") + (f" · n{sp['n_slots']}/h{sp['hidden']}" if sp else "") + " · trained on THIS core" + f" \n{FRAME_TAG.get(t['frame'], t['frame'])} · ends on " f"{t['turn_end']} · T={dt:g}/top-p {dp:g}" + (f" · *{arm['behavior']}*" if arm.get("behavior") else "") + (f" \n⚠️ {t['note']}" if t.get("note") else "")) else: arm_line = ("**Arm** none attached — raw core" + (f" ({len(arms)} available here)" if arms else "")) return " \n".join(filter(None, [ f"**{c.name}** · **step {step:,}**" + (f" · {meta['tokens']/1e9:.2f}B bytes" if meta.get("tokens") else "") # say WHERE the bpb was measured: the manifest carries the last # eval's value forward onto checkpoints saved between evals + (f" · bpb {meta['val_bpb']:.3f}" + ("" if c.bpb_at.get(step) == step else f" (measured at step {c.bpb_at[step]:,})") if meta.get("val_bpb") else "") + f" · {c.core_params/1e6:.1f}M core · ctx {c.cfg.context}" + (f" · newest is {newest:,}" if step != newest else ""), arm_line, # which side of the chat anneal THIS core sits on — the mission # ends in a before/after pair and they are not interchangeable (f"**Phase:** `{c.phase_of(step)}` · " + ("conversation frame taught here" if c.chats(step) else "**before the chat anneal** — this core has only ever read " "documents") if c.fmt == "specials" and c.phase_of(step) else ""), # a craft still in pretraining has never seen its chat frame: # say so rather than letting the Chat tab imply otherwise ("⚠️ this craft is **still pretraining** (phase " f"`{c.active_phase}`, {c.tokens_seen/1e9:.2f}B bytes) and has not " "reached the anneal that teaches its conversation frame — Chat " "will read as off-distribution. **Completion** is the honest " "view of what she can do today." if c.fmt == "specials" and not c.frame_trained else ""), # the craft finished its chat anneal but THIS checkpoint predates # it: the pre-chat baseline continues documents rather than # taking turns (measured, not inferred) ("⚠️ this is the **pre-chat baseline** — saved just before the " "anneal that taught the conversation frame. It continues text " "instead of answering; the newest checkpoint is the one that " "chats. Kept selectable because the pair is the cleanest " "before/after the mission produced." if c.fmt == "specials" and c.frame_trained and not c.chats(step) else ""), ("ℹ️ this checkpoint is post-anneal: its core chats on its own, so " "*Core only* no longer shows an unconditioned model" if c.name == "mini-beatrix-1" and step >= 58664 else ""), # a turn-end arm IS doing conversational work (it supplies the # turn boundary), so it gets its own line rather than the # blanket "not a conversational adapter" warning ("ℹ️ no chat arm exists for this core — `" + arm["name"] + "` is a " "turn-end arm: it supplies the turn boundary, not the voice" if arm and arm["kind"] == "turn-end" and not any(a["kind"] == "chat" for a in arms) else ""), ("⚠️ no chat arm exists for this checkpoint — the Chat tab runs the " "core (or whichever behavior arm you pick), which is not a " "conversational adapter" if arms and not any(a["kind"] == "chat" for a in arms) and (arm is None or arm["kind"] not in ("chat", "turn-end")) else ""), ("ℹ️ this arm was trained on a **single-turn** frame, so Chat sends " "one question at a time (no history) — that is the frame it " "learned; the Completion tab is the plainer view" if arm and _tmpl(arm)["frame"] == "qa" else ""), ("⚠️ this is a **representation** arm with no turn convention: it " "was trained on raw lines, not conversations. Use Completion — " "Chat will not read as dialogue" if arm and not _tmpl(arm)["chat_ok"] else ""), # true of 2s post-mission: no arms exist for it, and none are # needed — its anneal taught the conversation frame, so the core # answers on its own. Say that rather than implying a gap. ("ℹ️ no arm has been trained on this lineage yet — and none is " "needed to talk to it: the conversation frame was taught to the " "core itself, in the anneal that closed its mission." if not c.arm_by_path and c.chats(step) else ""), ("ℹ️ no arm has been trained on this lineage yet — every reply " "here is the raw pretrained core." if not c.arm_by_path and not c.chats(step) else ""), ])) CORE_EX = [["Hello!"], ["Who are you?"], ["What can you do?"], ["How are you today?"], ["Tell me about books."]] def _decode_of(arm): d = (arm or {}).get("decode") or {} return (float(d.get("temperature", 0.7)), float(d.get("top_p", 0.95)), d.get("why", "")) def _examples_of(arm): ex = (arm or {}).get("examples") return [[e] for e in ex] if ex else CORE_EX def on_arm(craft, step, arm_path, auto_decode, fam_examples): """Selecting an arm re-tunes what the arm needs and shows what it knows: a task arm collapses to turn-end off-distribution and turns to punctuation under sampling (measured), so it is served greedy and offered its own training-frame prompts. Both automations are switchable — the checkboxes leave the controls alone when off.""" c = S(craft) arm = _resolve_arm(c, arm_path, _step(c, step)) t, p, _ = _decode_of(arm) return (status_md(craft, step, arm_path), gr.update(value=t) if auto_decode else gr.update(), gr.update(value=p) if auto_decode else gr.update(), gr.update(samples=_examples_of(arm)) if fam_examples else gr.update()) def on_checkpoint(craft, step, auto_decode, fam_examples): """Repopulate the arm picker for the newly selected core: its own arms only, defaulting to its chat arm if it has one and to the bare core if it does not.""" c = S(craft) step = _step(c, step) val = c.default_arm(step) # write the coerced step BACK: the dropdown is a free-text combobox # (allow_custom_value), so a typed or stale value would otherwise # keep displaying a checkpoint the app is not running. Gradio only # dispatches change when the value actually differs, so this # converges after one extra round trip. return (gr.update(value=step), gr.update(choices=c.arm_choices(step), value=val, interactive=bool(c.arm_by_path)), *on_arm(craft, step, val, auto_decode, fam_examples)) def on_craft(craft, auto_decode, fam_examples): """Switching craft repopulates everything below it: a different lineage has its own checkpoints, its own arms (or none), its own context length and its own byte format.""" c = S(craft) step = c.step val = c.default_arm(step) return (gr.update(choices=[(c.label(s), s) for s in c.steps], value=step), gr.update(choices=c.arm_choices(step), value=val, interactive=bool(c.arm_by_path)), *on_arm(c.name, step, val, auto_decode, fam_examples)) # ------------------------------------------------------------ generation def _encode(text): return np.frombuffer(text.encode("utf-8", errors="replace"), dtype=np.uint8).astype(np.int64) def _room(context, max_new): """Prompt budget: the window minus a reserved reply. ONE definition, shared by the prompt builders and by _fit — when they disagreed, the builder fit the frame to context-16 and _fit then cut it anyway, which is how a whole SYS+USER opener could vanish. The reserve TRACKS the request (it used to be capped at 256, so asking for 2,048 bytes after a long paste silently yielded 256) but never takes more than half the window — past that the prompt is being deleted to make room for an answer about nothing.""" return max(64, context - max(16, min(int(max_new), context // 2)) - 1) def _fit(ids, max_new, context, fmt="v1tags"): """Keep as much of the prompt as fits WITHOUT starving the reply. The reply budget is reserved first: trimming a long prompt to the last 16 bytes of the window used to leave max_new = 15, so a long paste silently truncated the answer as well as the prompt. A cut is then made on a boundary the format understands — a UTF-8 lead byte for plain text, and for the specials format a TURN OPENER (⟦USER⟧/⟦MODEL⟧/⟦SYS⟧). Cutting mid-frame there would hand the model a turn that no opener started: every special is >= 0xC0, so the UTF-8 continuation skip can never land on a frame boundary by itself.""" raw = np.asarray(ids, dtype=np.int64) room = _room(context, max_new) # keep room to answer if raw.size > room: cut = raw.size - room if fmt == "specials": openers = (SP.SYS, SP.USER, SP.MODEL) while cut < raw.size and int(raw[cut]) not in openers: cut += 1 if cut >= raw.size: # no opener in range: cut = raw.size - room # fall back to a plain while cut < raw.size and 0x80 <= int(raw[cut]) < 0xC0: cut += 1 # UTF-8-safe cut else: while cut < raw.size and 0x80 <= int(raw[cut]) < 0xC0: cut += 1 raw = raw[cut:] return raw, min(int(max_new), max(8, context - raw.size - 1)) def _render(out, fmt): """Bytes -> text. A specials craft renders its structural tokens VISIBLY (⟦DOC⟧, ⟦END⟧) instead of as replacement characters, so a stray frame token in a reply is legible rather than mojibake.""" if fmt == "specials": return SP.decode_visible(out) return bytes(out).decode("utf-8", errors="replace") def _rep_penalty(logits, produced, penalty, n=8, window=256): """Soft n-gram repetition penalty, sized for a BYTE vocabulary. The textbook CTRL penalty (damp every id already seen) is wrong here: with 256 byte values, ' ' and 'e' are "seen" within the first few bytes, so it would tax ordinary English rather than repetition. The failure these models actually show is PHRASE LOOPING ("I am a small model." over and over), so the penalty targets exactly that — if the last n-1 bytes already occurred in the recent window, the bytes that followed those occurrences are damped. penalty=1.0 is off; a large penalty approaches hard n-gram blocking. Scope is the REPLY only: penalising prompt bytes would punish the model for quoting the question back, which is not the disease. Lineage: Keskar et al. (CTRL) for the divide/multiply form, Paulus et al. for n-gram blocking; the byte-level windowing is the part that had to change. """ if penalty <= 1.0 or len(produced) < n: return logits win = produced[-window:] suffix = tuple(win[-(n - 1):]) nxt = {win[i + n - 1] for i in range(len(win) - (n - 1)) if tuple(win[i:i + n - 1]) == suffix} if not nxt: return logits out = logits.clone() row = out[:, -1] idx = torch.tensor(sorted(nxt), device=row.device, dtype=torch.long) v = row[:, idx] row[:, idx] = torch.where(v > 0, v / penalty, v * penalty) return out @spaces.GPU(duration=120) def _stream(craft, ids, max_new, temperature, top_p, use_cache, arm_path, ckpt_step, stop=(), stop_ids=(), guard=False, rep=1.0): """Yields (text, stats, prov) — prov is what actually RAN, resolved by _select, not what the dropdowns asked for. It travels back through the yield because it has to cross a process boundary: the caller cannot read state this body mutates. Runs inside a ZeroGPU worker PROCESS THAT IS REUSED across requests (spaces.zero pulls an idle registered worker and only forks a new one when none is live), and each worker serialises its own tasks. So every mutation here — the arm flags, the loaded bookkeeping, the in-place weight copies — survives into the next request that worker serves, and must be left consistent rather than merely correct for this call. No lock is needed (one worker never interleaves two tasks) and a threading.Lock would be actively unsafe under fork: one held by any parent thread at fork time is inherited locked with no owner, and the child would block forever.""" prov = _select(craft, ckpt_step, arm_path) c = S(craft) model = c.model dev = next(model.parameters()).device # The no-cache path recomputes the WHOLE context per byte, so on the # bigger craft it is minutes where the ZeroGPU budget is 120s. This # used to be a silent max_new = min(max_new, 96): the slider said 512 # and you got 96 with nothing said. Now the byte count is RESPECTED # and time is the limit — generation runs until the requested bytes # OR the budget, and if the budget ends it that is stated in the # stats line and in the reply itself. A short prompt uncached often # gets well past 96; a long one stops honestly instead of lying. asked = int(max_new) raw, max_new = _fit(ids, max_new, c.cfg.context, c.fmt) # the window can force a smaller reply than the slider asked for; # say so rather than letting the shortfall look like the model # simply stopping (the same class of silence as the old 96 clamp) ctx_capped = max_new < asked tids = torch.tensor([raw.tolist()], device=dev) stop_ids = set(int(s) for s in stop_ids) out, t0 = [], time.time() deadline, budget_hit = t0 + GEN_BUDGET_S, False label = (f"{prov['craft']} · step {prov['checkpoint_step']:,} · " + (f"arm `{prov['arm'].split('@')[0]}`" if prov["arm"] else "core only") + (f" · switched in {prov['switch_s']}s" if prov["switch_s"] > 0.05 else "")) with torch.no_grad(): cache, prefill_s = None, 0.0 if use_cache: with torch.autocast("cuda", dtype=torch.bfloat16, enabled=dev.type == "cuda"): logits, cache = model.prefill(tids) prefill_s = time.time() - t0 t1 = time.time() for _ in range(max_new): # time, not an arbitrary byte cap, is the real constraint: # end the reply while it can still be delivered and say so if time.time() > deadline: budget_hit = True break if use_cache: nxt = model._sample(_rep_penalty(logits, out, rep), temperature, top_p) else: with torch.autocast("cuda", dtype=torch.bfloat16, enabled=dev.type == "cuda"): fl, _ = model(tids[:, -c.cfg.context:]) nxt = model._sample(_rep_penalty(fl, out, rep), temperature, top_p) tids = torch.cat([tids, nxt], dim=1) nid = int(nxt.item()) bps = len(out) / max(time.time() - t1, 1e-6) stats = (f"{label} · kv-cache **on** · prefill " f"{prefill_s*1000:.0f} ms · {bps:.0f} bytes/s" if use_cache else f"{label} · kv-cache **off** (whole context " f"recomputed per byte) · {bps:.0f} bytes/s") if rep > 1.0: stats += f" · rep **{rep:g}**/8-byte" if ctx_capped: stats += (f" · context capped **{max_new}** of {asked} " f"(window {c.cfg.context})") # TOKEN-level stop (specials): the frame terminator is one # byte that UTF-8 can never produce, so it is caught here # exactly rather than by matching decoded text. if nid in stop_ids: yield _render(out, c.fmt), stats, prov return out.append(nid) if use_cache: with torch.autocast("cuda", dtype=torch.bfloat16, enabled=dev.type == "cuda"): logits = model.decode_step(nxt, cache) text = _render(out, c.fmt) cut = min((text.index(s) for s in stop if s in text), default=-1) if cut >= 0: yield text[:cut], stats, prov return # OPTIONAL guard (off by default): a long unbroken run of # punctuation is the measured off-distribution collapse, not # content. Off by default because masking it would hide a # real behaviour from anyone studying the arm. if guard and len(out) >= 32: tail = text[-24:] if tail and all(not (c_.isalnum() or c_.isspace()) for c_ in tail): yield (text[:-24].rstrip() + " …[degenerate run cut]", stats, prov) return yield text, stats, prov if budget_hit: # the requested byte count was honoured as far as the GPU # allowance allowed; the shortfall is stated, never hidden note = (f" …[stopped at {len(out)} of {max_new} bytes — " f"{GEN_BUDGET_S:.0f}s GPU budget" + ("" if use_cache else "; the KV cache is off, so " "every byte recomputes the whole context") + "]") yield _render(out, c.fmt) + note, stats + " · **budget**", prov def complete(craft, prompt, max_new, temperature, top_p, use_cache, arm_path, ckpt_step, guard, rep=1.0, request: gr.Request = None): if not prompt or not prompt.strip(): yield "(enter a prompt)", "" return c = S(craft) step_i = _step(c, ckpt_step) # completion honours the arm's turn-end too: a newline-pair arm # that is never stopped on "\n\n" reads as if it still rambles. The # arm is resolved with the SAME core check _select applies — a stale # pick otherwise sets stops that belong to a model that never ran. stops = tuple(s for s in _tmpl(_resolve_arm(c, arm_path, step_i))["stops"] if s not in CHAT_STOPS) # a specials craft ends a document with DOC — the natural terminator # for a continuation, and the gauge the training reports track stop_ids = (SP.DOC,) if c.fmt == "specials" else () text, done = "", False prov = {"craft": c.name, "checkpoint_step": step_i, "arm": None, "arm_kind": "unrun"} try: for text, stats, prov in _stream(craft, _encode(prompt), max_new, temperature, top_p, use_cache, arm_path, step_i, stop=stops, stop_ids=stop_ids, guard=bool(guard), rep=float(rep)): yield prompt + text, stats done = True finally: # the log records what RAN (prov, resolved by _select), never the # raw dropdown values: this bucket is a published research # dataset and the page above it promises enforced provenance _log("completion", _session(request), {"prompt": prompt, "output": text, "completed": done}, prov) def _qa_prompt(message): """The single-turn frame the capability arms were trained on: header, one question, assistant tag, trailing space. No history — these arms never saw a multi-turn transcript.""" return (CHAT_HEADER + USER_TAG + message + "\n" + ASSISTANT_TAG + " ") def _transcript(history, message, context): """Header first, then as many recent turns as fit.""" head, tail = CHAT_HEADER, f"{USER_TAG}{message}\n{ASSISTANT_TAG}" budget = context - 16 - len(head.encode()) - len(tail.encode()) turns = [] for t in reversed(history): who = USER_TAG if t["role"] == "user" else ASSISTANT_TAG + " " line = f"{who}{t['content']}\n" if len(line.encode()) > budget: break budget -= len(line.encode()) turns.append(line) return head + "".join(reversed(turns)) + tail def _specials_chat_ids(history, message, context, budget=None): """The 0.8.0 frame as token ids: SYS block, then USER/MODEL turns each closed by END, then an open MODEL tag for the reply. Content is UTF-8, so it can never contain a frame token — the structure is unforgeable by anything typed here. Fits by dropping the oldest turns whole and then, if a single message still overflows, by truncating THAT message's text from the front on a UTF-8 boundary (its tail carries the actual ask). The SYS block, the turn structure and the open MODEL tag always survive — which is the whole point of building ids here rather than letting a byte-level trim cut the frame apart.""" turns = [{"role": ("user" if t["role"] == "user" else "model"), "content": t["content"]} for t in history] turns.append({"role": "user", "content": message}) opener = np.array([SP.MODEL], dtype=np.int64) room = (context - 16) if budget is None else budget while True: ids = np.concatenate([SP.render_chat_ids(turns), opener]) if ids.size <= room: return ids if len(turns) > 1: turns = turns[1:] # drop the oldest turn, keep the frame continue # one turn, still too long: keep its tail, on a byte boundary over = ids.size - room raw = turns[0]["content"].encode("utf-8", errors="replace") cut = min(len(raw), over + 8) while cut < len(raw) and 0x80 <= raw[cut] < 0xC0: cut += 1 if cut >= len(raw): # nothing left to give turns[0]["content"] = "" return np.concatenate([SP.render_chat_ids(turns), opener]) turns[0]["content"] = "…" + raw[cut:].decode("utf-8", errors="replace") def chat(craft, message, history, max_new, temperature, top_p, use_cache, arm_path, ckpt_step, guard, rep=1.0, convo_id="", request: gr.Request = None): if not message or not message.strip(): yield list(history or []), "", "" return c = S(craft) step_i = _step(c, ckpt_step) # On a plain-text craft the tags ARE the syntax, so a message # containing "User:" has to be defanged. On a specials craft the # frame lives on bytes text cannot contain, so the message needs no # mangling — defanging it there would contradict the guarantee the # status panel makes and quietly edit what the visitor typed. clean = _clean_turn if c.fmt != "specials" else ( lambda s: " ".join(str(s).split())) message = clean(message) history = [{"role": t["role"], "content": clean(_text(t["content"]))} for t in (history or [])] tmpl = _tmpl(_resolve_arm(c, arm_path, step_i)) stops, stop_ids = (), () room = _room(c.cfg.context, max_new) if c.fmt == "specials": # the frame's own terminators, as ids: END closes the turn, DOC # means she started a new document instead of answering. Built # to the SAME budget _fit enforces, so _fit never has to cut a # frame it cannot see the structure of. ids = _specials_chat_ids(history, message, c.cfg.context, room) stop_ids = (SP.END, SP.DOC, SP.USER) else: # the SELECTED arm's own frame and turn-end bytes, not one # hardcoded chat format: a Q&A arm never saw a transcript, and a # newline-pair arm's terminator is invisible to the "User:" stops if tmpl["frame"] == "chat": ids = _encode(_transcript(history, message, room + 16)) else: ids = _encode(_qa_prompt(message)) # qa + raw: one turn only stops = tuple(dict.fromkeys( list(tmpl["stops"]) + ["\n" + USER_TAG.strip(), "\n" + ASSISTANT_TAG, USER_TAG.strip(), ASSISTANT_TAG])) history = history + [{"role": "user", "content": message}, {"role": "assistant", "content": ""}] # a model that ends the turn instantly is SAYING something — that the # prompt is off its distribution. A blank bubble reads as breakage, # so the reason is shown instead (measured: task arms return empty # at greedy on chat prompts, punctuation soup when sampled). if c.fmt == "specials" and not c.chats(step_i): empty = ("*(this checkpoint predates the anneal phase that " "teaches the conversation frame — it has only ever read " "documents. The Completion tab shows what it can do " "today.)*") elif tmpl["frame"] == "qa": empty = ("*(this arm ended the turn immediately — it was trained " "on single-turn task prompts, not conversation. Try one " "of its examples below, or switch arms.)*") elif not tmpl["chat_ok"]: empty = ("*(representation arm — it shapes internal states and " "has no turn behavior. Use the Completion tab.)*") else: empty = "*(empty reply)*" # A pretrained web-text core never returns EMPTY on a chat prompt — # it returns fluent, confident, unrelated prose. So the caveat has to # travel WITH the reply rather than only standing in for a blank one: # a visitor reading the bubble alone would otherwise take that prose # for an attempted answer. preface = ("*(pre-chat checkpoint — this core has not learned a " "conversation frame; this is document continuation, " "not an answer)*\n\n" if c.fmt == "specials" and not c.chats(step_i) else "") done = False prov = {"craft": c.name, "checkpoint_step": step_i, "arm": None, "arm_kind": "unrun"} try: for text, stats, prov in _stream(craft, ids, max_new, temperature, top_p, use_cache, arm_path, step_i, stop=stops, stop_ids=stop_ids, guard=bool(guard), rep=float(rep)): body = _finish_reply(text, stops) history[-1]["content"] = (preface + body) if body else empty yield history, "", stats done = True finally: # what RAN, not what the dropdowns said (see complete()) _log("chat", _session(request), {"messages": history, "completed": done, # first conversation of a session has no id yet; each Clear # mints one, so the dataset shows real conversation bounds "conversation": convo_id or "c0"}, prov) TITLE = "# Beatrix — byte-level AlephLLMs you can talk to" # The Space is embedded in an iframe with scrolling="no", sized to the # app's reported height: the OUTER page scrolls, and only while that # height stays in sync. A tall page therefore does not just look long — # it pushes the tabs and controls out of reach whenever the resize lags. # Everything explanatory lives in a closed accordion for that reason. DESCRIPTION = """ Two lineages of the same architecture, served side by side. Pick the **Craft** first — everything below it follows. **mini-beatrix-2s** — 237.1M parameters, **full splat**: the signed address read runs in *every* block (v1 used it in three), with a governor holding the anchors apart from birth. Its mission is **complete** — 16.1 billion bytes over 61,422 steps, closing with a two-phase anneal that taught the conversation frame, so this core **chats on its own**: no arm is required, and none have been trained on it yet. Its format is specials-native — SYS/USER/MODEL/END are single reserved bytes rather than typed text, so the turn frame cannot be forged by anything you type. **mini-beatrix-1** — 112.5M parameters, pretrained on 15.3 billion raw bytes, carrying a **library of detachable arms**: small adapters (3.2M–5.3M parameters) each trained on one exact frozen core. Pick a checkpoint, pick an arm, and switch it off mid-session to hear the raw base model underneath. **How she reads text.** Beatrix consumes raw UTF-8 bytes. Each position composes the byte with its two predecessors (a trigram embedding with a dedicated pad row), so "tokens" are learned inside the network rather than fixed by a vocabulary. **What's inside.** Pre-norm layers where routing uses *signed geometric addresses* — dispatch weights `sinh(u_k)/Σcosh(u_j)` over learned unit anchors, with no softmax-over-choices, no top-k, and inhibition (negative weights) as a first-class citizen. Each layer holds an anchored expert bank **born contributing exactly zero**; the aleph attention layers read in linear cost with a constant-size decode state. The banks *elected themselves* into load-bearing work: born at exactly zero, they cost **+1.73 bits per byte** to remove from 2s at step 8,320. The splat hubs are a different measurement — on 2s they are the **entire** attention (no softmax attention anywhere in the model), so their +2.04 is what the attention is worth, not an election. On v1, where three of sixteen layers are aleph beside thirteen softmax ones, the same ablations read +2.1 and +3.0. **The specials format (2s).** Thirteen structural tokens live on the byte values UTF-8 can *never* produce: `⟦DOC⟧` ends every document, `⟦SYS⟧ ⟦USER⟧ ⟦MODEL⟧ ⟦END⟧` frame a conversation. Because encoded text cannot contain them, nothing typed into this box can forge the frame — the guarantee is arithmetic, not convention. They render visibly here rather than as mojibake. **The arm library (v1).** Every adapter trained on a shipped core is selectable in the *Arm* dropdown — the chat arms, the turn-end arms, and the behavior arms from the experiment lines (subtraction, deduction chains, definitions, identity, distillation cells, and the byte-frame alloys that import BERT and T5 geometry). **Off is exact.** Choose *Core only* and the model is the pretrained base again, bit for bit — available for every checkpoint, with or without arms. **Provenance is enforced, not assumed.** An arm is only ever attached to the core it was trained on: the dropdown lists that core's arms alone, and the adapter's own metadata is re-verified at every switch. A checkpoint with **no chat arm never pretends to have one**. **What to expect.** They are small and early: conversational in shape, thin on knowledge, confidently wrong at times. Short exchanges suit them best; longer answers drift. *Training record:* [alephllm-mini-beatrix-training](https://huggingface.co/AbstractPhil/alephllm-mini-beatrix-training) · *code:* [github.com/AbstractEyes/alephllm](https://github.com/AbstractEyes/alephllm) · *adapters:* [amoe-lora](https://github.com/AbstractEyes/amoe-lora) """ # Keep the parent iframe's height in step with the content. Gradio's # own resizer fires on its events; a DOM observer covers the rest # (status text growing, examples swapping, a reply streaming in), which # is what used to leave the embedded page stuck until you clicked # something. RESIZE_JS = """ () => { const ping = () => window.dispatchEvent(new Event('resize')); let t = null; new MutationObserver(() => { clearTimeout(t); t = setTimeout(ping, 150); }) .observe(document.body, {childList: true, subtree: true, characterData: true}); ping(); } """ _C0 = STATE[CRAFT] with gr.Blocks(title="Beatrix — AlephLLM chat") as demo: gr.Markdown(TITLE) with gr.Accordion("About Beatrix — the two crafts, how she reads text, " "what the arms are, what to expect", open=False): gr.Markdown(DESCRIPTION) gr.Markdown(LOG_NOTE) status_box = gr.Markdown(status_md(CRAFT, STEP, _C0.default_arm(STEP))) with gr.Row(): craft_pick = gr.Dropdown( [(STATE[n].title(), n) for n in CRAFT_NAMES], value=CRAFT, label="Craft", scale=3, interactive=len(CRAFT_NAMES) > 1) ckpt = gr.Dropdown([(_C0.label(s), s) for s in _C0.steps], value=STEP, label="Checkpoint", scale=3, allow_custom_value=True) # allow_custom_value: the choice lists are repopulated per craft # and per checkpoint, but gradio validates a submitted value # against the component's BOOT-TIME choices — without this, # picking any arm (or step) from another craft is rejected # server-side ("not in the list of choices"). Values are # validated by this app instead: unknown or cross-core paths # fall back to core-only in _select. arm_pick = gr.Dropdown(_C0.arm_choices(STEP), value=_C0.default_arm(STEP), label="Arm (adapter)", scale=4, allow_custom_value=True, interactive=bool(_C0.arm_by_path)) with gr.Row(): max_new = gr.Slider(16, MAX_NEW_CAP, value=192, step=16, label="Max new bytes") # seed the decode sliders from the DEFAULT arm's own metadata: # auto-decode only fires on a change event, so a default arm that # asks for greedy was otherwise served sampled on first paint — # the exact setting its metadata says degenerates _dt, _dp, _ = _decode_of(_C0.arm_by_path.get(_C0.default_arm(STEP))) temperature = gr.Slider(0.0, 1.5, value=_dt, step=0.05, label="Temperature") top_p = gr.Slider(0.1, 1.0, value=_dp, step=0.05, label="Top-p") # byte-level: this damps whatever would CONTINUE a repeated # 8-byte phrase, not every byte already seen — on a 256-value # vocabulary the textbook form would just tax ordinary English rep = gr.Slider(1.0, 2.0, value=1.15, step=0.05, label="Repetition penalty", info="damps bytes that would repeat a recent " "8-byte phrase · 1.0 = off") use_cache = gr.Checkbox(value=True, label="KV cache") with gr.Accordion("Automation", open=False), gr.Row(): auto_decode = gr.Checkbox( value=True, label="Auto decode per arm", info="task arms switch to greedy on attach — they degenerate " "into punctuation when sampled off-distribution") fam_examples = gr.Checkbox( value=True, label="Arm-family examples", info="offer prompts from the selected arm's own training " "frames instead of chat openers") guard = gr.Checkbox( value=False, label="Cut degenerate runs", info="stop a reply once it collapses into unbroken " "punctuation (off by default — the collapse is real " "behaviour worth seeing)") stats_md = gr.Markdown("") with gr.Tab("Chat"): chatbot = gr.Chatbot(label="Beatrix", height=360) # lines/max_lines pinned: an unpinned 1-row textbox was being # flex-stretched to 373px, adding half a viewport of dead space with gr.Row(): msg = gr.Textbox(label="Message", lines=2, max_lines=6, placeholder="Say hello to Beatrix…", scale=8) # a multi-line textbox takes Enter as a newline, so without # this the only way to send was a modifier chord send = gr.Button("Send", variant="primary", scale=1, min_width=96) # halt a reply that is running long WITHOUT losing it: Stop # cancels the generator, so whatever had streamed stays in # the bubble (Clear is the one that also wipes it) stop_chat = gr.Button("Stop", variant="stop", scale=1, min_width=80) ex = gr.Examples( _examples_of(_C0.arm_by_path.get(_C0.default_arm(STEP))), inputs=msg, label="Examples for the selected arm") # a conversation id that CHANGES on Clear: the published log used # to key on the browser session alone, so turns from before and # after a Clear ran together as one conversation convo = gr.State("") _chat_in = [craft_pick, msg, chatbot, max_new, temperature, top_p, use_cache, arm_pick, ckpt, guard, rep, convo] _chat_out = [chatbot, msg, stats_md] _ev1 = msg.submit(chat, _chat_in, _chat_out) _ev2 = send.click(chat, _chat_in, _chat_out) # gr.ClearButton only wipes the display: a reply still streaming # keeps yielding the WHOLE history, so the cleared conversation # reappeared byte by byte. Clear must CANCEL the generation, and # it starts a new conversation rather than continuing the old one. gr.Button("Clear", variant="secondary").click( lambda: ([], "", uuid.uuid4().hex[:12], ""), None, [chatbot, msg, convo, stats_md], cancels=[_ev1, _ev2]) with gr.Tab("Completion"): # max_lines pinned: unpinned textboxes auto-grow to 20 rows in # 6.23.1, and a streaming continuation would push the tabs out # of reach inside the non-scrolling iframe prompt = gr.Textbox(label="Prompt", lines=4, max_lines=4, value="The history of mathematics begins") comp_out = gr.Textbox(label="Continuation (streams)", lines=12, max_lines=12) with gr.Row(): comp_go = gr.Button("Complete", variant="primary", scale=3) stop_comp = gr.Button("Stop", variant="stop", scale=1, min_width=80) _ev3 = comp_go.click( complete, [craft_pick, prompt, max_new, temperature, top_p, use_cache, arm_pick, ckpt, guard, rep], [comp_out, stats_md]) # ONE halt for every generator, wired after both tabs so a Stop on # either side reaches the run in flight. Cancelling closes the # generator at its yield: the bytes already streamed stay on screen, # and each caller's finally: still logs the turn (completed=False), # so a halted reply is recorded as halted rather than vanishing. for _b in (stop_chat, stop_comp): _b.click(lambda: "⏹ **stopped** — the bytes already generated " "are kept; press Send or Complete to continue", None, stats_md, cancels=[_ev1, _ev2, _ev3]) # craft drives the checkpoint list; the checkpoint drives the arm # list (only arms trained on THAT core, defaulting to its chat arm # or nothing); decode and examples then follow the ARM, unless their # toggles say otherwise craft_pick.change(on_craft, [craft_pick, auto_decode, fam_examples], [ckpt, arm_pick, status_box, temperature, top_p, ex.dataset]) ckpt.change(on_checkpoint, [craft_pick, ckpt, auto_decode, fam_examples], [ckpt, arm_pick, status_box, temperature, top_p, ex.dataset]) arm_pick.change(on_arm, [craft_pick, ckpt, arm_pick, auto_decode, fam_examples], [status_box, temperature, top_p, ex.dataset]) demo.load(js=RESIZE_JS) demo.launch()