Prajanya Gupta commited on
Commit
4daa56c
Β·
1 Parent(s): 1217b6a
Files changed (2) hide show
  1. app.py +375 -202
  2. requirements.txt +8 -9
app.py CHANGED
@@ -2,16 +2,20 @@ from __future__ import annotations
2
 
3
  import io
4
  import os
5
- import random
6
  import sys
 
7
  from pathlib import Path
8
- from typing import Dict, List
9
 
10
  import gradio as gr
 
 
11
  import matplotlib.pyplot as plt
 
12
  import numpy as np
13
  import pretty_midi
14
  import torch
 
15
  import torch.nn.functional as F
16
  from huggingface_hub import hf_hub_download
17
  from PIL import Image
@@ -24,262 +28,431 @@ if str(SRC_DIR) not in sys.path:
24
  from compound import AXIS_SIZES, N_AXES, SENTINELS, STEP_BOS, STEP_EOS, decode_compound
25
  from compound_model import CompoundGPT, CompoundGPTConfig, default_compound_config
26
 
27
- HF_REPO_ID = os.getenv("HF_REPO_ID", "prajanya23/bachgpt-midi")
28
- HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_HUB_TOKEN")
29
- TMP_MIDI_PATH = "/tmp/output.mid"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
 
31
- # If your uploaded filenames differ, set these in Space variables.
32
- V1_FILENAME = os.getenv("V1_CKPT_FILENAME", "")
33
- V2_FILENAME = os.getenv("V2_CKPT_FILENAME", "")
34
 
35
- CHECKPOINT_CANDIDATES: Dict[str, List[str]] = {
36
- "v1": [V1_FILENAME, "compound_v1_3m.pt", "compound_3m.pt", "v1_3m.pt"],
37
- "v2": [V2_FILENAME, "compound_best.pt", "compound_v2_25m.pt", "compound_25m.pt"],
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
 
 
 
 
 
40
 
41
- def _download_checkpoint(model_key: str) -> str:
42
- last_error: Exception | None = None
43
- for filename in CHECKPOINT_CANDIDATES[model_key]:
44
- if not filename:
45
- continue
46
- try:
47
- return hf_hub_download(
48
- repo_id=HF_REPO_ID,
49
- filename=filename,
50
- token=HF_TOKEN,
51
- )
52
- except Exception as exc: # pragma: no cover - defensive for remote failures
53
- last_error = exc
54
- raise RuntimeError(
55
- f"Unable to download {model_key} checkpoint from repo '{HF_REPO_ID}'. "
56
- f"Tried: {[f for f in CHECKPOINT_CANDIDATES[model_key] if f]}. "
57
- "If this is a private/gated repo, set HF_TOKEN (or HUGGINGFACE_HUB_TOKEN) "
58
- "with read access and optionally set HF_REPO_ID to the correct model repo. "
59
- f"Last error: {last_error}"
60
- )
61
 
 
 
 
62
 
63
- def _load_compound_model(ckpt_path: str) -> CompoundGPT:
64
- # CPU-only per requirement.
65
- ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=True)
66
- cfg = default_compound_config()
67
- raw_cfg = ckpt.get("config") if isinstance(ckpt, dict) else None
68
- if isinstance(raw_cfg, dict):
69
- for field in CompoundGPTConfig.__dataclass_fields__.keys():
70
- if field in raw_cfg:
71
- setattr(cfg, field, raw_cfg[field])
72
- model = CompoundGPT(cfg).to("cpu")
73
- state = ckpt.get("model_state_dict", ckpt)
74
- model.load_state_dict(state, strict=False)
75
- model.eval()
76
- return model
77
 
 
 
 
 
78
 
79
- def _sample_axis(logits: torch.Tensor, temperature: float, top_k: int) -> int:
80
- scaled = logits / temperature
81
- if top_k > 0 and top_k < scaled.numel():
82
- values, _ = torch.topk(scaled, top_k)
83
- cutoff = values[-1]
84
- scaled = torch.where(
85
- scaled < cutoff,
86
- torch.tensor(float("-inf"), device=scaled.device),
87
- scaled,
88
- )
89
- probs = F.softmax(scaled, dim=-1)
90
- return int(torch.multinomial(probs, num_samples=1).item())
91
 
 
 
 
92
 
93
- def _random_seed_step() -> List[int]:
94
- step = [random.randrange(size) for size in AXIS_SIZES]
95
- if step[0] == STEP_EOS:
96
- step[0] = STEP_BOS
97
- return step
98
 
 
99
 
100
- def _parse_seed_text(seed_text: str) -> List[List[int]]:
101
- # Format: one or more steps, separated by ';'
102
- # Each step: "a,b,c,d,e,f,g" (7 axes)
103
- text = seed_text.strip()
104
- if not text:
105
- return [_random_seed_step()]
 
 
106
 
107
- steps: List[List[int]] = []
108
- chunks = [chunk.strip() for chunk in text.split(";") if chunk.strip()]
109
- for chunk in chunks:
110
- parts = [p.strip() for p in chunk.split(",")]
111
- if len(parts) != N_AXES:
112
- continue
113
- try:
114
- step = [int(p) for p in parts]
115
- except ValueError:
116
- continue
117
- bounded = [
118
- max(0, min(value, AXIS_SIZES[idx] - 1))
119
- for idx, value in enumerate(step)
120
- ]
121
- steps.append(bounded)
 
 
 
 
122
 
123
- return steps if steps else [_random_seed_step()]
124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
 
126
  @torch.no_grad()
127
- def _generate_steps(
128
- model: CompoundGPT,
129
- seed_steps: List[List[int]],
130
- temperature: float,
131
- top_k: int,
132
- max_new_steps: int,
 
133
  ) -> List[List[int]]:
134
- generated = [list(s) for s in seed_steps]
135
- if not generated:
136
- bos = list(SENTINELS)
137
- bos[0] = STEP_BOS
138
- generated.append(bos)
139
-
140
- for _ in range(max_new_steps):
141
- step_ids = torch.tensor([generated], dtype=torch.long, device="cpu")
142
- if step_ids.size(1) > model.config.block_size:
 
 
143
  break
144
- position_ids = torch.arange(step_ids.size(1), dtype=torch.long).unsqueeze(0)
145
- logits_per_axis = model(idx=step_ids, position_ids=position_ids)
146
-
147
- next_step: List[int] = []
148
- for axis_logits in logits_per_axis:
149
- sampled = _sample_axis(
150
- logits=axis_logits[0, -1, :],
151
- temperature=temperature,
152
- top_k=top_k,
153
- )
154
- next_step.append(sampled)
 
 
 
 
 
 
 
 
 
155
 
156
  if next_step[0] == STEP_EOS:
157
- next_step = [STEP_EOS] + SENTINELS[1:]
158
- generated.append(next_step)
159
  break
160
-
161
  generated.append(next_step)
162
 
163
  return generated
164
 
165
 
166
- def _render_pianoroll(pm: pretty_midi.PrettyMIDI) -> Image.Image:
167
- notes = []
168
- for inst in pm.instruments:
169
- for note in inst.notes:
170
- notes.append((note.start, note.end, note.pitch, note.velocity))
 
 
 
 
 
 
 
171
 
172
- fig, ax = plt.subplots(figsize=(12, 4), dpi=120)
173
  if notes:
174
- for start, end, pitch, velocity in notes:
175
- width = max(0.01, end - start)
176
- color = plt.cm.magma(velocity / 127.0)
177
- ax.broken_barh([(start, width)], (pitch - 0.45, 0.9), facecolors=color)
178
- max_end = max(n[1] for n in notes)
179
- min_pitch = min(n[2] for n in notes) - 2
180
- max_pitch = max(n[2] for n in notes) + 2
181
- ax.set_xlim(0.0, max_end + 0.25)
182
- ax.set_ylim(min_pitch, max_pitch)
183
- else:
184
- ax.text(
185
- 0.5,
186
- 0.5,
187
- "No notes generated",
188
- ha="center",
189
- va="center",
190
- transform=ax.transAxes,
191
- )
192
 
193
- ax.set_title("Generated Piano Roll")
194
- ax.set_xlabel("Time (seconds)")
195
- ax.set_ylabel("Pitch (MIDI)")
196
- ax.grid(alpha=0.2)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
197
 
198
  buf = io.BytesIO()
199
- fig.tight_layout()
200
- fig.savefig(buf, format="png")
201
  plt.close(fig)
202
  buf.seek(0)
203
  return Image.open(buf).convert("RGB")
204
 
205
 
206
- MODELS: Dict[str, CompoundGPT] = {}
207
-
208
-
209
- def _get_model(model_key: str) -> CompoundGPT:
210
- if model_key in MODELS:
211
- return MODELS[model_key]
212
- ckpt_path = _download_checkpoint(model_key)
213
- model = _load_compound_model(ckpt_path)
214
- MODELS[model_key] = model
215
- return model
216
-
217
 
218
  def generate(
219
- model_choice: str,
220
  temperature: float,
221
- top_k: int,
222
- max_new_tokens: int,
223
- seed_text: str,
224
  ):
225
- model_key = "v1" if model_choice.startswith("v1") else "v2"
 
 
 
226
  try:
227
- model = _get_model(model_key)
228
- except RuntimeError as exc:
229
- raise gr.Error(str(exc)) from exc
230
-
231
- seed_steps = _parse_seed_text(seed_text)
232
- steps = _generate_steps(
233
- model=model,
234
- seed_steps=seed_steps,
235
- temperature=float(temperature),
236
- top_k=int(top_k),
237
- max_new_steps=int(max_new_tokens),
 
 
 
 
 
 
 
 
 
238
  )
239
- steps = [s for s in steps if int(s[0]) != 9] # drop STEP_PB for decode safety
240
 
241
  pm = decode_compound(steps)
242
- pm.write(TMP_MIDI_PATH)
243
- image = _render_pianoroll(pm)
244
- return image, TMP_MIDI_PATH
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
 
 
246
 
247
- with gr.Blocks(title="CODA MIDI Generator") as demo:
248
- gr.Markdown("# CODA MIDI Generator")
249
- gr.Markdown("CPU inference: generation usually takes **30–60s**.")
250
 
251
- with gr.Row():
252
- model_choice = gr.Radio(
253
- choices=["v1 (3M)", "v2 (25M)"],
254
- value="v2 (25M)",
255
- label="Model",
256
- )
257
- temperature = gr.Slider(0.5, 1.5, value=1.0, step=0.01, label="Temperature")
258
- top_k = gr.Slider(1, 100, value=50, step=1, label="Top-k")
259
-
260
- max_new_tokens = gr.Slider(
261
- 128,
262
- 1024,
263
- value=512,
264
- step=1,
265
- label="Max new tokens",
 
266
  )
267
- seed_text = gr.Textbox(
268
- label="Optional seed text",
269
- placeholder="Optional compound seed. Format: a,b,c,d,e,f,g; ... (leave empty for random vocab seed)",
270
  lines=2,
 
 
271
  )
272
 
273
- run_btn = gr.Button("Generate", variant="primary")
 
 
 
 
 
274
 
275
  with gr.Row():
276
- roll_out = gr.Image(type="pil", label="Piano Roll")
277
- midi_out = gr.File(label="MIDI")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
278
 
279
- run_btn.click(
280
  fn=generate,
281
- inputs=[model_choice, temperature, top_k, max_new_tokens, seed_text],
282
- outputs=[roll_out, midi_out],
283
  )
284
 
285
 
 
2
 
3
  import io
4
  import os
 
5
  import sys
6
+ import random
7
  from pathlib import Path
8
+ from typing import Dict, List, Optional, Tuple
9
 
10
  import gradio as gr
11
+ import matplotlib
12
+ matplotlib.use("Agg")
13
  import matplotlib.pyplot as plt
14
+ import matplotlib.patches as mpatches
15
  import numpy as np
16
  import pretty_midi
17
  import torch
18
+ import torch.nn as nn
19
  import torch.nn.functional as F
20
  from huggingface_hub import hf_hub_download
21
  from PIL import Image
 
28
  from compound import AXIS_SIZES, N_AXES, SENTINELS, STEP_BOS, STEP_EOS, decode_compound
29
  from compound_model import CompoundGPT, CompoundGPTConfig, default_compound_config
30
 
31
+ # ── Config ────────────────────────────────────────────────────────────────────
32
+ HF_REPO_ID = os.getenv("HF_REPO_ID", "Prajanya23/Coda")
33
+ HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_HUB_TOKEN")
34
+ TMP_MIDI = "/tmp/coda_output.mid"
35
+ N_PREFIX = 8
36
+ CLAP_DIM = 256
37
+
38
+ # Checkpoint paths inside the HF repo (set via Space secrets to override)
39
+ GPT_FILE = os.getenv("GPT_CKPT", "checkpoints/compound_best.pt")
40
+ CLAP_FILE = os.getenv("CLAP_CKPT", "checkpoints/clap_compound_best.pt")
41
+ PREFIX_FILE = os.getenv("PREFIX_CKPT", "checkpoints/prefix_projector_best.pt")
42
+
43
+ EXAMPLES = [
44
+ "a slow melancholic piano piece in a minor key with sparse flowing notes",
45
+ "an upbeat jazz trio with piano bass and drums syncopated and energetic",
46
+ "ambient electronic music with synthesizer pads slow and atmospheric",
47
+ "fast energetic rock band with electric guitar and drums",
48
+ "a gentle classical piece for piano and strings moderate tempo",
49
+ "a funky groove with bass guitar and brass instruments",
50
+ "a soft acoustic guitar piece fingerpicked quiet and introspective",
51
+ "an orchestral piece with strings and brass building to a climax",
52
+ ]
53
+
54
+ VOICE_COLORS = ["#534AB7", "#0F6E56", "#BA7517", "#993C1D",
55
+ "#185FA5", "#639922", "#A32D2D", "#D4537E"]
56
+
57
+ # ── Model cache ───────────────────────────────────────────────────────────────
58
+ _CACHE: Dict[str, object] = {}
59
+
60
+
61
+ def _dl(filename: str) -> str:
62
+ return hf_hub_download(repo_id=HF_REPO_ID, filename=filename, token=HF_TOKEN)
63
+
64
+
65
+ # ── GPT loader ────────────────────────────────────────────────────────────────
66
+
67
+ def _load_gpt() -> CompoundGPT:
68
+ if "gpt" in _CACHE:
69
+ return _CACHE["gpt"] # type: ignore[return-value]
70
+ ckpt = torch.load(_dl(GPT_FILE), map_location="cpu", weights_only=True)
71
+ cfg = default_compound_config()
72
+ raw = ckpt.get("config") if isinstance(ckpt, dict) else None
73
+ if isinstance(raw, dict):
74
+ for k, v in raw.items():
75
+ if hasattr(cfg, k):
76
+ setattr(cfg, k, v)
77
+ model = CompoundGPT(cfg)
78
+ model.load_state_dict(ckpt.get("model_state_dict", ckpt), strict=False)
79
+ model.eval()
80
+ _CACHE["gpt"] = model
81
+ return model
82
 
 
 
 
83
 
84
+ # ── CLAP text encoder (lightweight β€” no CompoundGPT inside) ──────────────────
85
+
86
+ class _TextEncoder(nn.Module):
87
+ """Sentence-transformer + CLAP text projection, reconstructed from checkpoint."""
88
+
89
+ def __init__(self, clap_state: dict):
90
+ super().__init__()
91
+ from sentence_transformers import SentenceTransformer
92
+ self._st = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
93
+
94
+ # Extract text_projection weights from CLAP state dict
95
+ proj = {k[len("text_projection."):]: v
96
+ for k, v in clap_state.items()
97
+ if k.startswith("text_projection.")}
98
+
99
+ weight_keys = sorted(k for k in proj if k.endswith(".weight"))
100
+ layers: List[nn.Module] = []
101
+ for i, wk in enumerate(weight_keys):
102
+ w = proj[wk]
103
+ bk = wk.replace(".weight", ".bias")
104
+ lin = nn.Linear(w.shape[1], w.shape[0], bias=(bk in proj))
105
+ lin.weight.data.copy_(w)
106
+ if bk in proj:
107
+ lin.bias.data.copy_(proj[bk])
108
+ layers.append(lin)
109
+ if i < len(weight_keys) - 1:
110
+ layers.append(nn.ReLU())
111
+ self._proj = nn.Sequential(*layers)
112
+
113
+ @torch.no_grad()
114
+ def encode(self, text: str) -> torch.Tensor:
115
+ raw = self._st.encode([text], convert_to_tensor=True, show_progress_bar=False)
116
+ emb = self._proj(raw.float())
117
+ return F.normalize(emb, dim=-1) # (1, 256)
118
+
119
+
120
+ # ── Prefix projector (reconstructed from checkpoint) ─────────────────────────
121
+
122
+ class _PrefixProjector(nn.Module):
123
+ def __init__(self, state: dict, n_prefix: int = N_PREFIX):
124
+ super().__init__()
125
+ self.n_prefix = n_prefix
126
+ weight_keys = sorted(k for k in state if k.endswith(".weight"))
127
+ self._gpt_dim = state[weight_keys[-1]].shape[0] // n_prefix
128
+
129
+ layers: List[nn.Module] = []
130
+ for i, wk in enumerate(weight_keys):
131
+ w = state[wk]
132
+ bk = wk.replace(".weight", ".bias")
133
+ lin = nn.Linear(w.shape[1], w.shape[0], bias=(bk in state))
134
+ lin.weight.data.copy_(w)
135
+ if bk in state:
136
+ lin.bias.data.copy_(state[bk])
137
+ layers.append(lin)
138
+ if i < len(weight_keys) - 1:
139
+ layers.append(nn.GELU())
140
+ self._net = nn.Sequential(*layers)
141
+
142
+ @torch.no_grad()
143
+ def forward(self, text_emb: torch.Tensor) -> torch.Tensor: # (1,256) β†’ (1,8,768)
144
+ return self._net(text_emb).view(text_emb.shape[0], self.n_prefix, self._gpt_dim)
145
+
146
+
147
+ def _load_clap() -> Tuple[Optional[_TextEncoder], Optional[_PrefixProjector]]:
148
+ if "text_enc" in _CACHE:
149
+ return _CACHE.get("text_enc"), _CACHE.get("prefix_proj") # type: ignore
150
 
151
+ try:
152
+ # weights_only=False needed because args is argparse.Namespace
153
+ clap_ckpt = torch.load(_dl(CLAP_FILE), map_location="cpu", weights_only=False)
154
+ prefix_ckpt = torch.load(_dl(PREFIX_FILE), map_location="cpu", weights_only=False)
155
 
156
+ clap_state = clap_ckpt.get("model_state_dict", clap_ckpt)
157
+ prefix_state = prefix_ckpt.get("model_state_dict", prefix_ckpt)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
158
 
159
+ text_enc = _TextEncoder(_LightStateDict(clap_state))
160
+ prefix_proj = _PrefixProjector(_LightStateDict(prefix_state))
161
+ text_enc.eval(); prefix_proj.eval()
162
 
163
+ _CACHE["text_enc"] = text_enc
164
+ _CACHE["prefix_proj"] = prefix_proj
165
+ print("[CODA] CLAP text conditioning loaded βœ“")
166
+ return text_enc, prefix_proj
 
 
 
 
 
 
 
 
 
 
167
 
168
+ except Exception as exc:
169
+ print(f"[CODA] CLAP not available ({exc}). Falling back to unconditioned generation.")
170
+ _CACHE["text_enc"] = _CACHE["prefix_proj"] = None
171
+ return None, None
172
 
 
 
 
 
 
 
 
 
 
 
 
 
173
 
174
+ class _LightStateDict(dict):
175
+ """Passthrough so _TextEncoder / _PrefixProjector constructors work with raw state dicts."""
176
+ pass
177
 
 
 
 
 
 
178
 
179
+ # ── Compound step embeddings (needed to prepend prefix) ───────────────────────
180
 
181
+ def _compound_embeds(model: CompoundGPT, step_ids: torch.Tensor) -> torch.Tensor:
182
+ """
183
+ Sum per-axis embeddings to get (B, T, n_embd) float tensor.
184
+ Searches common attribute names used in CompoundGPT implementations.
185
+ """
186
+ B, T, _ = step_ids.shape
187
+ d = model.config.n_embd
188
+ emb = torch.zeros(B, T, d)
189
 
190
+ axis_list = None
191
+ if hasattr(model, "input_embeds"):
192
+ axis_list = model.input_embeds
193
+ else:
194
+ for attr in ["axis_embeds", "axis_embed", "embed_axes", "wtes"]:
195
+ if hasattr(model, attr):
196
+ axis_list = getattr(model, attr); break
197
+ if hasattr(model, "transformer") and hasattr(model.transformer, attr):
198
+ axis_list = getattr(model.transformer, attr); break
199
+
200
+ if axis_list is None:
201
+ raise AttributeError(
202
+ "Cannot find axis embedding layers in CompoundGPT (tried 'input_embeds' "
203
+ "and common fallbacks). Check compound_model.py."
204
+ )
205
+
206
+ for i, layer in enumerate(axis_list):
207
+ emb = emb + layer(step_ids[:, :, i])
208
+ return emb
209
 
 
210
 
211
+ # ── Sampling ──────────────────────────────────────────────────────────────────
212
+
213
+ def _sample(logits: torch.Tensor, temp: float, top_k: int, top_p: float) -> int:
214
+ scaled = logits / max(temp, 1e-6)
215
+ # top-k
216
+ if 0 < top_k < scaled.numel():
217
+ v, _ = torch.topk(scaled, top_k)
218
+ scaled = scaled.masked_fill(scaled < v[-1], float("-inf"))
219
+ probs = F.softmax(scaled, dim=-1)
220
+ # top-p (nucleus)
221
+ if 0.0 < top_p < 1.0:
222
+ sp, si = torch.sort(probs, descending=True)
223
+ cum = torch.cumsum(sp, dim=-1)
224
+ sp[cum - sp > top_p] = 0.0
225
+ probs = torch.zeros_like(scaled).scatter_(0, si, sp)
226
+ probs = probs / probs.sum().clamp(min=1e-8)
227
+ return int(torch.multinomial(probs, 1).item())
228
+
229
+
230
+ # ── Generation ────────────────────────────────────────────────────────────────
231
 
232
  @torch.no_grad()
233
+ def _generate(
234
+ model: CompoundGPT,
235
+ prefix_embs: Optional[torch.Tensor], # (1, N_PREFIX, n_embd) or None
236
+ temperature: float,
237
+ top_k: int,
238
+ top_p: float,
239
+ max_steps: int,
240
  ) -> List[List[int]]:
241
+
242
+ bos = list(SENTINELS); bos[0] = STEP_BOS
243
+ generated: List[List[int]] = [bos]
244
+ conditioned = prefix_embs is not None
245
+
246
+ for _ in range(max_steps):
247
+ step_ids = torch.tensor([generated], dtype=torch.long) # (1, T, 7)
248
+ T = step_ids.shape[1]
249
+ n_pre = prefix_embs.shape[1] if conditioned else 0
250
+
251
+ if T + n_pre > model.config.block_size:
252
  break
253
+
254
+ if conditioned:
255
+ try:
256
+ step_e = _compound_embeds(model, step_ids) # (1, T, d)
257
+ full_e = torch.cat([prefix_embs, step_e], dim=1) # (1, n_pre+T, d)
258
+ pos_ids = torch.arange(full_e.shape[1]).unsqueeze(0)
259
+ logits = model(inputs_embeds=full_e, position_ids=pos_ids)
260
+ except (AttributeError, TypeError):
261
+ # Embedding layer name mismatch β€” fall back silently
262
+ conditioned = False
263
+ prefix_embs = None
264
+
265
+ if not conditioned:
266
+ pos_ids = torch.arange(T).unsqueeze(0)
267
+ logits = model(idx=step_ids, position_ids=pos_ids)
268
+
269
+ next_step = [
270
+ _sample(ax[0, -1, :], temperature, top_k, top_p)
271
+ for ax in logits
272
+ ]
273
 
274
  if next_step[0] == STEP_EOS:
 
 
275
  break
 
276
  generated.append(next_step)
277
 
278
  return generated
279
 
280
 
281
+ # ── Piano roll ────────────────────────────────────────────────────────────────
282
+
283
+ def _piano_roll(pm: pretty_midi.PrettyMIDI) -> Image.Image:
284
+ notes = [
285
+ (n.start, n.end, n.pitch, n.velocity, i)
286
+ for i, inst in enumerate(pm.instruments)
287
+ for n in inst.notes
288
+ ]
289
+
290
+ fig, ax = plt.subplots(figsize=(12, 3.2), dpi=140)
291
+ fig.patch.set_facecolor("#F9F8F5")
292
+ ax.set_facecolor("#F9F8F5")
293
 
 
294
  if notes:
295
+ min_p = max(0, min(n[2] for n in notes) - 3)
296
+ max_p = min(127, max(n[2] for n in notes) + 3)
297
+ max_t = max(n[1] for n in notes)
298
+
299
+ for start, end, pitch, vel, vi in notes:
300
+ ax.broken_barh(
301
+ [(start, max(0.02, end - start))],
302
+ (pitch - 0.42, 0.84),
303
+ facecolors=VOICE_COLORS[vi % len(VOICE_COLORS)],
304
+ alpha=0.45 + 0.55 * vel / 127,
305
+ linewidth=0,
306
+ )
 
 
 
 
 
 
307
 
308
+ ax.set_xlim(0, max_t + 0.2)
309
+ ax.set_ylim(min_p, max_p)
310
+
311
+ n_inst = len(pm.instruments)
312
+ if n_inst > 1:
313
+ patches = [
314
+ mpatches.Patch(color=VOICE_COLORS[i % len(VOICE_COLORS)],
315
+ label=pm.instruments[i].name or f"voice {i+1}")
316
+ for i in range(n_inst)
317
+ ]
318
+ ax.legend(handles=patches, fontsize=7, loc="upper right",
319
+ framealpha=0.8, edgecolor="none", facecolor="#F9F8F5")
320
+ else:
321
+ ax.text(0.5, 0.5, "No notes generated", ha="center", va="center",
322
+ transform=ax.transAxes, color="#888780", fontsize=11)
323
+
324
+ ax.set_xlabel("Time (s)", fontsize=9, color="#5F5E5A")
325
+ ax.set_ylabel("Pitch", fontsize=9, color="#5F5E5A")
326
+ ax.tick_params(labelsize=8, colors="#5F5E5A")
327
+ for sp in ax.spines.values():
328
+ sp.set_visible(False)
329
+ ax.spines["bottom"].set_visible(True)
330
+ ax.spines["left"].set_visible(True)
331
+ ax.spines["bottom"].set_color("#D3D1C7"); ax.spines["bottom"].set_linewidth(0.5)
332
+ ax.spines["left"].set_color("#D3D1C7"); ax.spines["left"].set_linewidth(0.5)
333
+ ax.grid(axis="y", alpha=0.12, linewidth=0.5, color="#888780")
334
 
335
  buf = io.BytesIO()
336
+ fig.tight_layout(pad=0.6)
337
+ fig.savefig(buf, format="png", facecolor="#F9F8F5")
338
  plt.close(fig)
339
  buf.seek(0)
340
  return Image.open(buf).convert("RGB")
341
 
342
 
343
+ # ── Main callable ─────────────────────────────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
344
 
345
  def generate(
346
+ prompt: str,
347
  temperature: float,
348
+ top_k: int,
349
+ top_p: float,
350
+ max_steps: int,
351
  ):
352
+ prompt = (prompt or "").strip()
353
+ if not prompt:
354
+ raise gr.Error("Please enter a text prompt describing the music you want.")
355
+
356
  try:
357
+ model = _load_gpt()
358
+ except Exception as exc:
359
+ raise gr.Error(f"Failed to load model: {exc}") from exc
360
+
361
+ text_enc, prefix_proj = _load_clap()
362
+ prefix_embs = None
363
+ mode = "unconditioned"
364
+
365
+ if text_enc is not None and prefix_proj is not None:
366
+ try:
367
+ text_emb = text_enc.encode(prompt) # (1, 256)
368
+ prefix_embs = prefix_proj(text_emb) # (1, 8, 768)
369
+ mode = "CLAP-conditioned"
370
+ except Exception as exc:
371
+ print(f"[CODA] Text conditioning failed: {exc}")
372
+
373
+ steps = _generate(
374
+ model=model, prefix_embs=prefix_embs,
375
+ temperature=float(temperature), top_k=int(top_k),
376
+ top_p=float(top_p), max_steps=int(max_steps),
377
  )
378
+ steps = [s for s in steps if int(s[0]) != 9] # drop STEP_PB
379
 
380
  pm = decode_compound(steps)
381
+ pm.write(TMP_MIDI)
382
+ image = _piano_roll(pm)
383
+
384
+ all_notes = [n for inst in pm.instruments for n in inst.notes]
385
+ n_notes = len(all_notes)
386
+ n_voices = len(pm.instruments)
387
+ duration = round(max((n.end for n in all_notes), default=0.0), 1)
388
+ pitches = [n.pitch for n in all_notes]
389
+ p_std = round(float(np.std(pitches)), 1) if pitches else 0.0
390
+
391
+ info = (
392
+ f"**Mode:** {mode} &nbsp;|&nbsp; "
393
+ f"**Notes:** {n_notes} &nbsp;|&nbsp; "
394
+ f"**Voices:** {n_voices} &nbsp;|&nbsp; "
395
+ f"**Duration:** {duration}s &nbsp;|&nbsp; "
396
+ f"**Pitch Οƒ:** {p_std}"
397
+ )
398
 
399
+ return image, TMP_MIDI, info
400
 
 
 
 
401
 
402
+ # ── UI ────────────────────────────────────────────────────────────────────────
403
+
404
+ CSS = """
405
+ .prompt-box textarea { font-size: 15px !important; line-height: 1.6 !important; }
406
+ .generate-btn { font-size: 15px !important; }
407
+ .info-md p { font-size: 13px; color: var(--body-text-color-subdued); }
408
+ .gr-examples table td { font-size: 12px; }
409
+ footer { display: none !important; }
410
+ """
411
+
412
+ with gr.Blocks(title="CODA", css=CSS) as demo:
413
+
414
+ gr.Markdown("## CODA")
415
+ gr.Markdown(
416
+ "Text-conditioned symbolic MIDI generation Β· compound tokenization + CLAP alignment \n"
417
+ "<small>CPU inference β€” generation takes ~30–60 s</small>"
418
  )
419
+
420
+ prompt = gr.Textbox(
421
+ placeholder="Describe the music you want to generate…",
422
  lines=2,
423
+ show_label=False,
424
+ elem_classes=["prompt-box"],
425
  )
426
 
427
+ gr.Examples(
428
+ examples=[[p] for p in EXAMPLES],
429
+ inputs=[prompt],
430
+ label="Example prompts",
431
+ examples_per_page=8,
432
+ )
433
 
434
  with gr.Row():
435
+ temperature = gr.Slider(0.5, 1.5, value=0.9, step=0.05, label="Temperature")
436
+ top_k = gr.Slider(1, 100, value=40, step=1, label="Top-k")
437
+
438
+ with gr.Accordion("Advanced", open=False):
439
+ with gr.Row():
440
+ top_p = gr.Slider(0.5, 1.0, value=0.92, step=0.01, label="Top-p (nucleus)")
441
+ max_steps = gr.Slider(64, 512, value=256, step=64, label="Max steps")
442
+
443
+ gen_btn = gr.Button("Generate", variant="primary", elem_classes=["generate-btn"])
444
+
445
+ roll_out = gr.Image(type="pil", label="Piano roll", show_download_button=True)
446
+
447
+ with gr.Row():
448
+ midi_out = gr.File(label="Download MIDI")
449
+
450
+ info_out = gr.Markdown("", elem_classes=["info-md"])
451
 
452
+ gen_btn.click(
453
  fn=generate,
454
+ inputs=[prompt, temperature, top_k, top_p, max_steps],
455
+ outputs=[roll_out, midi_out, info_out],
456
  )
457
 
458
 
requirements.txt CHANGED
@@ -1,9 +1,8 @@
1
- --extra-index-url https://download.pytorch.org/whl/cpu
2
- torch==2.11.0
3
- gradio>=4.0.0
4
- huggingface_hub>=0.20.0
5
- pretty_midi>=0.2.11
6
- matplotlib>=3.7
7
- numpy>=1.24
8
- Pillow>=10.0.0
9
- mido>=1.3.0
 
1
+ torch
2
+ gradio
3
+ pretty_midi
4
+ matplotlib
5
+ numpy
6
+ Pillow
7
+ huggingface_hub
8
+ sentence-transformers