Spaces:
Paused
Paused
fix: WER=0 bug when no segments match in evaluation
Browse filesWhen zero predicted segments overlap with gold segments in time,
summarise_matches reported WER=0.0 instead of WER=1.0 (due to
0/max(0,1)=0 division). This made grid search pick "no overlap"
configs as winners, corrupting all subsequent parameter iterations.
Fix: WER=1.0 when no matches. Also add recall-weighted WER/CER
that penalizes missed gold segments (if only 50% of gold matched,
even with perfect matched WER, overall WER = 0.5*0 + 0.5*1 = 50%).
Add matched_wer, matched_cer, recall to summary output.
Made-with: Cursor
- evaluation.py +22 -4
evaluation.py
CHANGED
|
@@ -376,18 +376,36 @@ def summarise_matches(
|
|
| 376 |
only_pred = [m for m in matches if m.get("gold") is None]
|
| 377 |
only_gold = [m for m in matches if m.get("pred") is None]
|
| 378 |
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
|
| 383 |
summary = {
|
| 384 |
"total_predicted_segments": len(only_pred) + len(matched),
|
| 385 |
-
"total_gold_segments":
|
| 386 |
"matched_segments": len(matched),
|
| 387 |
"unmatched_pred": len(only_pred),
|
| 388 |
"missed_gold": len(only_gold),
|
| 389 |
"avg_wer": round(avg_wer, 4),
|
| 390 |
"avg_cer": round(avg_cer, 4),
|
|
|
|
|
|
|
|
|
|
| 391 |
"speaker_accuracy": round(speaker_correct, 4),
|
| 392 |
}
|
| 393 |
|
|
|
|
| 376 |
only_pred = [m for m in matches if m.get("gold") is None]
|
| 377 |
only_gold = [m for m in matches if m.get("pred") is None]
|
| 378 |
|
| 379 |
+
total_gold = len(only_gold) + len(matched)
|
| 380 |
+
|
| 381 |
+
if matched:
|
| 382 |
+
matched_wer = sum(m["wer"] for m in matched) / len(matched)
|
| 383 |
+
matched_cer = sum(m["cer"] for m in matched) / len(matched)
|
| 384 |
+
speaker_correct = sum(1 for m in matched if m.get("speaker_match")) / len(matched)
|
| 385 |
+
else:
|
| 386 |
+
matched_wer = 1.0
|
| 387 |
+
matched_cer = 1.0
|
| 388 |
+
speaker_correct = 0.0
|
| 389 |
+
|
| 390 |
+
if total_gold > 0:
|
| 391 |
+
recall = len(matched) / total_gold
|
| 392 |
+
avg_wer = matched_wer * recall + 1.0 * (1.0 - recall)
|
| 393 |
+
avg_cer = matched_cer * recall + 1.0 * (1.0 - recall)
|
| 394 |
+
else:
|
| 395 |
+
avg_wer = matched_wer
|
| 396 |
+
avg_cer = matched_cer
|
| 397 |
|
| 398 |
summary = {
|
| 399 |
"total_predicted_segments": len(only_pred) + len(matched),
|
| 400 |
+
"total_gold_segments": total_gold,
|
| 401 |
"matched_segments": len(matched),
|
| 402 |
"unmatched_pred": len(only_pred),
|
| 403 |
"missed_gold": len(only_gold),
|
| 404 |
"avg_wer": round(avg_wer, 4),
|
| 405 |
"avg_cer": round(avg_cer, 4),
|
| 406 |
+
"matched_wer": round(matched_wer, 4),
|
| 407 |
+
"matched_cer": round(matched_cer, 4),
|
| 408 |
+
"recall": round(recall, 4) if total_gold > 0 else 0.0,
|
| 409 |
"speaker_accuracy": round(speaker_correct, 4),
|
| 410 |
}
|
| 411 |
|