"""CharacterBench: validate the judge as deployed against the human annotation of the test set. Every test sample ships with the response the annotators scored (response_messages.response, translated under translation_en) and their score (annotation_score), plus the authors' own CharacterJudge predictions (pred_{lang} and pred_{lang}_sc). This script builds the judge input with the authors' function (process_wo_context_{lang}_all.py process), sends it to the deployed CharacterJudge as the platform does, and compares: - greedy decoding (temperature 0) and self-consistency (10 samples with the checkpoint's generation_config, temperature 0.7 / top-p 0.8 / top-k 20 / repetition penalty 1.05, seed derived from the sample as on the platform, score = mean of the 10 samples, each read as all_eval.py's check() reads it, as on the platform) with the human scores: Pearson per test file, as in the paper's Table 3. For comparison the report also gives the correlation when only in-range integer outputs are averaged (the platform's reading before protocol 1.2); - both with the authors' released predictions. export CHARACTERJUDGE_BASE_URL=http://host:port/v1 CHARACTERJUDGE_API_KEY=... python3 -I tools/experiments/characterbench_judge_validation.py run /tmp/cb data/benchmarks/characterbench zh val_zh.jsonl python3 -I tools/experiments/characterbench_judge_validation.py report val_zh.jsonl val_en.jsonl > report.json `run` is resumable (it skips samples already in the output file). """ import ast import concurrent.futures as cf import copy import hashlib import json import math import os import sys import tempfile import threading import urllib.request FILES = [ # test file, platform column key, score range ("memory_consistency", "mc", 4), ("fact_accuracy", "fa", 4), ("boundary_consistency", "bck", 3), ("attribute_consistency_bot", "acb", 4), ("attribute_consistency_human", "ach", 4), ("behavior_consistency_bot", "bcpb", 4), ("behavior_consistency_human", "bcph", 3), ("emotion_self_regulation", "es", 4), ("empathetic_responsiveness", "er", 4), ("morality_stability", "ms", 2), ("morality_robustness", "mr", 2), ("human_likeness", "hl", 5), ("engagement", "eg", 5), ] # Paper, Table 3: Pearson (%) of CharacterJudge (with self-consistency) and without it, zh/en, per column. PAPER = { "sc": {"zh": [80, 92, 71, 80, 63, 62, 65, 67, 65, 66, 61, 52, 58], "en": [81, 88, 65, 76, 57, 57, 58, 65, 62, 64, 55, 53, 53]}, "greedy": {"zh": [80, 89, 70, 80, 59, 58, 55, 65, 60, 54, 59, 46, 59], "en": [78, 87, 62, 70, 56, 54, 55, 55, 57, 58, 48, 51, 51]}, "avg": {"sc": {"zh": 68, "en": 64}, "greedy": {"zh": 64, "en": 60}}, } SAMPLING = {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "repetition_penalty": 1.05} def source_of(path, names): src = open(path, encoding="utf-8").read() return "\n\n".join(ast.get_source_segment(src, n) for n in ast.parse(src).body if isinstance(n, ast.FunctionDef) and n.name in names) def constants(path): return {n.targets[0].id: n.value.value for n in ast.parse(open(path, encoding="utf-8").read()).body if isinstance(n, ast.Assign) and isinstance(n.value, ast.Constant) and isinstance(n.value.value, str)} def data_info(repo, data, lang): tree = ast.parse(open(f"{repo}/construct_prompts/data_info_{lang}.py", encoding="utf-8").read()) env = {} for n in tree.body: if isinstance(n, ast.ImportFrom): c = constants(f"{data}/construct_prompts/eval_prompts_{lang}/{n.module.split('.')[-1]}.py") for a in n.names: env[a.asname or a.name] = c.get(a.name, "") for n in tree.body: if isinstance(n, ast.Assign): env[n.targets[0].id] = eval(compile(ast.Expression(n.value), "data_info", "eval"), {}, env) return env["data_info"] def platform_seed(seed_id): """engine/runner.ts judgeSeed: first 4 bytes of sha1("judge:" + seed id), 31 bits.""" return int.from_bytes(hashlib.sha1(f"judge:{seed_id}".encode()).digest()[:4], "big") & 0x7FFFFFFF def post(body): base = os.environ["CHARACTERJUDGE_BASE_URL"].rstrip("/") key = os.environ.get("CHARACTERJUDGE_API_KEY", "") err = None for _ in range(4): try: req = urllib.request.Request(f"{base}/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json", **({"Authorization": f"Bearer {key}"} if key else {})}) return json.loads(urllib.request.urlopen(req, timeout=180).read()) except Exception as e: # network errors and 5xx: retry err = e raise RuntimeError(repr(err)) def run(repo, data, lang, out): ns = {"json": json, "deepcopy": copy.deepcopy, "data_info": data_info(repo, data, lang)} exec(source_of(f"{repo}/construct_prompts/process_wo_context_{lang}_all.py", {"process"}), ns) done = set() if os.path.exists(out): done = {json.loads(l)["key"] for l in open(out)} jobs = [] tmp = tempfile.mkdtemp() for file, col, _ in FILES: rows = json.load(open(f"{data}/eval_data/raw_data/{file}_test.json", encoding="utf-8")) prepared = [] for i, it in enumerate(rows): it = copy.deepcopy(it) en = it["translation_en"] # The annotated response; its translation for English. resp_zh = it["response_messages"]["response"] resp_en = (en.get("response_messages") or {}).get("response", resp_zh) it["response_zh"], en["response_en"] = resp_zh, resp_en prepared.append((i, it)) path = f"{tmp}/{file}.json" json.dump([p[1] for p in prepared], open(path, "w", encoding="utf-8"), ensure_ascii=False) outs, _ = ns["process"](path, file) # process() keeps the samples whose response is not empty, in order. kept = [p for p in prepared if (p[1]["response_zh"] if lang == "zh" else p[1]["translation_en"]["response_en"]) != ""] assert len(kept) == len(outs), (file, len(kept), len(outs)) for (i, it), o in zip(kept, outs): key = f"{file}:{i}" if key not in done: jobs.append((file, col, i, it, o["instruction"])) print(lang, "to do", len(jobs), flush=True) lock = threading.Lock() def one(job): file, col, i, it, prompt = job msgs = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}] # CB_SC_ONLY=1 skips the greedy request (repeat and seed-variance runs); CB_SEED_OFFSET shifts every seed. g = None if os.environ.get("CB_SC_ONLY") else post({"model": "character-judge", "messages": msgs, "temperature": 0, "max_tokens": 512}) seed_id = f"cb-{lang}-{col}-{it.get('id', i)}" seed = (platform_seed(seed_id) + int(os.environ.get("CB_SEED_OFFSET", "0"))) & 0x7FFFFFFF s = post({"model": "character-judge", "messages": msgs, "max_tokens": 512, "n": 10, "seed": seed, **SAMPLING}) rec = {"key": f"{file}:{i}", "file": file, "i": i, "human": it["annotation_score"], "released": it.get(f"pred_{lang}"), "released_sc": it.get(f"pred_{lang}_sc"), "greedy": g["choices"][0]["message"]["content"] if g else None, "sc": [c["message"]["content"] for c in s["choices"]], "seed": seed, "prompt_tokens": s.get("usage", {}).get("prompt_tokens")} with lock, open(out, "a", encoding="utf-8") as f: f.write(json.dumps(rec, ensure_ascii=False) + "\n") errors = 0 with cf.ThreadPoolExecutor(int(os.environ.get("CONCURRENCY", "8"))) as ex: for fut in cf.as_completed([ex.submit(one, j) for j in jobs]): if fut.exception(): errors += 1 print(lang, "finished; failed requests", errors, flush=True) def read_one(text, top): """In-range integer on the first line, else None (the platform's reading before protocol 1.2).""" first = (text or "").strip().split("\n")[0].strip() try: v = float(first) except ValueError: return None return v if v.is_integer() and 1 <= v <= top else None def read_official(text): """all_eval.py check(): with a line break the first line as a number, else the whole output, else 3.0; a score outside the column's range is kept. (A first line that is not a number stops the official script; read as 3.0.) Returns (score, unreadable).""" text = text or "" try: return float(text.split("\n")[0] if "\n" in text else text), False except ValueError: return 3.0, True def pearson(x, y): if len(x) < 3: return None mx, my = sum(x) / len(x), sum(y) / len(y) num = sum((a - mx) * (b - my) for a, b in zip(x, y)) den = math.sqrt(sum((a - mx) ** 2 for a in x) * sum((b - my) ** 2 for b in y)) return num / den if den else None def report(paths): out = {} for path in paths: recs = [json.loads(l) for l in open(path, encoding="utf-8")] lang = "zh" if "_zh" in os.path.basename(path) else "en" totals = {"n": 0, "greedy_invalid": 0, "sc_invalid_samples": 0, "sc_all_invalid": 0, "sc_out_of_range": 0, "sc_unreadable": 0, "greedy_out_of_range": 0, "greedy_unreadable": 0} cols = [] for k, (file, col, top) in enumerate(FILES): rs = [r for r in recs if r["file"] == file] # Official reading (protocol 1.2): every output counts. go = [(read_official(r["greedy"]), r) for r in rs] so = [] for r in rs: vals = [read_official(t) for t in r["sc"]] totals["sc_out_of_range"] += sum(1 for v, _ in vals if not 1 <= v <= top) totals["sc_unreadable"] += sum(1 for _, bad in vals if bad) so.append((sum(v for v, _ in vals) / len(vals), r)) totals["greedy_out_of_range"] += sum(1 for (v, _), _r in go if not 1 <= v <= top) totals["greedy_unreadable"] += sum(1 for (_, bad), _r in go if bad) # Valid-only reading (before 1.2), for comparison. g = [(read_one(r["greedy"], top), r) for r in rs] sc = [] for r in rs: vals = [read_one(t, top) for t in r["sc"]] ok = [v for v in vals if v is not None] totals["sc_invalid_samples"] += len(vals) - len(ok) if not ok: totals["sc_all_invalid"] += 1 sc.append((sum(ok) / len(ok) if ok else None, r)) totals["n"] += len(rs) totals["greedy_invalid"] += sum(1 for v, _ in g if v is None) gv = [(v, r) for v, r in g if v is not None] sv = [(v, r) for v, r in sc if v is not None] cols.append({ "file": file, "column": col, "n": len(rs), "pearson_greedy": pearson([v for (v, _), _r in go], [r["human"] for _, r in go]), "pearson_sc": pearson([v for v, _ in so], [r["human"] for _, r in so]), "pearson_greedy_valid_only": pearson([v for v, _ in gv], [r["human"] for _, r in gv]), "pearson_sc_valid_only": pearson([v for v, _ in sv], [r["human"] for _, r in sv]), "paper_greedy": PAPER["greedy"][lang][k] / 100, "paper_sc": PAPER["sc"][lang][k] / 100, "released_pearson_sc": pearson([r["released_sc"] for r in rs], [r["human"] for r in rs]), "greedy_equals_released": sum(1 for (v, _), r in go if v == r["released"]) / max(1, len(rs)), "sc_mae_vs_released_sc": sum(abs(v - r["released_sc"]) for v, r in so) / max(1, len(so)), # Column score on the benchmark's 5-point scale for the annotated responses (mean / max × 5). "score_sc": sum(v for v, _ in so) / max(1, len(so)) / top * 5, "score_released_sc": sum(r["released_sc"] for r in rs) / max(1, len(rs)) / top * 5, "score_human": sum(r["human"] for r in rs) / max(1, len(rs)) / top * 5, }) mean = lambda xs: sum(xs) / len(xs) if xs else None out[lang] = { "columns": cols, "avg_pearson_greedy": mean([c["pearson_greedy"] for c in cols if c["pearson_greedy"] is not None]), "avg_pearson_sc": mean([c["pearson_sc"] for c in cols if c["pearson_sc"] is not None]), "avg_pearson_greedy_valid_only": mean([c["pearson_greedy_valid_only"] for c in cols if c["pearson_greedy_valid_only"] is not None]), "avg_pearson_sc_valid_only": mean([c["pearson_sc_valid_only"] for c in cols if c["pearson_sc_valid_only"] is not None]), "avg_pearson_released_sc": mean([c["released_pearson_sc"] for c in cols if c["released_pearson_sc"] is not None]), "paper_avg": {"greedy": PAPER["avg"]["greedy"][lang] / 100, "sc": PAPER["avg"]["sc"][lang] / 100}, **totals, } json.dump(out, sys.stdout, ensure_ascii=False, indent=1) if __name__ == "__main__": if sys.argv[1] == "run": run(*sys.argv[2:6]) else: report(sys.argv[2:])