ChatDev/execorchestra/scripts/research_pilot.py
2026-11-05 16:22:12 +08:00

205 lines
10 KiB
Python

"""Run the frozen framework-only probe cohort using the existing solver."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def read(path):
return json.loads(path.read_text(encoding='utf-8'))
def save(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
path.chmod(0o600)
def verify(experiment, root=ROOT):
protocol = read(experiment / 'experiment-protocol.json')
if protocol['protocol'] != 'xbench-pilot-experiment-v2-framework-only':
raise ValueError('unexpected_protocol')
if list(protocol['arms']) != ['framework_native']:
raise ValueError('framework_only_required')
sample = read(experiment / 'sample-manifest.json')
expected = {
experiment / 'sample-manifest.json': protocol['sample_manifest_sha256'],
experiment / 'solver/tasks.jsonl': sample['sample_solver_sha256'],
experiment / 'framework-config.private.json': protocol['arms']['framework_native']['configuration_sha256'],
root / 'sgar_mvp/config.json': protocol['identities']['original_config_sha256'],
root / 'Pool/resources/json/effective_combine.json': protocol['identities']['effective_pool_sha256'],
}
expected.update({root / 'sgar_mvp/config' / name: digest
for name, digest in protocol['identities']['policy_sha256'].items()})
for path, digest in expected.items():
if sha(path) != digest:
raise ValueError('frozen_identity_mismatch:' + path.name)
tasks = [json.loads(x) for x in (experiment / 'solver/tasks.jsonl').read_text().splitlines() if x.strip()]
if [x['id'] for x in tasks] != protocol['sample_ids'] or sample['sample_ids'] != protocol['sample_ids']:
raise ValueError('sample_id_mismatch')
fields = {'id', 'benchmark', 'query', 'root_url', 'query_sha256'}
if any(set(x) != fields or hashlib.sha256(x['query'].encode()).hexdigest() != x['query_sha256'] for x in tasks):
raise ValueError('solver_projection_or_query_mismatch')
probes = protocol['execution']['initial_phase_ids']
if probes != sample['probe_ids'] or probes != protocol['probe_ids'] or len(probes) != 3:
raise ValueError('probe_selection_mismatch')
return protocol
def hold_reason(row, limits):
if row.get('prediction_extraction',{}).get('status')=='error':
return 'prediction_extraction_error'
if row['pipeline_status'] in {'complete_success','success_with_warnings'} and not row.get('prediction','').strip():
return 'missing_delivered_prediction'
cost = row.get('cost_summary')
if not isinstance(cost, dict):
return 'missing_cost_summary'
if cost.get('max_generation_requests') != limits['max_generation_requests']:
return 'request_limit_mismatch'
if cost.get('resolved_cost_policy') != limits['model_cost_policy']:
return 'cost_policy_mismatch'
for key in ['accounting_persistence_failure_count', 'pending_call_count',
'response_usage_invalid_count', 'response_usage_missing_count',
'response_usage_unpriced_count', 'no_response_attempt_count']:
if key not in cost or cost[key] != 0:
return 'accounting_or_provider_issue:' + key
budget_stop = cost.get('budget_status', {}).get('limit_reached', False)
if row['return_code'] and not (row.get('timed_out') or budget_stop):
return 'solver_nonzero_exit_requires_inspection'
if row['pipeline_status'] not in {'complete_success', 'success_with_warnings'} and not (row.get('timed_out') or budget_stop):
return 'incomplete_pipeline_requires_inspection'
return None
def stop(proc):
try:
os.killpg(proc.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
try:
os.killpg(proc.pid, signal.SIGKILL)
except ProcessLookupError:
pass
proc.wait()
def command(argv, logfile, timeout):
with logfile.open('wb') as stream:
logfile.chmod(0o600)
proc = subprocess.Popen(argv, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
try:
return proc.wait(timeout=timeout)
except BaseException:
stop(proc)
raise
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--experiment', required=True)
p.add_argument('--check-only', action='store_true')
p.add_argument('--probe-id', help='Run one member of the frozen probe independently for diagnosis')
for name in ['output', 'references', 'image', 'image-sha256', 'apptainer', 'tools-prefix']:
p.add_argument('--' + name)
a = p.parse_args()
experiment = Path(a.experiment).resolve()
protocol = verify(experiment)
if a.probe_id is not None and a.probe_id not in protocol['probe_ids']:
raise ValueError('diagnostic_id_outside_frozen_probe')
selected_ids = [a.probe_id] if a.probe_id is not None else protocol['probe_ids']
scope = 'independent_diagnostic' if a.probe_id is not None else 'frozen_probe'
if a.check_only:
print(json.dumps({'verified': True, 'probe_ids': protocol['probe_ids'], 'framework_only': True,
'selected_ids': selected_ids, 'scope': scope}))
return 0
if not os.environ.get('SLURM_JOB_ID'):
raise ValueError('allocated_job_required')
if not all(getattr(a, name) for name in ['output', 'references', 'image', 'image_sha256', 'apptainer', 'tools_prefix']):
raise ValueError('runtime_arguments_required')
out = Path(a.output).resolve()
out.mkdir(parents=True, exist_ok=False, mode=0o700)
save(out / 'protocol-snapshot.json', protocol)
limits = protocol['shared_limits']
save(out / 'run-identity.json', {
'job_id': os.environ['SLURM_JOB_ID'], 'protocol_sha256': sha(experiment / 'experiment-protocol.json'),
'sample_manifest_sha256': sha(experiment / 'sample-manifest.json'),
'selected_ids': selected_ids, 'scope': scope, 'deadline_seconds': limits['deadline_seconds'],
'max_generation_requests': limits['max_generation_requests'],
'source_sha256': {name: sha(ROOT / 'scripts' / name) for name in ['research_pilot.py', 'research_benchmark.py', 'run_apptainer.py']},
})
summary = {'job_id': os.environ['SLURM_JOB_ID'], 'selected_ids': selected_ids, 'scope': scope,
'state': 'running', 'trials': [], 'hold_reason': None}
save(out / 'summary.json', summary)
container = ['--image', a.image, '--image-sha256', a.image_sha256,
'--apptainer', a.apptainer, '--tools-prefix', a.tools_prefix]
try:
for ident in selected_ids:
print(json.dumps({'event': 'task_started', 'id': ident}), flush=True)
dest = out / ('task-' + ident)
argv = [sys.executable, '-m', 'scripts.research_benchmark', 'run',
'--tasks', str(experiment / 'solver/tasks.jsonl'), '--output', str(dest),
'--config', str(experiment / 'framework-config.private.json'),
'--limit', '1', '--id', ident, '--deadline-sec', str(limits['deadline_seconds']),
'--max-generation-requests', str(limits['max_generation_requests']), *container]
started = time.monotonic()
code = command(argv, out / (ident + '-solver.log'), limits['deadline_seconds'] + 120)
predictions = dest / 'predictions.jsonl'
if not predictions.exists():
summary['hold_reason'] = 'missing_trial_record:' + ident
summary['trials'].append({'id': ident, 'solver_exit_code': code, 'elapsed_seconds': round(time.monotonic() - started, 3)})
break
rows = [json.loads(x) for x in predictions.read_text().splitlines() if x.strip()]
if len(rows) != 1 or rows[0]['id'] != ident:
raise ValueError('prediction_identity_mismatch')
row = rows[0]
record = {key: row[key] for key in ['id', 'return_code', 'timed_out', 'elapsed_seconds', 'pipeline_status']}
record.update(prediction_present=bool(row['prediction'].strip()), cost_summary=row['cost_summary'])
gradefile = out / (ident + '-scores.jsonl')
gradecode = command([sys.executable, '-m', 'scripts.research_benchmark', 'score',
'--tasks', str(experiment / 'solver/tasks.jsonl'), '--references', a.references,
'--predictions', str(predictions), '--output', str(gradefile),
'--judge-model', protocol['evaluation']['judge_model']],
out / (ident + '-judge.log'), 210)
if gradefile.exists():
grades = [json.loads(x) for x in gradefile.read_text().splitlines() if x.strip()]
if len(grades) == 1 and grades[0]['id'] == ident:
record['grade'] = {k: grades[0].get(k) for k in ['score', 'status', 'usage', 'judge_model']}
record['judge_exit_code'] = gradecode
summary['trials'].append(record)
summary['hold_reason'] = hold_reason(row, limits)
save(out / 'summary.json', summary)
print(json.dumps({'event': 'task_finished', 'id': ident, 'pipeline_status': row['pipeline_status'],
'score': record.get('grade', {}).get('score'), 'hold_reason': summary['hold_reason']}), flush=True)
if summary['hold_reason']:
break
summary['state'] = 'held' if summary['hold_reason'] else 'completed'
scores = [x.get('grade', {}).get('score') for x in summary['trials']]
summary['answer_accuracy'] = sum(scores) / len(scores) if len(scores) == len(protocol['probe_ids']) and all(x is not None for x in scores) else None
save(out / 'summary.json', summary)
return 4 if summary['hold_reason'] else 0
except BaseException as exc:
summary.update(state='interrupted_or_error', error_class=type(exc).__name__)
save(out / 'summary.json', summary)
raise
if __name__ == '__main__':
def interrupted(signum, frame):
raise SystemExit(128 + signum)
for sig in [signal.SIGINT, signal.SIGTERM]:
signal.signal(sig, interrupted)
raise SystemExit(main())