mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
205 lines
10 KiB
Python
205 lines
10 KiB
Python
"""Run the frozen framework-only probe cohort using the existing solver."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_text(encoding='utf-8'))
|
|
|
|
|
|
def save(path, value):
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
path.chmod(0o600)
|
|
|
|
|
|
def verify(experiment, root=ROOT):
|
|
protocol = read(experiment / 'experiment-protocol.json')
|
|
if protocol['protocol'] != 'xbench-pilot-experiment-v2-framework-only':
|
|
raise ValueError('unexpected_protocol')
|
|
if list(protocol['arms']) != ['framework_native']:
|
|
raise ValueError('framework_only_required')
|
|
sample = read(experiment / 'sample-manifest.json')
|
|
expected = {
|
|
experiment / 'sample-manifest.json': protocol['sample_manifest_sha256'],
|
|
experiment / 'solver/tasks.jsonl': sample['sample_solver_sha256'],
|
|
experiment / 'framework-config.private.json': protocol['arms']['framework_native']['configuration_sha256'],
|
|
root / 'sgar_mvp/config.json': protocol['identities']['original_config_sha256'],
|
|
root / 'Pool/resources/json/effective_combine.json': protocol['identities']['effective_pool_sha256'],
|
|
}
|
|
expected.update({root / 'sgar_mvp/config' / name: digest
|
|
for name, digest in protocol['identities']['policy_sha256'].items()})
|
|
for path, digest in expected.items():
|
|
if sha(path) != digest:
|
|
raise ValueError('frozen_identity_mismatch:' + path.name)
|
|
tasks = [json.loads(x) for x in (experiment / 'solver/tasks.jsonl').read_text().splitlines() if x.strip()]
|
|
if [x['id'] for x in tasks] != protocol['sample_ids'] or sample['sample_ids'] != protocol['sample_ids']:
|
|
raise ValueError('sample_id_mismatch')
|
|
fields = {'id', 'benchmark', 'query', 'root_url', 'query_sha256'}
|
|
if any(set(x) != fields or hashlib.sha256(x['query'].encode()).hexdigest() != x['query_sha256'] for x in tasks):
|
|
raise ValueError('solver_projection_or_query_mismatch')
|
|
probes = protocol['execution']['initial_phase_ids']
|
|
if probes != sample['probe_ids'] or probes != protocol['probe_ids'] or len(probes) != 3:
|
|
raise ValueError('probe_selection_mismatch')
|
|
return protocol
|
|
|
|
|
|
def hold_reason(row, limits):
|
|
if row.get('prediction_extraction',{}).get('status')=='error':
|
|
return 'prediction_extraction_error'
|
|
if row['pipeline_status'] in {'complete_success','success_with_warnings'} and not row.get('prediction','').strip():
|
|
return 'missing_delivered_prediction'
|
|
cost = row.get('cost_summary')
|
|
if not isinstance(cost, dict):
|
|
return 'missing_cost_summary'
|
|
if cost.get('max_generation_requests') != limits['max_generation_requests']:
|
|
return 'request_limit_mismatch'
|
|
if cost.get('resolved_cost_policy') != limits['model_cost_policy']:
|
|
return 'cost_policy_mismatch'
|
|
for key in ['accounting_persistence_failure_count', 'pending_call_count',
|
|
'response_usage_invalid_count', 'response_usage_missing_count',
|
|
'response_usage_unpriced_count', 'no_response_attempt_count']:
|
|
if key not in cost or cost[key] != 0:
|
|
return 'accounting_or_provider_issue:' + key
|
|
budget_stop = cost.get('budget_status', {}).get('limit_reached', False)
|
|
if row['return_code'] and not (row.get('timed_out') or budget_stop):
|
|
return 'solver_nonzero_exit_requires_inspection'
|
|
if row['pipeline_status'] not in {'complete_success', 'success_with_warnings'} and not (row.get('timed_out') or budget_stop):
|
|
return 'incomplete_pipeline_requires_inspection'
|
|
return None
|
|
|
|
|
|
def stop(proc):
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
try:
|
|
proc.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
proc.wait()
|
|
|
|
|
|
def command(argv, logfile, timeout):
|
|
with logfile.open('wb') as stream:
|
|
logfile.chmod(0o600)
|
|
proc = subprocess.Popen(argv, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
|
try:
|
|
return proc.wait(timeout=timeout)
|
|
except BaseException:
|
|
stop(proc)
|
|
raise
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--experiment', required=True)
|
|
p.add_argument('--check-only', action='store_true')
|
|
p.add_argument('--probe-id', help='Run one member of the frozen probe independently for diagnosis')
|
|
for name in ['output', 'references', 'image', 'image-sha256', 'apptainer', 'tools-prefix']:
|
|
p.add_argument('--' + name)
|
|
a = p.parse_args()
|
|
experiment = Path(a.experiment).resolve()
|
|
protocol = verify(experiment)
|
|
if a.probe_id is not None and a.probe_id not in protocol['probe_ids']:
|
|
raise ValueError('diagnostic_id_outside_frozen_probe')
|
|
selected_ids = [a.probe_id] if a.probe_id is not None else protocol['probe_ids']
|
|
scope = 'independent_diagnostic' if a.probe_id is not None else 'frozen_probe'
|
|
if a.check_only:
|
|
print(json.dumps({'verified': True, 'probe_ids': protocol['probe_ids'], 'framework_only': True,
|
|
'selected_ids': selected_ids, 'scope': scope}))
|
|
return 0
|
|
if not os.environ.get('SLURM_JOB_ID'):
|
|
raise ValueError('allocated_job_required')
|
|
if not all(getattr(a, name) for name in ['output', 'references', 'image', 'image_sha256', 'apptainer', 'tools_prefix']):
|
|
raise ValueError('runtime_arguments_required')
|
|
out = Path(a.output).resolve()
|
|
out.mkdir(parents=True, exist_ok=False, mode=0o700)
|
|
save(out / 'protocol-snapshot.json', protocol)
|
|
limits = protocol['shared_limits']
|
|
save(out / 'run-identity.json', {
|
|
'job_id': os.environ['SLURM_JOB_ID'], 'protocol_sha256': sha(experiment / 'experiment-protocol.json'),
|
|
'sample_manifest_sha256': sha(experiment / 'sample-manifest.json'),
|
|
'selected_ids': selected_ids, 'scope': scope, 'deadline_seconds': limits['deadline_seconds'],
|
|
'max_generation_requests': limits['max_generation_requests'],
|
|
'source_sha256': {name: sha(ROOT / 'scripts' / name) for name in ['research_pilot.py', 'research_benchmark.py', 'run_apptainer.py']},
|
|
})
|
|
summary = {'job_id': os.environ['SLURM_JOB_ID'], 'selected_ids': selected_ids, 'scope': scope,
|
|
'state': 'running', 'trials': [], 'hold_reason': None}
|
|
save(out / 'summary.json', summary)
|
|
container = ['--image', a.image, '--image-sha256', a.image_sha256,
|
|
'--apptainer', a.apptainer, '--tools-prefix', a.tools_prefix]
|
|
try:
|
|
for ident in selected_ids:
|
|
print(json.dumps({'event': 'task_started', 'id': ident}), flush=True)
|
|
dest = out / ('task-' + ident)
|
|
argv = [sys.executable, '-m', 'scripts.research_benchmark', 'run',
|
|
'--tasks', str(experiment / 'solver/tasks.jsonl'), '--output', str(dest),
|
|
'--config', str(experiment / 'framework-config.private.json'),
|
|
'--limit', '1', '--id', ident, '--deadline-sec', str(limits['deadline_seconds']),
|
|
'--max-generation-requests', str(limits['max_generation_requests']), *container]
|
|
started = time.monotonic()
|
|
code = command(argv, out / (ident + '-solver.log'), limits['deadline_seconds'] + 120)
|
|
predictions = dest / 'predictions.jsonl'
|
|
if not predictions.exists():
|
|
summary['hold_reason'] = 'missing_trial_record:' + ident
|
|
summary['trials'].append({'id': ident, 'solver_exit_code': code, 'elapsed_seconds': round(time.monotonic() - started, 3)})
|
|
break
|
|
rows = [json.loads(x) for x in predictions.read_text().splitlines() if x.strip()]
|
|
if len(rows) != 1 or rows[0]['id'] != ident:
|
|
raise ValueError('prediction_identity_mismatch')
|
|
row = rows[0]
|
|
record = {key: row[key] for key in ['id', 'return_code', 'timed_out', 'elapsed_seconds', 'pipeline_status']}
|
|
record.update(prediction_present=bool(row['prediction'].strip()), cost_summary=row['cost_summary'])
|
|
gradefile = out / (ident + '-scores.jsonl')
|
|
gradecode = command([sys.executable, '-m', 'scripts.research_benchmark', 'score',
|
|
'--tasks', str(experiment / 'solver/tasks.jsonl'), '--references', a.references,
|
|
'--predictions', str(predictions), '--output', str(gradefile),
|
|
'--judge-model', protocol['evaluation']['judge_model']],
|
|
out / (ident + '-judge.log'), 210)
|
|
if gradefile.exists():
|
|
grades = [json.loads(x) for x in gradefile.read_text().splitlines() if x.strip()]
|
|
if len(grades) == 1 and grades[0]['id'] == ident:
|
|
record['grade'] = {k: grades[0].get(k) for k in ['score', 'status', 'usage', 'judge_model']}
|
|
record['judge_exit_code'] = gradecode
|
|
summary['trials'].append(record)
|
|
summary['hold_reason'] = hold_reason(row, limits)
|
|
save(out / 'summary.json', summary)
|
|
print(json.dumps({'event': 'task_finished', 'id': ident, 'pipeline_status': row['pipeline_status'],
|
|
'score': record.get('grade', {}).get('score'), 'hold_reason': summary['hold_reason']}), flush=True)
|
|
if summary['hold_reason']:
|
|
break
|
|
summary['state'] = 'held' if summary['hold_reason'] else 'completed'
|
|
scores = [x.get('grade', {}).get('score') for x in summary['trials']]
|
|
summary['answer_accuracy'] = sum(scores) / len(scores) if len(scores) == len(protocol['probe_ids']) and all(x is not None for x in scores) else None
|
|
save(out / 'summary.json', summary)
|
|
return 4 if summary['hold_reason'] else 0
|
|
except BaseException as exc:
|
|
summary.update(state='interrupted_or_error', error_class=type(exc).__name__)
|
|
save(out / 'summary.json', summary)
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
def interrupted(signum, frame):
|
|
raise SystemExit(128 + signum)
|
|
for sig in [signal.SIGINT, signal.SIGTERM]:
|
|
signal.signal(sig, interrupted)
|
|
raise SystemExit(main())
|