ChatDev/execorchestra/scripts/prepare_research_pilot.py
2026-11-05 16:22:12 +08:00

75 lines
3.5 KiB
Python

"""Freeze a reproducible task sample using IDs only, without reading answers."""
import argparse
import datetime
import hashlib
import json
from pathlib import Path
def sha(data):
return hashlib.sha256(data).hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
for name in ['tasks', 'source-manifest', 'output']:
parser.add_argument('--' + name, required=True)
parser.add_argument('--seed', default='20261030')
parser.add_argument('--count', type=int, default=15)
parser.add_argument('--probe-count', type=int, default=3)
args = parser.parse_args()
source = json.loads(Path(args.source_manifest).read_text(encoding='utf-8'))
raw = Path(args.tasks).read_bytes()
if sha(raw) != source['solver_inputs_sha256']:
raise ValueError('source_task_file_identity_mismatch')
tasks = [json.loads(line) for line in raw.decode('utf-8').splitlines() if line.strip()]
if len(tasks) != source['count'] or len({x['id'] for x in tasks}) != len(tasks):
raise ValueError('source_count_or_duplicate_id')
if not 0 < args.probe_count <= args.count <= len(tasks):
raise ValueError('invalid_sample_sizes')
for task in tasks:
if set(task) != {'id', 'benchmark', 'query', 'root_url', 'query_sha256'}:
raise ValueError('unexpected_solver_fields')
if sha(task['query'].encode()) != task['query_sha256']:
raise ValueError('query_identity_mismatch')
def rank(task):
key = '\n'.join(['research-pilot-v1', args.seed, source['source_sha256'], task['id']])
return sha(key.encode('utf-8')), task['id']
selected = sorted(tasks, key=rank)[:args.count]
encoded = ''.join(json.dumps(x, ensure_ascii=False) + '\n' for x in selected).encode('utf-8')
manifest = {
'protocol': 'research-pilot-sample-v1',
'created_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'seed': args.seed,
'selection': 'ascending SHA256(research-pilot-v1 + newline + seed + newline + source_sha256 + newline + task_id); task_id breaks ties',
'selection_uses_question_text': False,
'selection_uses_answers_or_outcomes': False,
'source_revision': source['source_revision'],
'source_sha256': source['source_sha256'],
'source_solver_sha256': sha(raw),
'source_count': len(tasks),
'sample_count': len(selected),
'probe_ids': [x['id'] for x in selected[:args.probe_count]],
'sample_ids': [x['id'] for x in selected],
'sample_solver_sha256': sha(encoded),
'selected_queries': [{'id': x['id'], 'query_sha256': x['query_sha256'], 'rank_sha256': rank(x)[0]}
for x in selected],
'public_distribution': 'manifest only; solver task plaintext stays private',
'analysis_scope': 'development pilot; not a held-out final performance estimate',
'replace_failed_tasks': False,
}
output = Path(args.output).resolve()
output.mkdir(parents=True, exist_ok=False, mode=0o700)
solver = output/'solver'
solver.mkdir(mode=0o700)
taskfile = solver/'tasks.jsonl'
taskfile.write_bytes(encoded)
taskfile.chmod(0o600)
(output/'sample-manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'sample_count':len(selected), 'sample_ids':manifest['sample_ids'],
'probe_ids':manifest['probe_ids'], 'sample_solver_sha256':sha(encoded)}))
if __name__ == '__main__':
main()