mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
75 lines
3.5 KiB
Python
75 lines
3.5 KiB
Python
"""Freeze a reproducible task sample using IDs only, without reading answers."""
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
for name in ['tasks', 'source-manifest', 'output']:
|
|
parser.add_argument('--' + name, required=True)
|
|
parser.add_argument('--seed', default='20261030')
|
|
parser.add_argument('--count', type=int, default=15)
|
|
parser.add_argument('--probe-count', type=int, default=3)
|
|
args = parser.parse_args()
|
|
source = json.loads(Path(args.source_manifest).read_text(encoding='utf-8'))
|
|
raw = Path(args.tasks).read_bytes()
|
|
if sha(raw) != source['solver_inputs_sha256']:
|
|
raise ValueError('source_task_file_identity_mismatch')
|
|
tasks = [json.loads(line) for line in raw.decode('utf-8').splitlines() if line.strip()]
|
|
if len(tasks) != source['count'] or len({x['id'] for x in tasks}) != len(tasks):
|
|
raise ValueError('source_count_or_duplicate_id')
|
|
if not 0 < args.probe_count <= args.count <= len(tasks):
|
|
raise ValueError('invalid_sample_sizes')
|
|
for task in tasks:
|
|
if set(task) != {'id', 'benchmark', 'query', 'root_url', 'query_sha256'}:
|
|
raise ValueError('unexpected_solver_fields')
|
|
if sha(task['query'].encode()) != task['query_sha256']:
|
|
raise ValueError('query_identity_mismatch')
|
|
def rank(task):
|
|
key = '\n'.join(['research-pilot-v1', args.seed, source['source_sha256'], task['id']])
|
|
return sha(key.encode('utf-8')), task['id']
|
|
selected = sorted(tasks, key=rank)[:args.count]
|
|
encoded = ''.join(json.dumps(x, ensure_ascii=False) + '\n' for x in selected).encode('utf-8')
|
|
manifest = {
|
|
'protocol': 'research-pilot-sample-v1',
|
|
'created_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
'seed': args.seed,
|
|
'selection': 'ascending SHA256(research-pilot-v1 + newline + seed + newline + source_sha256 + newline + task_id); task_id breaks ties',
|
|
'selection_uses_question_text': False,
|
|
'selection_uses_answers_or_outcomes': False,
|
|
'source_revision': source['source_revision'],
|
|
'source_sha256': source['source_sha256'],
|
|
'source_solver_sha256': sha(raw),
|
|
'source_count': len(tasks),
|
|
'sample_count': len(selected),
|
|
'probe_ids': [x['id'] for x in selected[:args.probe_count]],
|
|
'sample_ids': [x['id'] for x in selected],
|
|
'sample_solver_sha256': sha(encoded),
|
|
'selected_queries': [{'id': x['id'], 'query_sha256': x['query_sha256'], 'rank_sha256': rank(x)[0]}
|
|
for x in selected],
|
|
'public_distribution': 'manifest only; solver task plaintext stays private',
|
|
'analysis_scope': 'development pilot; not a held-out final performance estimate',
|
|
'replace_failed_tasks': False,
|
|
}
|
|
output = Path(args.output).resolve()
|
|
output.mkdir(parents=True, exist_ok=False, mode=0o700)
|
|
solver = output/'solver'
|
|
solver.mkdir(mode=0o700)
|
|
taskfile = solver/'tasks.jsonl'
|
|
taskfile.write_bytes(encoded)
|
|
taskfile.chmod(0o600)
|
|
(output/'sample-manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps({'sample_count':len(selected), 'sample_ids':manifest['sample_ids'],
|
|
'probe_ids':manifest['probe_ids'], 'sample_solver_sha256':sha(encoded)}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|