mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
35 lines
2.0 KiB
Python
35 lines
2.0 KiB
Python
"""Download official public data at resolved revisions; keep plaintext private."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import httpx
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__); p.add_argument('--output',required=True); a=p.parse_args()
|
|
root=Path(a.output).resolve(); root.mkdir(parents=True,exist_ok=False,mode=0o700)
|
|
manifest={}
|
|
with httpx.Client(proxy=os.environ.get('SGAR_TOOL_HTTPS_PROXY') or None,trust_env=False,timeout=60,follow_redirects=True) as c:
|
|
r=c.get('https://api.github.com/repos/xbench-ai/xbench-evals/commits/main'); r.raise_for_status(); rev=r.json()['sha']
|
|
r=c.get(f'https://raw.githubusercontent.com/xbench-ai/xbench-evals/{rev}/data/DeepSearch-2510.csv'); r.raise_for_status()
|
|
path=root/'xbench-encrypted.csv'; path.write_bytes(r.content); path.chmod(0o600)
|
|
manifest['xbench']={'repository':'xbench-ai/xbench-evals','revision':rev,'sha256':hashlib.sha256(r.content).hexdigest()}
|
|
r=c.get('https://huggingface.co/api/datasets/callanwu/WebWalkerQA'); r.raise_for_status(); webrev=r.json()['sha']
|
|
from datasets import load_dataset
|
|
rows=load_dataset('callanwu/WebWalkerQA',split='main',revision=webrev,cache_dir=str(root/'cache'))
|
|
path=root/'webwalker.jsonl'
|
|
path.write_text(''.join(json.dumps(dict(x),ensure_ascii=False)+'\n' for x in rows),encoding='utf-8'); path.chmod(0o600)
|
|
manifest['webwalker']={'repository':'callanwu/WebWalkerQA','revision':webrev,'split':'main','count':len(rows),'sha256':hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
for bench,file in [('xbench','xbench-encrypted.csv'),('webwalker','webwalker.jsonl')]:
|
|
subprocess.run([sys.executable,'-m','scripts.research_benchmark','prepare','--benchmark',bench,
|
|
'--source',str(root/file),'--revision',manifest[bench]['revision'],'--output',str(root/bench)],check=True)
|
|
(root/'download-manifest.json').write_text(json.dumps(manifest,indent=2))
|
|
print(json.dumps(manifest),flush=True)
|
|
|
|
|
|
if __name__=='__main__': main()
|