File size: 2,164 Bytes
a484e22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | #!/usr/bin/env bash
# Download the benchmark data used by the SpecMem harness into data/.
#
# We do not redistribute any third-party benchmark; this script fetches each
# file directly from its official source. See data/README.md for provenance
# and licenses, and for the two datasets that need a manual step
# (ToolBench: Google Drive; tau2-bench: generated from the tau2-bench repo).
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
DATA="$ROOT/data"
GH_RAW="https://raw.githubusercontent.com"
fetch() { # fetch <url> <dest>
mkdir -p "$(dirname "$2")"
if [ -s "$2" ]; then echo " [skip] $2 (exists)"; return; fi
echo " [get ] $2"
curl -fsSL --retry 3 "$1" -o "$2"
}
echo "== BFCL v4 (Berkeley Function-Call Leaderboard, Apache-2.0) =="
BFCL="$GH_RAW/ShishirPatil/gorilla/main/berkeley-function-call-leaderboard/bfcl_eval/data"
for f in BFCL_v4_simple_python.json BFCL_v4_multiple.json BFCL_v4_parallel.json; do
fetch "$BFCL/$f" "$DATA/bfcl/$f"
done
echo "== Seal-Tools (Apache-2.0) =="
SEAL="$GH_RAW/fairyshine/Seal-Tools/master/Seal-Tools_Dataset"
fetch "$SEAL/tool.jsonl" "$DATA/sealtools/tool.jsonl"
fetch "$SEAL/test_in_domain.jsonl" "$DATA/sealtools/test_in_domain.jsonl"
echo "== ToolAlpaca (Apache-2.0) =="
TA="$GH_RAW/tangqiaoyu/ToolAlpaca/main/data"
fetch "$TA/eval_simulated.json" "$DATA/toolalpaca/eval_simulated.json"
fetch "$TA/eval_real.json" "$DATA/toolalpaca/eval_real.json"
echo "== API-Bank (MIT; HF dataset liminghao1630/API-Bank) =="
AB="https://huggingface.co/datasets/liminghao1630/API-Bank/resolve/main/test-data"
fetch "$AB/level-1-api.json" "$DATA/apibank/level-1-api.json"
fetch "$AB/level-2-api.json" "$DATA/apibank/level-2-api.json"
echo
echo "Done. Two datasets need a manual step (only for the corresponding experiments):"
echo " * ToolBench -> data/toolbench/test_instruction/G{1,2,3}_*.json"
echo " from the official OpenBMB/ToolBench Google Drive release."
echo " * tau2-bench -> data/tau2/tools_{airline,retail,telecom}.json"
echo " generated from github.com/sierra-research/tau2-bench;"
echo " see data/README.md."
|