scripts/download_data.py

"""Download a versioned scPerturb count matrix and verify its source checksum."""

from pathlib import Path
import argparse, hashlib, os
import requests

FILES = {
    "norman": ("NormanWeissman2019_filtered.h5ad", "c870e6967d91c017d9da827bab183cd6"),
    "replogle_k562": (
        "ReplogleWeissman2022_K562_essential.h5ad",
        "d8cba17576d1a8afc0f7d71b79cad0f7",
    ),
}


def md5(path):
    h = hashlib.md5()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(8 * 1024**2), b""):
            h.update(block)
    return h.hexdigest()


def download(dataset, output):
    name, digest = FILES[dataset]
    out = Path(output)
    out.mkdir(parents=True, exist_ok=True)
    dest = out / name
    if dest.exists():
        if md5(dest) != digest:
            raise ValueError(f"Checksum mismatch: {dest}")
        return dest
    part = dest.with_suffix(".h5ad.partial")
    offset = part.stat().st_size if part.exists() else 0
    url = f"https://zenodo.org/records/10044268/files/{name}?download=1"
    with requests.get(
        url,
        stream=True,
        headers={"Range": f"bytes={offset}-"} if offset else {},
        timeout=(30, 120),
    ) as response:
        response.raise_for_status()
        append = offset > 0 and response.status_code == 206
        with open(part, "ab" if append else "wb") as f:
            for chunk in response.iter_content(8 * 1024**2):
                f.write(chunk)
    if md5(part) != digest:
        raise ValueError(f"Source checksum mismatch: {part}")
    os.replace(part, dest)
    return dest


if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("dataset", choices=FILES)
    p.add_argument("--output", default="data/raw")
    a = p.parse_args()
    print(download(a.dataset, a.output))