#!/usr/bin/env python3 """Build the HuggingFace dataset package from the GitHub release. Usage: python3 build_hf.py [out_dir] Source: https://github.com/sarkar-dipankar/on-device-auction-audit (commit 5a6ad86). Stdlib only and deterministic: files are walked in sorted order, rows keep their source order, JSON is re-serialised with a fixed key order, and every column keeps one type across all splits of a config (the HF viewer rejects mixed-type columns). Running it twice gives byte-identical output. Outputs (under out_dir, default: the directory holding this script): data/contexts/part-000N.jsonl data/opportunities_teacher_50k.jsonl, split into 2 shards on line boundaries. The shards are byte-exact slices: concatenated in order they reproduce the source file and its sha256 (16ad2a17...) recorded in every manifest. data//.jsonl results/*/*.jsonl, one typed value per column, nested values as JSON strings raw/results, raw/scripts exact copies of the release directories raw/data/ipinyou_1458.json exact copy raw/SHA256SUMS, raw/LICENSE exact copies (the MIT notice travels with raw/scripts) The release ships no simulator source. Any *.tar* / *.tgz file in the source tree aborts the build, so a withheld source snapshot can never be packaged. """ import hashlib import json import os import shutil import sys CONTEXTS = "data/opportunities_teacher_50k.jsonl" CONTEXT_SHA256 = "16ad2a17e55bb2185bfe4633521c70c9c03962131bc430358cc13df018ecbbf8" CONTEXT_SHARDS = 2 MAX_FILE_BYTES = 8 * 1024 * 1024 # per-file cap of our upload path SYN = "results/track_b_smoke" # synthetic fixed-bid demand ("smoke" is the run's legacy name) IPY = "results/track_b_ipinyou" # iPinYou-parameterised log-normal demand # (config, [(split, source file)]) CONFIGS = [ ("primary_replicates", [("synthetic", f"{SYN}/replicates.jsonl"), ("ipinyou", f"{IPY}/replicates.jsonl")]), ("pressure_balance", [("train", "results/track_b_pressure_balance/replicates.jsonl")]), ("bursty_heterogeneous", [("train", "results/track_b_bursty_heterogeneous/replicates.jsonl")]), ] + [ (name, [("synthetic", f"{SYN}/{name}.jsonl"), ("ipinyou", f"{IPY}/{name}.jsonl")]) for name in ("replicate_summary", "per_campaign", "calibration", "factorial_ablation", "factorial_calibration", "sensitivity") ] + [ ("label_perturbation", [("synthetic", f"{SYN}/label_perturbation.jsonl")]), ("label_perturbation_calibration", [("synthetic", f"{SYN}/label_perturbation_calibration.jsonl")]), ] def jstr(obj): """Compact, deterministic JSON string for nested values.""" return json.dumps(obj, ensure_ascii=False, separators=(",", ":")) def read_jsonl(path): # split on \n only: str.splitlines() would also break on U+2028/U+2029 with open(path, encoding="utf-8", newline="") as f: return [json.loads(line) for line in f.read().split("\n") if line.strip()] def kind(v): if v is None: return None if isinstance(v, bool): return "bool" if isinstance(v, int): return "int" if isinstance(v, float): return "float" return "str" def normalise(split_rows): """Give every split of a config the same columns, in first-seen order. Missing keys become null, dict/list values become JSON strings. A column holding both int and float is widened to float. Any other type mix is an error, so a schema change in the source fails the build loudly. """ cols = [] for rows in split_rows: for r in rows: for k in r: if k not in cols: cols.append(k) out = [[{c: (jstr(r[c]) if isinstance(r.get(c), (dict, list)) else r.get(c)) for c in cols} for r in rows] for rows in split_rows] for c in cols: kinds = {kind(r[c]) for rows in out for r in rows} - {None} if kinds == {"int", "float"}: for rows in out: for r in rows: if r[c] is not None: r[c] = float(r[c]) elif len(kinds) > 1: raise SystemExit(f"column {c!r} mixes types {sorted(kinds)}") return out def write_jsonl(path, rows): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8", newline="\n") as f: for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n") return len(rows) def shard_contexts(src, data): with open(os.path.join(src, CONTEXTS), "rb") as f: blob = f.read() if hashlib.sha256(blob).hexdigest() != CONTEXT_SHA256: raise SystemExit(f"{CONTEXTS}: sha256 differs from the manifests") lines = blob.split(b"\n") if lines[-1] != b"": raise SystemExit(f"{CONTEXTS}: no trailing newline") lines = lines[:-1] per = -(-len(lines) // CONTEXT_SHARDS) outdir = os.path.join(data, "contexts") os.makedirs(outdir, exist_ok=True) for i in range(CONTEXT_SHARDS): part = lines[i * per:(i + 1) * per] with open(os.path.join(outdir, f"part-{i:04d}.jsonl"), "wb") as f: f.write(b"\n".join(part) + b"\n") print(f"data/contexts/part-{i:04d}.jsonl: {len(part)} rows (byte-exact slice of {CONTEXTS})") def copy_raw(src, out): raw = os.path.join(out, "raw") shutil.copytree(os.path.join(src, "results"), os.path.join(raw, "results")) shutil.copytree(os.path.join(src, "scripts"), os.path.join(raw, "scripts"), ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) os.makedirs(os.path.join(raw, "data")) shutil.copyfile(os.path.join(src, "data", "ipinyou_1458.json"), os.path.join(raw, "data", "ipinyou_1458.json")) for name in ("SHA256SUMS", "LICENSE"): shutil.copyfile(os.path.join(src, name), os.path.join(raw, name)) def refuse_archives(src): for root, dirs, files in os.walk(src): dirs[:] = sorted(d for d in dirs if d != ".git") for name in sorted(files): if ".tar" in name or name.endswith((".tgz", ".zip")): raise SystemExit(f"refusing to build: archive present: {os.path.join(root, name)}") def main(): if len(sys.argv) < 2: sys.exit(__doc__) src = os.path.abspath(sys.argv[1]) out = os.path.abspath(sys.argv[2]) if len(sys.argv) > 2 else os.path.dirname(os.path.abspath(__file__)) refuse_archives(src) for sub in ("data", "raw"): if os.path.exists(os.path.join(out, sub)): shutil.rmtree(os.path.join(out, sub)) data = os.path.join(out, "data") shard_contexts(src, data) for config, splits in CONFIGS: normed = normalise([read_jsonl(os.path.join(src, p)) for _, p in splits]) for (split, p), rows in zip(splits, normed): n = write_jsonl(os.path.join(data, config, f"{split}.jsonl"), rows) print(f"data/{config}/{split}.jsonl: {n} rows from {p}") copy_raw(src, out) print("raw/: results/, scripts/, data/ipinyou_1458.json, SHA256SUMS, LICENSE copied") for root, _, files in os.walk(out): for name in files: p = os.path.join(root, name) if os.path.getsize(p) > MAX_FILE_BYTES: raise SystemExit(f"{p} exceeds {MAX_FILE_BYTES} bytes") if __name__ == "__main__": main()