crates/storage-git-gix/parked-gix/.nornir/bench/transfer-basedist.py
#!/usr/bin/env python3
"""Where do the delta bases of a FETCH's objects actually live?
For each repository and each client cut depth, classify every object the fetch
would send that is stored as a delta in gunnar's pack:
in_set the base is also being sent -- an ordinary OFS_DELTA, no voucher
needed, nothing can go wrong.
client_0_50 / _50_200 / _200_800 / _deep
the base IS in the client's history, at that commit distance
behind the cut. These are the vouchable ones, and the distance is
what decides whether a BOUNDED voucher frontier can reach them.
absent the base is in neither -- reachable on the server from some other
ref, or simply not in this client's history. No voucher can ever
help; only re-choosing the base can.
The distance buckets are the whole point: if the mass sits in `client_0_50` a
bounded frontier already covers it and the write side has nothing to win. If it
sits in `_deep`, choosing a nearer base at gc is worth something the read side
cannot buy without unbounded read amplification.
"""
import subprocess, sys, os, collections
SRC = os.environ.get("SRC", "/home/rickard/git")
STORE = os.environ["STORE"] # a gunnar data root
REPOS = os.environ.get("REPOS", "").split()
DEPTHS = [int(d) for d in os.environ.get("DEPTHS", "50 200").split()]
CHECKPOINTS = [50, 200, 800]
def git(repo, *args):
r = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True)
if r.returncode != 0:
raise RuntimeError(f"git {' '.join(args)} in {repo}: {r.stderr.strip()[:200]}")
return r.stdout
def objects_at(repo, rev):
"""Object ids reachable from `rev`, or None when the history is shorter."""
try:
git(repo, "rev-parse", "--verify", rev + "^{commit}")
except RuntimeError:
return None
return {l.split()[0] for l in git(repo, "rev-list", "--objects", rev).splitlines() if l}
def deltas(store_repo):
"""oid -> base oid, for every delta in the store's packs."""
out = {}
packdir = os.path.join(store_repo, "objects", "pack")
for f in sorted(os.listdir(packdir)):
if not f.endswith(".idx"):
continue
txt = subprocess.run(["git", "verify-pack", "-v", os.path.join(packdir, f)],
capture_output=True, text=True).stdout
for line in txt.splitlines():
p = line.split()
# <sha> <type> <size> <sizeinpack> <offset> [<depth> <base>]
if len(p) >= 7 and p[1] in ("blob", "tree", "commit", "tag"):
out[p[0]] = p[6]
return out
print("repo\tdepth\tsent\tsent_deltas\tin_set\tc0_50\tc50_200\tc200_800\tc_deep\tabsent")
for r in REPOS:
src = os.path.join(SRC, r)
store = os.path.join(STORE, "repos", "bench", f"{r}.git")
if not os.path.isdir(store):
continue
d = deltas(store)
if not d:
print(f"# {r}: no deltas in store", file=sys.stderr)
# Object sets at the checkpoints, once per repo.
cps = {}
for c in CHECKPOINTS:
cps[c] = objects_at(src, f"HEAD~{c}")
allobj = objects_at(src, "HEAD")
for D in DEPTHS:
client = objects_at(src, f"HEAD~{D}")
if client is None:
continue
sent = allobj - client
n = collections.Counter()
sd = 0
for o in sent:
b = d.get(o)
if b is None:
continue
sd += 1
if b in sent:
n["in_set"] += 1
elif b not in client:
n["absent"] += 1
else:
# In the client's history. How far behind the cut?
bucket = "c_deep"
for lo, hi, name in ((D, D + 50, "c0_50"), (D + 50, D + 200, "c50_200"),
(D + 200, D + 800, "c200_800")):
pass
# Distance measured against the checkpoints already computed:
# the base is at least `c` commits back when it survives into
# objects_at(HEAD~c) for the largest such c.
deep = 0
for c in CHECKPOINTS:
if cps[c] is not None and b in cps[c]:
deep = c
if deep <= D:
bucket = "c0_50"
elif deep <= 200:
bucket = "c50_200"
elif deep <= 800:
bucket = "c200_800"
n[bucket] += 1
print(f"{r}\t{D}\t{len(sent)}\t{sd}\t{n['in_set']}\t{n['c0_50']}\t"
f"{n['c50_200']}\t{n['c200_800']}\t{n['c_deep']}\t{n['absent']}")