crates/storage-git-gix/parked-gix/.nornir/assets/deltarow.py
#!/usr/bin/env python3
"""Round 2: strip the pack entry header (redundant with the columns), and measure
the per-push batch granularity a real gunnar push would produce."""
import sys, os, io, zlib
import pyarrow as pa
import pyarrow.ipc as ipc
vp, packpath = sys.argv[1], sys.argv[2]
rows = []
for line in open(vp):
f = line.split()
if len(f) < 5 or len(f[0]) != 40:
continue
try: int(f[0], 16)
except ValueError: continue
rows.append((f[0], f[1], int(f[2]), int(f[3]), int(f[4]),
int(f[5]) if len(f) > 6 else 0, f[6] if len(f) > 6 else None))
rows.sort(key=lambda r: r[4])
pack = open(packpath, 'rb').read()
packlen = len(pack)
def entry_header_len(buf, off):
"""Length of the pack entry header at `off`: type+size varint, plus the
OFS_DELTA varint or the 20-byte REF_DELTA base oid."""
i = off
typ = (buf[i] >> 4) & 7
while buf[i] & 0x80: i += 1
i += 1
if typ == 6: # OFS_DELTA
while buf[i] & 0x80: i += 1
i += 1
elif typ == 7: # REF_DELTA
i += 20
return i - off
payloads, hdrs = [], 0
for (oid, typ, size, insize, off, depth, base) in rows:
h = entry_header_len(pack, off)
hdrs += h
payloads.append(pack[off + h: off + insize])
# RED-FIRST CONTROL: every payload must inflate to exactly `inflated_size` for a
# non-delta object. If the header arithmetic is wrong the slices are garbage and
# every byte figure below is meaningless.
checked = bad = 0
for (r, p) in zip(rows, payloads):
if r[6] is None and r[1] in ('blob', 'commit', 'tree', 'tag'):
try:
if len(zlib.decompress(p)) != r[2]: bad += 1
except Exception: bad += 1
checked += 1
if checked >= 400: break
print(f"control: {checked} base-object payloads inflated, {bad} wrong "
f"-> {'SLICES ARE REAL' if bad == 0 else 'SLICING IS BROKEN'}")
n = len(rows)
ptot = sum(len(p) for p in payloads)
print(f"pack {packlen:,} B objects={n} entry headers {hdrs:,} B "
f"({hdrs/packlen:.2%}) deflate payloads {ptot:,} B")
def build(payload=True, cols_extra=True):
c = {'oid': pa.array([bytes.fromhex(r[0]) for r in rows], pa.binary(20)),
'base_oid': pa.array([bytes.fromhex(r[6]) if r[6] else None for r in rows],
pa.binary(20))}
if cols_extra:
c['kind'] = pa.array([r[1] for r in rows]).dictionary_encode()
c['inflated_size'] = pa.array([r[2] for r in rows], pa.uint64())
if payload:
c['payload'] = pa.array(payloads, pa.large_binary())
return pa.table(c)
def sz(tbl, codec=None, chunk=None):
if chunk:
tbl = pa.Table.from_batches(tbl.combine_chunks().to_batches(max_chunksize=chunk))
sink = io.BytesIO()
with ipc.new_stream(sink, tbl.schema,
options=ipc.IpcWriteOptions(compression=codec)) as w:
w.write_table(tbl)
return len(sink.getvalue())
t = build()
print()
print(f"{'shape':<46}{'bytes':>14} x pack")
for label, b in [("git pack (today, stored opaque in znippy)", packlen),
("arrow delta-per-row, headers stripped, 1 batch", sz(t)),
(" + zstd on the buffers", sz(t, 'zstd'))]:
print(f"{label:<46}{b:>14,} {b/packlen:.4f}")
print()
print("batch granularity — a real push adds ~10 objects:")
one = sz(t)
for chunk in (13431, 4096, 256, 32, 10, 1):
b = sz(t, None, chunk if chunk < n else None)
nb = (n + chunk - 1)//chunk
print(f" {chunk:>6} obj/batch {nb:>6} batches {b:>13,} B "
f"{b/packlen:.4f}x pack framing +{b-one:,} B ({(b-one)/one:.2%})")
print()
idx = packpath[:-5] + '.idx'
m = build(payload=False)
print(f"metadata-only projection (oid, base_oid, kind, inflated_size): "
f"{sz(m):,} B = {sz(m)/n:.1f} B/obj")
print(f" zstd: {sz(m,'zstd'):,} B = {sz(m,'zstd')/n:.1f} B/obj")
if os.path.exists(idx):
i = os.path.getsize(idx)
print(f" the .idx that exists today: {i:,} B = {i/n:.1f} B/obj "
f"(no base_oid, no kind, no size)")
print(f" -> the delta DAG as a projectable column costs {sz(m)/i:.2f}x the .idx"
f" ({sz(m,'zstd')/i:.2f}x zstd)")