Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/edda

crates/gunnar-pack/src/write.rs

//! The pack **writer**, used by the server's `upload-pack` and the client's
//! `send-pack`.

use std::io::Write;

use gix_hash::{Kind, ObjectId};

use crate::{checksum::TrailerHasher, Error, ObjectSource, Result, PACK_SIGNATURE, PACK_VERSION};

/// How to write a pack.
///
/// There is deliberately no `Default`: the hash kind must come from the repository
/// being served, never from a default (TOTAL-PROMPT §6).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WriteOptions {
    /// The repository's hash kind. Decides both the width of every object id and the
    /// width of the pack's trailing checksum.
    pub hash_kind: Kind,
    /// zlib deflate level, `0..=9`. Values above 9 are clamped by `flate2`.
    ///
    /// Defaults to 6, which is what `git` uses when `pack.compression` and
    /// `core.compression` are unset. A busy server may prefer 1.
    pub compression_level: u32,
    /// Whether the reader can decode `OBJ_OFS_DELTA` entries, i.e. whether the
    /// client advertised git's `ofs-delta` capability.
    ///
    /// Only [`crate::write_pack_copy`] reads this; [`write_pack`] emits no deltas
    /// at all. When it is `false` a copied delta is named by its base's
    /// **object id** (`OBJ_REF_DELTA`) instead of by its distance backwards
    /// (`OBJ_OFS_DELTA`). The delta bytes are identical either way — only the
    /// header differs — so the reuse survives, which is the entire point:
    /// `OBJ_REF_DELTA` is the older of the two forms and needs no capability,
    /// so there is no reason for a conservative client to be handed a pack with
    /// no delta compression in it.
    ///
    /// Defaults to `true`, which is what every modern `git` asks for.
    pub ofs_delta: bool,
    /// Whether [`crate::write_pack_copy`] may add an **unselected delta base**
    /// to the pack rather than expanding the delta that names it, and the
    /// largest object it may add.
    ///
    /// `None` turns it off. `Some(max)` turns it on and caps a base at `max`
    /// bytes of content. MEASURED on the 19-repository corpus, a single-branch
    /// clone gets **-2.82% wire bytes and -51.9% CPU** for +0.14% objects; see
    /// `crate::copy::base_closure` for the whole table and for the restriction
    /// that makes it safe.
    ///
    /// # Why a ceiling and not a `bool`
    ///
    /// Because the constraints on it are of three different kinds, and
    /// collapsing any two of them has already gone wrong once each.
    ///
    /// * A **negotiated** fetch must pass `None`. On a clone an unselected base
    ///   is an object nobody is getting; on a fetch it is usually an object the
    ///   client already holds, and shipping it is the duplication thin packs
    ///   exist to avoid. MEASURED **+77% wire bytes** on a fetch 8 commits deep
    ///   from a client that withheld `thin-pack`.
    /// * A **filtered** request — `--filter=blob:limit=1k` — has said which
    ///   objects it will accept. An unselected base is by construction an
    ///   object the selection did not choose, and a small blob's delta base is
    ///   usually a large one, so a closure under a filter ships exactly the
    ///   object the filter excluded. That caller passes `None`.
    /// * The server's own object ceiling (`gunnar_wire::Limits::object_bytes`,
    ///   1 GiB by default) is "bytes of a single object gunnar will put in a
    ///   pack". It is `Some` on **every** request, so a caller that folded it
    ///   into the same boolean as the filter turned the closure off
    ///   everywhere — which is what the first version of this field did, and
    ///   what `tests/base_closure.rs` caught by measuring the applied output
    ///   rather than the intent.
    ///
    /// The cap is applied to the object the chain bottoms out at, whose size
    /// its entry header carries. A delta entry's header carries the size of the
    /// *delta stream*, not of the object it reconstructs, so an intermediate
    /// entry in the same chain is bounded only by being another revision of the
    /// same content. At the 1 GiB default that residue is theoretical; at a
    /// deliberately small ceiling it is not, and this says so rather than
    /// implying an exactness it does not have.
    pub base_closure: Option<u64>,
}

impl WriteOptions {
    /// Options for a repository using `hash_kind`, at git's default compression level.
    pub fn new(hash_kind: Kind) -> Self {
        WriteOptions {
            hash_kind,
            compression_level: 6,
            ofs_delta: true,
            base_closure: Some(u64::MAX),
        }
    }

    /// Set the zlib deflate level, `0..=9`.
    pub fn compression_level(mut self, level: u32) -> Self {
        self.compression_level = level;
        self
    }

    /// Declare whether the reader can decode `OBJ_OFS_DELTA`. See
    /// [`WriteOptions::ofs_delta`].
    pub fn ofs_delta(mut self, ofs_delta: bool) -> Self {
        self.ofs_delta = ofs_delta;
        self
    }

    /// Declare whether an unselected delta base may be added to the pack, and
    /// how large it may be. See [`WriteOptions::base_closure`]; a filtered
    /// request must pass `None`.
    pub fn base_closure(mut self, base_closure: Option<u64>) -> Self {
        self.base_closure = base_closure;
        self
    }
}

/// What a completed pack turned out to be.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WriteOutcome {
    /// How many objects the pack contains. Zero is legal.
    pub num_objects: u32,
    /// The pack's identity: the checksum over every byte preceding it, which is also
    /// the pack's trailing bytes and the `pack-<id>.pack` name git gives it.
    pub pack_id: ObjectId,
    /// Total bytes written, header and trailer included.
    pub bytes_written: u64,
    /// Objects whose already-deflated pack entry was **copied** verbatim: no
    /// inflate, no deflate, delta chains preserved (`P-001`).
    ///
    /// This and the next field are `P-001`'s applied output, and they exist for
    /// the same reason [`gunnar_store::RepackOutcome`] carries them: a byte
    /// count and a wall clock cannot tell a pack-copy from a re-deflate. A
    /// serve path that silently stopped copying would still produce a correct
    /// pack, and only these two numbers would say so.
    ///
    /// [`gunnar_store::RepackOutcome`]: https://docs.rs/gunnar-store
    pub objects_copied_from_pack: usize,
    /// Objects this writer deflated from their decompressed content. That is
    /// every object on [`write_pack`]'s path, and on [`crate::write_pack_copy`]'s
    /// path only the ones with no usable pack residency.
    pub objects_recompressed: usize,
    /// Objects emitted as a **newly computed** delta against another object in
    /// the same pack. Only `write_pack_deltified` (the `delta` feature) can
    /// make this non-zero; it is 0 on both other paths, which have no encoder.
    ///
    /// It exists for the same reason as the two fields above, and the reason is
    /// now measured rather than argued. `bytes_written` alone cannot tell a
    /// deltified pack from a whole-object one: on 2026-08-03 the client shipped
    /// 407.08 MiB of pack to a server at **0.0% deltified**, and every byte
    /// count, every wall clock and every correctness test in the tree was green
    /// while it did. `git verify-pack -v` was the only instrument that saw it.
    /// This number is that instrument, on the writing side, where it can go red
    /// in a unit test instead of on a customer's disk.
    pub objects_deltified: usize,
    /// Thin-delta bases `gix` **offered** — entries it copied as an
    /// `OBJ_REF_DELTA` naming a base outside this pack.
    ///
    /// Zero unless [`crate::write_pack_copy_thin`] was asked for a thin pack.
    ///
    /// # Why the offer is counted and not only the refusal
    ///
    /// `objects_recompressed` says how many objects lost their delta, and
    /// cannot say how many *could have kept it*. A voucher that refuses
    /// everything and a transfer that had no thin deltas to begin with produce
    /// the same recompressed count and are opposite situations: one is a
    /// defect worth chasing and the other is nothing to chase. The refusal
    /// *rate* — `thin_bases_vetoed` over this — is the number that tells them
    /// apart, and no other field in this struct can be made to yield it.
    pub thin_bases_offered: usize,
    /// Of those, the ones [`crate::write_pack_copy_thin`]'s `client_has` would
    /// not vouch for, whose objects were therefore written whole.
    ///
    /// A refusal is only *correct* when the receiver really lacks the base; a
    /// voucher that cannot see a base the client holds costs that object its
    /// delta for nothing. This field is what makes that gap measurable — it is
    /// counted here, and whether each refusal was right is a question only the
    /// receiving repository can answer, which is what `GUNNAR_THIN_VETO_LOG`
    /// exists for.
    pub thin_bases_vetoed: usize,
    /// Deltas [`WriteOptions::base_closure`] declined because the base it would
    /// have named is an object this pack **already carries**, at a location in
    /// another pack of a multi-pack store. See
    /// `crate::copy::BaseClosure::already_carried`.
    ///
    /// # Why it is reported and not just avoided
    ///
    /// Because it is the only number that says the closure was *offered* the
    /// case it got wrong. A fixture without it cannot fail a guard about it,
    /// and every fixture in this tree was without it: `git clone --bare`
    /// repacks, a repacked store holds each object once, and one object at one
    /// location cannot be reached by two routes. The guard that watches for the
    /// duplicate has to be able to say the fixture still contains the
    /// condition, and this is what it asks.
    ///
    /// Zero on a single-pack store, on a negotiated fetch and whenever the
    /// closure is off.
    pub closure_bases_declined: usize,
}

/// The compression scratch buffer, allocated once per writer.
///
/// Every compressed byte of every object passes through this and nothing else,
/// so it, and not the largest blob in the repository, is the writer's memory
/// ceiling. 64 KiB is a whole pkt-line's worth of sideband payload, which is
/// the unit the bytes leave on anyway.
const DEFLATE_CHUNK: usize = 64 * 1024;

/// An entry's type-and-size varint never exceeds this, so it is written on the
/// stack. A 64-bit size is at most ten 7-bit groups, and the first group carries
/// only four size bits.
const MAX_ENTRY_HEADER: usize = 16;

/// The same, for a header that may also carry a delta base reference: an
/// `ofs-delta`'s base distance is a second varint of up to ten bytes, and a
/// `ref-delta`'s base id is a whole 32-byte SHA-256. 16 + 32 rounded up.
const MAX_DELTA_ENTRY_HEADER: usize = 64;

/// Streams a packfile out, object by object.
///
/// The object count goes into the 12-byte header and therefore has to be known before
/// the first object is written. That is the format's rule, not this crate's. Use
/// [`write_pack`] when the set of ids is already in hand.
///
/// Every byte written is fed to the trailer hash as it goes, so nothing is buffered in
/// memory. This is what lets `upload-pack` stream a pack straight into a sideband.
///
/// That claim is true **per object on the way out**, and S-015 is what made it
/// so: the deflate of one object runs against a fixed [`DEFLATE_CHUNK`] scratch
/// buffer and each chunk is hashed and written before the next is produced.
/// `tests/streaming.rs` asserts the shape of the byte stream rather than
/// trusting this paragraph.
///
/// # It is **not** true on the way in, and this paragraph used to say it was
///
/// The sentence here read *"a client that asks for a four-gigabyte blob does
/// not make the server hold four gigabytes"*. That was wrong, and wrong in the
/// direction that matters. [`PackWriter::add_object`] takes `data: &[u8]`, and
/// the only way to get one is [`gix_object::Find::try_find`], which fills a
/// caller's `Vec` with the object's **whole inflated content**. So the peak of a
/// `write_pack` is the largest object in the transfer, exactly as it was before
/// S-015 — S-015 halved it (the compressed copy is gone), it did not remove it.
///
/// MEASURED, oden 2026-08-03, a v2 clone of a one-blob repository through
/// `gunnar_wire::UploadPack`, peak RSS of the fetch alone from `VmHWM` with
/// `clear_refs` reset before it:
///
/// | blob | peak RSS of the fetch |
/// |---|---|
/// | 192 MiB | 209 564 kB |
/// | 2048 MiB | 2 127 476 kB |
///
/// One object, one times its inflated size, every time. A ceiling is the only
/// answer available here, because `gix_object::Find` has no streaming form —
/// and the ceiling belongs where the size is still a header rather than a
/// buffer, which is `gunnar_wire::limits`, not this crate. Under it the same
/// 2 GiB fetch peaks at 11 232 kB.
pub struct PackWriter<W: Write> {
    out: W,
    /// The trailer checksum, **not** an object-id hasher. `crate::checksum`
    /// documents why the two roles use different code, and why swapping this
    /// back to `gix_hash::hasher` would cost 5.3× on SHA-1 repositories for no
    /// security property.
    hasher: TrailerHasher,
    hash_kind: Kind,
    declared: u32,
    written: u32,
    /// How many of `written` this writer deflated itself. [`PackWriter::add_object`]
    /// bumps it; [`PackWriter::add_entry`] does not, because an entry arrives
    /// already compressed and only its producer knows whether those bytes were
    /// copied out of a pack or freshly made (`P-001`).
    deflated: u32,
    /// How many of `written` are freshly computed deltas.
    /// [`PackWriter::add_delta`] bumps it and nothing else does.
    deltified: u32,
    bytes: u64,
    /// The deflate state, reset between objects rather than reallocated.
    compress: flate2::Compress,
    /// Compression output scratch, allocated once and reused by every object.
    scratch: Vec<u8>,
}

impl<W: Write> PackWriter<W> {
    /// Begin a pack of exactly `num_objects` objects, writing the 12-byte header
    /// immediately.
    pub fn new(out: W, num_objects: u32, options: &WriteOptions) -> Result<Self> {
        let mut writer = PackWriter {
            out,
            hasher: TrailerHasher::new(options.hash_kind),
            hash_kind: options.hash_kind,
            declared: num_objects,
            written: 0,
            deflated: 0,
            deltified: 0,
            bytes: 0,
            compress: flate2::Compress::new(
                flate2::Compression::new(options.compression_level.min(9)),
                // `true` is the zlib wrapper: the two-byte header and the
                // Adler-32 trailer a pack entry carries. Raw deflate here would
                // produce a pack no git implementation can read.
                true,
            ),
            scratch: vec![0u8; DEFLATE_CHUNK],
        };
        writer.emit(PACK_SIGNATURE)?;
        writer.emit(&PACK_VERSION.to_be_bytes())?;
        writer.emit(&num_objects.to_be_bytes())?;
        Ok(writer)
    }

    /// The hash kind this pack is written with.
    pub fn hash_kind(&self) -> Kind {
        self.hash_kind
    }

    /// How many objects have been written so far.
    pub fn objects_written(&self) -> u32 {
        self.written
    }

    /// How many more objects the header still expects.
    pub fn objects_remaining(&self) -> u32 {
        self.declared - self.written
    }

    /// The pack offset the **next** entry will start at, counted from the
    /// pack's first byte exactly as an `ofs-delta` base distance is.
    ///
    /// This is what makes [`PackWriter::add_entry`] usable for deltas: the
    /// caller records this before each entry and subtracts the base's recorded
    /// offset to get the distance the header carries. The writer does not keep
    /// that table itself, because it is one `(u64, bool)` per object and the
    /// whole-object path in [`write_pack`] never needs it.
    pub fn offset(&self) -> u64 {
        self.bytes
    }

    /// Append one whole (non-delta) object: a type+size varint header followed by its
    /// zlib-deflated content.
    pub fn add_object(&mut self, kind: gix_object::Kind, data: &[u8]) -> Result<()> {
        if self.written == self.declared {
            return Err(Error::PackFull {
                declared: self.declared,
            });
        }

        // On the stack: this used to be a `Vec` allocated once per object, on a
        // path that runs millions of times per clone.
        let mut header = [0u8; MAX_ENTRY_HEADER];
        let header_len = {
            let mut cursor = &mut header[..];
            entry_header(kind)
                .write_to(data.len() as u64, &mut cursor)
                .map_err(Error::Io)?;
            MAX_ENTRY_HEADER - cursor.len()
        };
        self.emit(&header[..header_len])?;

        self.deflate_into_output(data)?;

        self.written += 1;
        self.deflated += 1;
        Ok(())
    }

    /// Append one entry whose payload is **already deflated** (`P-001`).
    ///
    /// This is the whole of pack-copy on the emitting side: `compressed` is the
    /// zlib stream that was already sitting in a packfile (or that gix's
    /// generator produced for an object with no pack residency), and it goes
    /// out with a `memcpy` and a hash. Nothing is inflated and nothing is
    /// deflated, so the delta encoding the pushing client paid for survives
    /// onto the wire.
    ///
    /// `header` may be an `ofs-delta`, in which case its base distance must
    /// already have been computed against [`PackWriter::offset`]. The writer
    /// does not resolve delta bases, it only writes what it is given.
    ///
    /// # Why this exists rather than `gix_pack::data::output::bytes::FromEntriesIter`
    ///
    /// `P-020`. That type's sink is a private `gix_hash::io::Write`, whose
    /// constructor hardcodes `gix_hash::hasher`, so `sha1_checked` with
    /// collision detection on. Using it here would hand the trailer to sha1dc
    /// and give up `P-008`'s landed win in exactly the shape where it was
    /// measured at **4.02×** rather than 1.06×, because pack-copy removes the
    /// deflate that used to dominate. A pack written either way is byte-
    /// identical absent an attack, so no correctness test could ever see it.
    /// See `crate::checksum`.
    pub fn add_entry(
        &mut self,
        header: &gix_pack::data::entry::Header,
        decompressed_size: u64,
        compressed: &[u8],
    ) -> Result<()> {
        if self.written == self.declared {
            return Err(Error::PackFull {
                declared: self.declared,
            });
        }

        let mut buf = [0u8; MAX_DELTA_ENTRY_HEADER];
        let header_len = {
            let mut cursor = &mut buf[..];
            header
                .write_to(decompressed_size, &mut cursor)
                .map_err(Error::Io)?;
            MAX_DELTA_ENTRY_HEADER - cursor.len()
        };
        self.emit(&buf[..header_len])?;
        self.emit(compressed)?;

        self.written += 1;
        Ok(())
    }

    /// Append one entry whose payload is a **freshly computed delta** that this
    /// writer must still deflate.
    ///
    /// The third of the three shapes, and the one that was missing.
    /// [`PackWriter::add_object`] deflates a whole object;
    /// [`PackWriter::add_entry`] copies bytes that are already deflated
    /// (`P-001`); this deflates a delta instruction stream. It exists here,
    /// rather than in the caller, because the deflate is
    /// [`PackWriter::deflate_into_output`] — the same fixed-scratch, streaming,
    /// hashed-as-it-goes emitter every other entry uses (S-015, LAW 5). A
    /// caller that deflated its own delta would be a second compression path
    /// that could drift on level, on wrapper, and on peak memory.
    ///
    /// `payload` is the *uncompressed* delta, and its length is what goes in the
    /// entry header: git's size field on an `OFS_DELTA`/`REF_DELTA` entry is the
    /// size of the delta stream, **not** of the object it reconstructs. The
    /// object's size lives inside the delta's own two leading varints.
    ///
    /// `header` must already name its base: an [`OfsDelta`] carries the
    /// distance backwards from *this* entry's start, which the caller computes
    /// against [`PackWriter::offset`] taken before the call.
    ///
    /// [`OfsDelta`]: gix_pack::data::entry::Header::OfsDelta
    pub fn add_delta(
        &mut self,
        header: &gix_pack::data::entry::Header,
        payload: &[u8],
    ) -> Result<()> {
        debug_assert!(
            header.is_delta(),
            "add_delta was handed a non-delta header; the entry would claim the \
             delta stream's length as the object's size"
        );
        if self.written == self.declared {
            return Err(Error::PackFull {
                declared: self.declared,
            });
        }

        let mut buf = [0u8; MAX_DELTA_ENTRY_HEADER];
        let header_len = {
            let mut cursor = &mut buf[..];
            header
                .write_to(payload.len() as u64, &mut cursor)
                .map_err(Error::Io)?;
            MAX_DELTA_ENTRY_HEADER - cursor.len()
        };
        self.emit(&buf[..header_len])?;

        self.deflate_into_output(payload)?;

        self.written += 1;
        self.deltified += 1;
        Ok(())
    }

    /// Deflate `data` straight into the hasher and the sink, `DEFLATE_CHUNK` at
    /// a time.
    ///
    /// The fields are destructured because `scratch` is borrowed mutably by the
    /// compressor for the length of each call while `out` and `hasher` are
    /// written to. [`PackWriter::emit`] cannot be reused here for that reason,
    /// so the three lines it would have contributed are inlined instead.
    fn deflate_into_output(&mut self, data: &[u8]) -> Result<()> {
        let PackWriter {
            out,
            hasher,
            bytes,
            compress,
            scratch,
            ..
        } = self;

        compress.reset();
        let mut input = data;
        loop {
            let flush = if input.is_empty() {
                flate2::FlushCompress::Finish
            } else {
                flate2::FlushCompress::None
            };
            let (before_in, before_out) = (compress.total_in(), compress.total_out());
            let status = compress
                .compress(input, scratch, flush)
                .map_err(|err| Error::Io(std::io::Error::other(err)))?;
            let consumed = (compress.total_in() - before_in) as usize;
            let produced = (compress.total_out() - before_out) as usize;
            input = &input[consumed..];

            if produced > 0 {
                let chunk = &scratch[..produced];
                hasher.update(chunk);
                out.write_all(chunk).map_err(Error::Io)?;
                *bytes += produced as u64;
            }

            match status {
                flate2::Status::StreamEnd => return Ok(()),
                // Neither read nor wrote: another round would do the same thing
                // forever. zlib does not do this with a 64 KiB output buffer,
                // and a silent infinite loop in the serving path is not a thing
                // to leave undetected if it ever does.
                _ if consumed == 0 && produced == 0 => {
                    return Err(Error::Io(std::io::Error::other(
                        "the deflate stream stopped making progress",
                    )))
                }
                _ => {}
            }
        }
    }

    /// Write the trailing checksum and hand back what was produced.
    ///
    /// Fails if fewer objects were written than the header declared, because that pack
    /// would be truncated and no git implementation would accept it.
    pub fn finish(mut self) -> Result<WriteOutcome> {
        if self.written != self.declared {
            return Err(Error::ObjectCountMismatch {
                declared: self.declared,
                written: self.written,
            });
        }

        let pack_id = self.hasher.finalize()?;
        // The trailer is the hash of everything before it, and is itself not hashed.
        self.out.write_all(pack_id.as_slice()).map_err(Error::Io)?;
        self.bytes += pack_id.as_slice().len() as u64;
        self.out.flush().map_err(Error::Io)?;

        tracing::debug!(
            target: crate::GIT_TARGET,
            objects = self.written,
            bytes = self.bytes,
            pack = %pack_id,
            "wrote packfile"
        );

        Ok(WriteOutcome {
            num_objects: self.written,
            pack_id,
            bytes_written: self.bytes,
            // The writer reports only what it can see for itself. Everything
            // that went through `add_object` was deflated here; everything that
            // went through `add_entry` arrived compressed, and whoever produced
            // it is the only one who knows whether those bytes were copied out
            // of a pack. `write_pack_copy` fills these in from the generator's
            // own tally.
            objects_copied_from_pack: 0,
            objects_recompressed: self.deflated as usize,
            objects_deltified: self.deltified as usize,
            // Same reasoning: only the caller that asked for a thin pack knows
            // whether a base was offered, so the writer reports none.
            thin_bases_offered: 0,
            thin_bases_vetoed: 0,
            closure_bases_declined: 0,
        })
    }

    fn emit(&mut self, bytes: &[u8]) -> Result<()> {
        self.hasher.update(bytes);
        self.out.write_all(bytes).map_err(Error::Io)?;
        self.bytes += bytes.len() as u64;
        Ok(())
    }
}

/// Write a packfile containing exactly `ids`, reading their content from `source`.
///
/// Duplicate ids are written once, in first-seen order. An empty `ids` produces a
/// valid, empty pack.
///
/// Every object is written whole; see the crate docs on deltas.
///
/// # Errors
///
/// * [`Error::ObjectMissing`] if an id is not in `source`. The pack is abandoned
///   rather than silently shortened.
/// * [`Error::HashKindMismatch`] if an id's hash kind is not `options.hash_kind`.
pub fn write_pack<S, W>(
    ids: &[ObjectId],
    source: &S,
    out: W,
    options: &WriteOptions,
) -> Result<WriteOutcome>
where
    S: ObjectSource + ?Sized,
    W: Write,
{
    let (ids, count) = prepare_ids(ids, options)?;

    let mut writer = PackWriter::new(out, count, options)?;
    let mut buf = Vec::with_capacity(8 * 1024);
    for id in &ids {
        let object = source
            .try_find(id, &mut buf)
            .map_err(|source| Error::Find { id: *id, source })?
            .ok_or(Error::ObjectMissing { id: *id })?;
        writer.add_object(object.kind, object.data)?;
    }
    writer.finish()
}

/// [`write_pack`] into a fresh `Vec`, for callers that need the bytes rather than a
/// stream, chiefly tests and the small packs `send-pack` produces.
pub fn write_pack_to_vec<S>(
    ids: &[ObjectId],
    source: &S,
    options: &WriteOptions,
) -> Result<(Vec<u8>, WriteOutcome)>
where
    S: ObjectSource + ?Sized,
{
    let mut out = Vec::new();
    let outcome = write_pack(ids, source, &mut out, options)?;
    Ok((out, outcome))
}

/// Which of `ids` the `source` does not have, in the order given.
///
/// This is the cheap pre-flight for `upload-pack`'s `want` list: a missing want has to
/// be an error *before* a pack starts streaming, because once the sideband is open
/// there is nowhere left to report it.
pub fn missing_objects<S>(ids: &[ObjectId], source: &S) -> Vec<ObjectId>
where
    S: ObjectSource + ?Sized,
{
    ids.iter()
        .filter(|id| !source.exists(id))
        .copied()
        .collect()
}

/// De-duplicate the requested ids and check every one of them belongs to the
/// repository's hash kind, returning the list and the header's object count.
///
/// Shared by [`write_pack`] and [`crate::write_pack_copy`] so the two paths
/// cannot disagree about what a pack of `ids` even contains (LAW 5).
pub(crate) fn prepare_ids(
    ids: &[ObjectId],
    options: &WriteOptions,
) -> Result<(Vec<ObjectId>, u32)> {
    let ids = dedup_preserving_order(ids);
    for id in &ids {
        if id.kind() != options.hash_kind {
            return Err(Error::HashKindMismatch {
                id: *id,
                actual: id.kind(),
                expected: options.hash_kind,
            });
        }
    }
    let count = u32::try_from(ids.len()).map_err(|_| Error::TooManyObjects(ids.len()))?;
    Ok((ids, count))
}

fn dedup_preserving_order(ids: &[ObjectId]) -> Vec<ObjectId> {
    let mut seen = std::collections::HashSet::with_capacity(ids.len());
    let mut out = Vec::with_capacity(ids.len());
    for id in ids {
        if seen.insert(*id) {
            out.push(*id);
        }
    }
    out
}

/// Map an object kind onto the pack entry header gix already knows how to encode.
///
/// Reusing `gix_pack`'s encoder rather than hand-rolling the type+size varint is
/// deliberate: it is the same code path gix's own reader expects (LAW 5).
fn entry_header(kind: gix_object::Kind) -> gix_pack::data::entry::Header {
    use gix_pack::data::entry::Header;
    match kind {
        gix_object::Kind::Blob => Header::Blob,
        gix_object::Kind::Tree => Header::Tree,
        gix_object::Kind::Commit => Header::Commit,
        gix_object::Kind::Tag => Header::Tag,
    }
}

/// The bytes of an empty pack for `hash_kind`: a header declaring zero objects plus
/// the trailer over it.
///
/// A legal pack, and the one a server sends when the client is already up to date.
pub fn empty_pack(hash_kind: Kind) -> Result<Vec<u8>> {
    let mut out = Vec::new();
    PackWriter::new(&mut out, 0, &WriteOptions::new(hash_kind))?.finish()?;
    Ok(out)
}