crates/gunnar-cli/src/control/mirror.rs
//! **`gunnar control mirror` — the pull mirror, driven against a server.**
//!
//! # The one asymmetry this whole module is shaped by
//!
//! `Mirror.Sync` **enqueues**. It puts a job on the server's single mirror
//! runner and answers at once, with `refs_changed = 0` and no updates, because
//! at enqueue nothing has happened yet. A CLI that printed that answer as the
//! outcome would tell an operator *"0 refs changed"* about a fetch that had not
//! started — which is the same sentence a mirror that is genuinely up to date
//! produces, and the difference between them is whether anything is wrong.
//!
//! So every verb here obeys the same rule the rest of `control.rs` already
//! follows (`user add` lists the keys AFTER adding them): **an enqueue is a
//! request echo, and `Status` is the applied output.** `add` reads the mirror
//! back, `sync` prints the job it queued and nothing about refs, and `--wait`
//! is `Status` in a loop with a person at the other end.
//!
//! # `has_progress` is read before the pair, in one place
//!
//! `(0, 0)` with `has_progress = false` is *"the upstream has not said how
//! many objects there are yet"*, not zero percent. [`progress`] is the only
//! function that looks at those three fields, so there is one place that can
//! get it wrong and it is written down (one implementation). A bar drawn from
//! the pair without the flag sits at 0% for a minute and then jumps, and an
//! operator reads that as a hang — which is exactly the moment they kill the
//! fetch that was working.
//!
//! # The secret is never in this file's output, and never in its argv
//!
//! [`ControlMirrorCommand::Credential`] takes a PATH, never a value: an
//! argument is in the shell history, in `ps` for every user on the box, and in
//! the job table. What is read from that path is put into one request field and
//! dropped. Nothing here prints it, and nothing can print it back from the
//! server either — no RPC in the `Mirror` service returns a stored credential,
//! by design, so the read-back below names the KIND and the username and that
//! is the whole of what exists to name.
//!
//! # Where `gunnar-forge` enters, and where it must never
//!
//! [`import`] is the only caller of [`gunnar_forge`] in this binary, and this
//! module is the only file that names it. That crate speaks outbound HTTPS to a
//! forge's REST API and borrows a personal access token for the length of one
//! call. It is reached from the `control` feature alone — see `Cargo.toml` —
//! so the appliance profile (`--no-default-features --features server`) does
//! not resolve it, which is what keeps a bearer credential for somebody's whole
//! GitHub account off a box whose design is that it holds only repositories.
use std::io::Read as _;
use std::path::Path;
use anyhow::{bail, Context, Result};
use gunnar_proto::mirror_client::MirrorClient;
use gunnar_proto::{
MirrorConfig, MirrorCredentialKind, MirrorJob, MirrorJobState, MirrorState, RefUpdate,
};
use crate::cli::ControlMirrorCommand;
use crate::control::{authed, connect, explain, human_bytes, private_key, repo_path, runtime};
/// How often `--wait` asks the server where the job got to.
///
/// The same 500 ms `repo gc --wait` uses, and deliberately the same number: two
/// polling loops in one binary that disagree about their cadence is two
/// behaviours for one promise. A mirror is minutes to hours, so this is not
/// about latency — it is about the line moving often enough that a human can
/// tell a slow fetch from a stopped one.
const POLL: std::time::Duration = std::time::Duration::from_millis(500);
/// `gunnar control mirror …`
pub fn run(command: ControlMirrorCommand) -> Result<()> {
match command {
ControlMirrorCommand::Add {
path,
url,
prune,
create,
adopt,
identity,
at,
} => add(&at, identity.as_deref(), &path, &url, prune, create, adopt),
ControlMirrorCommand::Sync {
path,
wait,
timeout,
identity,
at,
} => sync(&at, identity.as_deref(), &path, wait, timeout),
ControlMirrorCommand::Status {
path,
namespace,
identity,
at,
} => status(
&at,
identity.as_deref(),
path.as_deref(),
namespace.as_deref(),
),
ControlMirrorCommand::Remove { path, identity, at } => {
remove(&at, identity.as_deref(), &path)
}
ControlMirrorCommand::Credential {
path,
ssh_key,
token_file,
username,
forget,
identity,
at,
} => credential(
&at,
identity.as_deref(),
&path,
ssh_key.as_deref(),
token_file.as_deref(),
username.as_deref(),
forget,
),
ControlMirrorCommand::Ca {
path,
ca_file,
forget,
identity,
at,
} => ca(&at, identity.as_deref(), &path, ca_file.as_deref(), forget),
#[cfg(not(feature = "federation"))]
ControlMirrorCommand::Import { .. } => bail!(
"this gunnar-cli has no forge import: listing another forge's repositories is the named \
extra `federation` (`cargo install gunnar-cli --features federation`); `control mirror add` \
mirrors one repository by its URL"
),
#[cfg(feature = "federation")]
ControlMirrorCommand::Import {
from,
token_file,
all,
pick,
into,
ssh,
https,
include_archived,
adopt,
identity,
at,
} => import(
&at,
identity.as_deref(),
&from,
token_file.as_deref(),
all,
pick,
into.as_deref(),
ssh,
https,
include_archived,
adopt,
),
}
}
// ───────────────────────────────────────────────────────────────────────────
// Rendering — one writer each, because two verbs print the same thing
// ───────────────────────────────────────────────────────────────────────────
/// **How far along a running job is, or that there is no denominator yet.**
///
/// The ONE reader of `has_progress`/`progress_done`/`progress_total`. See the
/// module docs: `(0, 0)` with the flag false is "counting", and rendering it as
/// `0/0` — or as 0% — is a claim about a number the server has not got.
fn progress(job: &MirrorJob) -> String {
if !job.has_progress {
return "counting…".to_owned();
}
format!("{}/{} objects", job.progress_done, job.progress_total)
}
/// One mirror job, in the shape every verb that returns one prints it.
///
/// The twin of `control.rs`'s `print_gc_job`, and written for the same reason:
/// `sync`, `sync --wait` and `status` all show a job, and three vocabularies
/// for one state is how an operator ends up unsure whether they are looking at
/// the same run.
fn job_line(job: &MirrorJob) -> String {
match MirrorJobState::try_from(job.state) {
Ok(MirrorJobState::Queued) => {
if job.queue_position > 0 {
format!("{} queued, {} ahead of it", job.job_id, job.queue_position)
} else {
format!("{} queued, next", job.job_id)
}
}
Ok(MirrorJobState::Running) => format!("{} running {}", job.job_id, progress(job)),
Ok(MirrorJobState::Done) => format!(
"{} done ({} objects, {})",
job.job_id,
job.objects,
human_bytes(job.bytes)
),
Ok(MirrorJobState::Failed) => format!("{} FAILED: {}", job.job_id, job.error),
// A state this build has no name for is reported as the number rather
// than guessed at: a newer server saying something is better read as
// "unknown" than as "done".
_ => format!("{} in an unknown state ({})", job.job_id, job.state),
}
}
/// The refs a DONE job moved, one per line.
///
/// **Only ever called for a DONE job**, and that is not a formatting choice. A
/// mirror's whole ref plan is applied in one compare-and-swap or none of it is,
/// so `updates` on a job that has not finished is a list of refs that have NOT
/// moved — printing it would report work that may yet be thrown away.
fn print_updates(updates: &[RefUpdate]) {
for update in updates {
println!(
"updates {} {} -> {}",
update.ref_name,
short(&update.old_oid),
short(&update.new_oid)
);
}
}
/// An oid, short enough to read and long enough to grep.
fn short(oid: &str) -> &str {
if oid.len() > 12 {
&oid[..12]
} else {
oid
}
}
/// The credential's KIND, and never its value — there is nothing else to print.
fn credential_line(mirror: &MirrorConfig) -> String {
match MirrorCredentialKind::try_from(mirror.credential) {
Ok(MirrorCredentialKind::SshKey) => "ssh-key (an OpenSSH private key; for an ssh:// \
remote)"
.to_owned(),
Ok(MirrorCredentialKind::Token) => {
if mirror.credential_username.is_empty() {
"token (sent as Bearer; for an https:// remote)".to_owned()
} else {
format!(
"token (username {}; for an https:// remote)",
mirror.credential_username
)
}
}
Ok(MirrorCredentialKind::Unspecified) => {
"none (dials anonymously — right for a public remote, a 401 for any other)".to_owned()
}
_ => format!("a kind this build has no name for ({})", mirror.credential),
}
}
/// One mirror, in the shape every verb that returns one prints it.
fn print_mirror(mirror: &MirrorConfig) {
match &mirror.repo {
Some(path) => println!("mirror {}/{}", path.namespace, path.repo),
None => println!("mirror (the server named no repository)"),
}
println!("remote {}", mirror.remote_url);
println!(
"prune {}",
if mirror.prune {
"on (refs the upstream deleted are deleted here too)"
} else {
"off (a branch deleted upstream stays here)"
}
);
println!(
"state {}",
match MirrorState::try_from(mirror.state) {
Ok(MirrorState::Idle) => "idle",
Ok(MirrorState::Syncing) => "syncing",
Ok(MirrorState::Failed) => "failed",
_ => "unspecified",
}
);
println!("credential {}", credential_line(mirror));
println!(
"synced {}",
match (mirror.sync_count, mirror.last_sync_at_unix_ms) {
(0, _) => "never".to_owned(),
(n, Some(ms)) => format!("{n} time(s), last at unix_ms {ms}"),
(n, None) => format!("{n} time(s)"),
}
);
// Not the same field as a failed JOB, and both are printed: `last_error` is
// what the previous sync left behind, and a job that is running right now
// has not replaced it yet.
if !mirror.last_error.is_empty() {
println!("last error {}", mirror.last_error);
}
match &mirror.job {
// Not "0 refs": the server is holding no job, which is a different
// fact from a sync that moved nothing.
None => println!("job none; no sync has been asked for recently"),
Some(job) => {
println!("job {}", job_line(job));
if job.state == MirrorJobState::Done as i32 {
print_updates(&job.updates);
}
}
}
}
// ───────────────────────────────────────────────────────────────────────────
// The verbs
// ───────────────────────────────────────────────────────────────────────────
/// One connected, authenticated `Mirror` client, and the runtime under it.
///
/// Every verb needs the same four lines; a struct so `import` — which makes
/// dozens of calls — dials once instead of once per repository.
struct Plane {
rt: tokio::runtime::Runtime,
client: MirrorClient<super::ControlChannel>,
signer: ssh_key::PrivateKey,
}
impl Plane {
fn open(at: &str, identity: Option<&Path>) -> Result<Plane> {
let signer = private_key(identity)?;
let rt = runtime()?;
let channel = connect(&rt, at)?;
Ok(Plane {
rt,
client: MirrorClient::new(channel),
signer,
})
}
fn add(&mut self, path: &str, url: &str, prune: bool, create: bool, adopt: bool) -> Result<()> {
let repo = repo_path(path)?;
self.rt
.block_on(self.client.add(authed(
&self.signer,
gunnar_proto::AddMirrorRequest {
repo: Some(repo),
remote_url: url.to_owned(),
prune,
// The hash kind is NOT passed and there is no flag for it:
// the server takes it from the remote (the wire negotiates it), because a
// mirror of a sha256 repository into a sha1 one is not a
// mirror and a caller guessing would learn that after the
// first fetch.
create_repo: create,
adopt,
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?;
Ok(())
}
fn enqueue(&mut self, path: &str) -> Result<MirrorJob> {
let repo = repo_path(path)?;
let answer = self
.rt
.block_on(self.client.sync(authed(
&self.signer,
gunnar_proto::SyncMirrorRequest { repo: Some(repo) },
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?
.into_inner();
answer.job.ok_or_else(|| {
anyhow::anyhow!("the server accepted the sync of {path} and reported no job")
})
}
/// The stored mirror for one path, applied rather than echoed.
fn one(&mut self, path: &str) -> Result<MirrorConfig> {
let repo = repo_path(path)?;
let mirrors = self
.rt
.block_on(self.client.status(authed(
&self.signer,
gunnar_proto::MirrorStatusRequest {
namespace: String::new(),
repo: Some(repo),
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?
.into_inner()
.mirrors;
mirrors.into_iter().next().ok_or_else(|| {
// A server that answers an empty list rather than NOT_FOUND is
// saying the same thing, and it is said the same way here.
anyhow::anyhow!(
"no mirror is registered for {path}. `gunnar control mirror add {path} <url>` \
registers one."
)
})
}
}
fn add(
at: &str,
identity: Option<&Path>,
path: &str,
url: &str,
prune: bool,
create: bool,
adopt: bool,
) -> Result<()> {
let mut plane = Plane::open(at, identity)?;
plane.add(path, url, prune, create, adopt)?;
// **Read back, do not echo.** `user add` lists the keys after adding one
// for the same reason: what an operator needs to see is the row the server
// is now holding, which is the only thing that will still be true tomorrow.
print_mirror(&plane.one(path)?);
println!();
println!("Nothing has been fetched yet. `gunnar control mirror sync {path} --wait` does that.");
Ok(())
}
fn status(
at: &str,
identity: Option<&Path>,
path: Option<&str>,
namespace: Option<&str>,
) -> Result<()> {
let mut plane = Plane::open(at, identity)?;
let repo = path.map(repo_path).transpose()?;
let mirrors = plane
.rt
.block_on(plane.client.status(authed(
&plane.signer,
gunnar_proto::MirrorStatusRequest {
namespace: namespace.unwrap_or_default().to_owned(),
repo,
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?
.into_inner()
.mirrors;
if mirrors.is_empty() {
match path {
Some(path) => println!("no mirror is registered for {path}"),
None => println!("no mirrors are registered here"),
}
return Ok(());
}
for (index, mirror) in mirrors.iter().enumerate() {
if index > 0 {
println!();
}
print_mirror(mirror);
}
Ok(())
}
fn remove(at: &str, identity: Option<&Path>, path: &str) -> Result<()> {
let mut plane = Plane::open(at, identity)?;
let repo = repo_path(path)?;
plane
.rt
.block_on(plane.client.remove(authed(
&plane.signer,
gunnar_proto::RemoveMirrorRequest { repo: Some(repo) },
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?;
println!("unmirrored {path}");
println!("The repository and everything already fetched into it are untouched;");
println!("only the upstream link and its credential are gone.");
Ok(())
}
/// **Enqueue a sync, and — with `--wait` — stand and watch it.**
///
/// Without `--wait` this prints the job and says where to look, because the
/// enqueue is all that happened. With it, the poll below is the line a human
/// watches, and it exits non-zero on a FAILED job: a zero exit with a failure
/// printed is a script that carries on.
fn sync(at: &str, identity: Option<&Path>, path: &str, wait: bool, timeout: u64) -> Result<()> {
let mut plane = Plane::open(at, identity)?;
let job = plane.enqueue(path)?;
if !wait {
println!("sync {path} enqueued");
println!("job {}", job_line(&job));
println!();
// Said out loud, because the ZERO in the enqueue's answer is the trap:
// it is "I have not started", not "I found nothing".
println!(
"Nothing has been fetched yet — `Sync` only queues. What it did is\n \
gunnar control mirror status {path}"
);
return Ok(());
}
// Said once, up front, because a wait behind six other repositories with
// no explanation looks like a hang — and so does a wait with no end, which
// is why the bound is named here rather than left implicit.
println!(
"waiting for {} on {path} (up to {timeout}s); one mirror sync runs at a time",
job.job_id
);
println!("job {}", job_line(&job));
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout);
let mut last = job;
loop {
match MirrorJobState::try_from(last.state) {
Ok(MirrorJobState::Done) => {
print_updates(&last.updates);
println!("synced {path} ({} refs moved)", last.updates.len());
return Ok(());
}
Ok(MirrorJobState::Failed) => {
// The server's own sentence, unchanged and in full. A refusal
// like "the credential stored for it is an ssh key, and smart
// http cannot carry it" is the whole value of the check that
// wrote it; summarising it as "sync failed" sends somebody off
// to rotate a key that was never the problem.
bail!(
"the mirror sync of {path} FAILED ({}): {}",
last.job_id,
last.error
);
}
_ => {}
}
if std::time::Instant::now() >= deadline {
bail!(
"gave up waiting for {} on {path} after {timeout}s. The job was NOT cancelled — \
it is the server's and it is still running. `gunnar control mirror status \
{path}` says where it got to.",
last.job_id
);
}
std::thread::sleep(POLL);
let seen = plane.one(path)?.job;
match seen {
Some(job) if job.job_id == last.job_id => {
// Only when something moved: a line per poll would bury the
// three that mattered under six hundred that did not.
if job.state != last.state
|| job.has_progress != last.has_progress
|| job.progress_done != last.progress_done
|| job.progress_total != last.progress_total
{
println!("job {}", job_line(&job));
}
last = job;
}
// The job we were waiting on is gone, or has been replaced by
// somebody else's sync. Either way it is no longer ours to report
// on, and inventing an outcome for it would be worse than saying so.
_ => bail!(
"the server stopped reporting job {} for {path}; somebody else may have synced \
it. `gunnar control mirror status {path}` says what it is holding now.",
last.job_id
),
}
}
}
// ───────────────────────────────────────────────────────────────────────────
// The credential
// ───────────────────────────────────────────────────────────────────────────
/// **Read a secret from a path — or from stdin — and from nowhere else.**
///
/// `@<path>` is accepted because that is how every other key argument in this
/// CLI spells a file (`user add --key @~/.ssh/id_ed25519.pub`), and `-` means
/// stdin so a secret can come out of a password manager down a pipe without
/// ever being a file.
///
/// **The trailing newline is stripped.** A token file written by a human ends
/// in one, and a token sent with a newline on it is a credential the forge
/// rejects with a 401 that names nothing. An ssh private key, by contrast,
/// needs its final newline: OpenSSH's armour is line-oriented and a key whose
/// `-----END-----` has no newline after it is refused by some parsers — so the
/// key path re-adds one.
fn secret_from(spec: &str, keep_trailing_newline: bool) -> Result<String> {
let raw = if spec == "-" {
let mut buffer = String::new();
std::io::stdin()
.read_to_string(&mut buffer)
.context("reading the secret from stdin")?;
buffer
} else {
let path = spec.strip_prefix('@').unwrap_or(spec);
std::fs::read_to_string(path).with_context(|| format!("reading the secret from {path}"))?
};
if raw.trim().is_empty() {
bail!("{spec} held no secret; it is empty or only whitespace");
}
let trimmed = raw.trim_end_matches(['\n', '\r']).to_owned();
Ok(if keep_trailing_newline {
format!("{trimmed}\n")
} else {
trimmed
})
}
#[allow(clippy::too_many_arguments)]
fn credential(
at: &str,
identity: Option<&Path>,
path: &str,
ssh_key: Option<&str>,
token_file: Option<&str>,
username: Option<&str>,
forget: bool,
) -> Result<()> {
// Read BEFORE dialling. A missing file is a local mistake and should be a
// local error with the file name in it, not a round trip to a box that may
// be on the other end of an ssh tunnel.
let (kind, secret) = match (ssh_key, token_file, forget) {
(Some(spec), None, false) => (MirrorCredentialKind::SshKey, secret_from(spec, true)?),
(None, Some(spec), false) => (MirrorCredentialKind::Token, secret_from(spec, false)?),
(None, None, true) => (MirrorCredentialKind::Unspecified, String::new()),
_ => bail!(
"say exactly one of --ssh-key <path> (for an ssh:// remote), --token-file <path> \
(for an https:// one) or --forget. There is no --secret <value>: an argument is in \
your shell history and in `ps` for everyone on this box."
),
};
let mut plane = Plane::open(at, identity)?;
let repo = repo_path(path)?;
let answer = plane
.rt
.block_on(plane.client.set_credential(authed(
&plane.signer,
gunnar_proto::SetMirrorCredentialRequest {
repo: Some(repo),
kind: kind as i32,
secret,
username: username.unwrap_or_default().to_owned(),
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?
.into_inner();
// Read back what is STORED. The value is not in this answer and cannot be
// asked for: `MirrorConfig` carries the kind and the username and nothing
// else, which is the whole of the design.
match answer.mirror {
Some(mirror) => println!("credential {}", credential_line(&mirror)),
None => println!(
"credential {} (the server reported no mirror back)",
kind_name(kind)
),
}
if kind == MirrorCredentialKind::Unspecified {
println!("The mirror is still registered; it will dial its upstream anonymously.");
}
Ok(())
}
/// `gunnar control mirror ca <ns/repo> --ca-file <pem> | --forget` (GUN-MIRROR-CA).
fn ca(
at: &str,
identity: Option<&Path>,
path: &str,
ca_file: Option<&str>,
forget: bool,
) -> Result<()> {
let ca_pem = match (ca_file, forget) {
(Some(spec), false) => secret_from(spec, true)?,
(None, true) => String::new(),
_ => bail!("say exactly one of --ca-file <path> (PEM) or --forget"),
};
let mut plane = Plane::open(at, identity)?;
let repo = repo_path(path)?;
let answer = plane
.rt
.block_on(plane.client.set_ca(authed(
&plane.signer,
gunnar_proto::SetMirrorCaRequest {
repo: Some(repo),
ca_pem,
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?
.into_inner();
match answer.mirror {
Some(mirror) if !mirror.ca_subject.is_empty() => println!(
"ca {} sha256 {}",
mirror.ca_subject, mirror.ca_fingerprint_sha256
),
Some(_) => println!("ca none: the public roots only"),
None => println!("ca (the server reported no mirror back)"),
}
Ok(())
}
fn kind_name(kind: MirrorCredentialKind) -> &'static str {
match kind {
MirrorCredentialKind::SshKey => "ssh-key",
MirrorCredentialKind::Token => "token",
MirrorCredentialKind::Unspecified => "none",
}
}
// ───────────────────────────────────────────────────────────────────────────
// import — the door out of another forge
// ───────────────────────────────────────────────────────────────────────────
#[cfg(feature = "federation")]
/// `--from github` / `--from forgejo=https://git.example.com` → a [`Forge`].
///
/// The `=<base>` form is not a convenience: all three of these forges are
/// routinely self-hosted, and the Codeberg-shaped case (a Forgejo at
/// `git.example.com`) is the one gunnar was actually asked for. Without it the
/// self-hosted instance would need a fourth `--from` word per forge.
///
/// The third value is the **host the clone URLs must be on** — see
/// [`check_clone_origin`]. It is derived here, from the spec the operator
/// typed, because this is the only place that knows which forge was asked for;
/// `Forge` deliberately does not hand its base back out.
///
/// [`Forge`]: gunnar_forge::Forge
fn forge_from(spec: &str) -> Result<(gunnar_forge::Forge, &'static str, String)> {
let (name, base) = match spec.split_once('=') {
Some((name, base)) => (name, Some(base.trim_end_matches('/'))),
None => (spec, None),
};
let forge = match (name, base) {
("github", None) => gunnar_forge::Forge::github(),
("github", Some(base)) => gunnar_forge::Forge::github_enterprise(base),
("codeberg", None) => gunnar_forge::Forge::codeberg(),
("codeberg", Some(base)) | ("forgejo", Some(base)) | ("gitea", Some(base)) => {
gunnar_forge::Forge::forgejo(base)
}
("gitlab", None) => gunnar_forge::Forge::gitlab(),
("gitlab", Some(base)) => gunnar_forge::Forge::gitlab_at(base),
("forgejo" | "gitea", None) => bail!(
"--from forgejo needs the instance: `--from forgejo=https://git.example.com`. \
There is no default Forgejo host — codeberg.org is `--from codeberg`."
),
(other, _) => bail!(
"{other:?} is not a forge gunnar can list. The four are `github`, `codeberg`, \
`gitlab` and `forgejo=<url>`; any of the first three take `=<url>` for a \
self-hosted instance."
),
};
// The conventional environment variable for this forge, used when no
// --token-file is given. Named per forge rather than one gunnar variable
// alone, because a developer already has these exported.
let env = match name {
"github" => "GITHUB_TOKEN",
"codeberg" => "CODEBERG_TOKEN",
"gitlab" => "GITLAB_TOKEN",
_ => "FORGEJO_TOKEN",
};
// **Where the clones must come from.** Not the API base: github.com's API
// lives on `api.github.com` while its clone URLs are on `github.com`, and
// those are the same forge. A self-hosted instance serves both from the
// host the operator named.
let host = match (name, base) {
("github", None) => "github.com".to_owned(),
("codeberg", None) => "codeberg.org".to_owned(),
("gitlab", None) => "gitlab.com".to_owned(),
(_, Some(base)) => url_host(base).ok_or_else(|| {
anyhow::anyhow!("{base:?} names no host, so there is nothing to compare a clone URL to")
})?,
// The remaining `(_, None)` pairs all bailed above.
(name, None) => bail!("{name:?} names no host"),
};
Ok((forge, env, host))
}
#[cfg(feature = "federation")]
/// **The host a URL names**, for the three shapes a clone URL comes in:
/// `https://host/path`, `ssh://git@host:port/path` and `git@host:path`.
///
/// Lower-cased, with the userinfo, the port and the path dropped, because the
/// question is only *which machine*. An IPv6 literal keeps its brackets off so
/// two spellings of one address compare equal.
fn url_host(url: &str) -> Option<String> {
let rest = match url.trim().split_once("://") {
Some((_, rest)) => rest,
// The scp-like form. Everything before the first `:` is the authority,
// and a bracketed literal is taken whole.
None => url.trim(),
};
let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
let authority = authority.rsplit('@').next().unwrap_or_default();
let host = match authority.strip_prefix('[') {
Some(inside) => inside.split(']').next().unwrap_or_default(),
None => authority.split(':').next().unwrap_or_default(),
};
(!host.is_empty()).then(|| host.to_ascii_lowercase())
}
#[cfg(feature = "federation")]
/// **Refuse a clone URL that is not on the forge the operator named.**
///
/// `Repo::clone_url_https` and `clone_url_ssh` are copied verbatim out of the
/// forge's own JSON answer, and `import_one` then stores the operator's forge
/// token against whatever they say — a personal access token, typically with
/// broad scope, sent to a host of the forge's choosing. A compromised forge, or
/// a self-hosted one an operator was talked into pointing at, needed only to
/// answer with `clone_url: "https://attacker.example/x.git"`. Nothing anywhere
/// compared the two: `forge_from` returned a `Forge` and an env name, so the
/// base was not even in scope to compare against.
///
/// Host equality and not prefix matching: `github.com.attacker.example` starts
/// with nothing useful, and a subdomain of the forge is still not the forge.
/// The port is not compared — an instance answering its API on 443 and its git
/// on another port is the same machine.
fn check_clone_origin(expected_host: &str, url: &str) -> Result<()> {
let Some(host) = url_host(url) else {
bail!("the forge answered with {url:?}, which names no host at all");
};
if host != expected_host.to_ascii_lowercase() {
bail!(
"the forge answered with a clone URL on {host}, and the forge you named is \
{expected_host}. gunnar will not register a mirror — or send your forge token — to \
a host the forge chose for itself. Nothing was queued for this repository."
);
}
Ok(())
}
#[cfg(feature = "federation")]
/// The forge token, from a file or from the environment — never from argv.
fn forge_token(token_file: Option<&str>, env_name: &str) -> Result<String> {
if let Some(spec) = token_file {
return secret_from(spec, false);
}
for name in ["GUNNAR_FORGE_TOKEN", env_name] {
if let Some(value) = std::env::var_os(name) {
let value = value.to_string_lossy().trim().to_owned();
if !value.is_empty() {
return Ok(value);
}
}
}
bail!(
"no forge token. Pass `--token-file <path>`, or export {env_name} (or \
GUNNAR_FORGE_TOKEN). There is no `--token <value>` on purpose: an argument is in your \
shell history and in `ps` for everyone on this box."
)
}
#[cfg(feature = "federation")]
/// The gunnar namespace the mirrors land in.
///
/// **Not asked of the server, and it cannot be.** The control plane
/// authenticates a KEY — every call signs a fresh `sshsig` — and there is no
/// RPC that names an account back from a fingerprint. So the default is the
/// local login, which is what an account is normally named after, and it is
/// printed so a wrong guess is visible rather than silent.
fn default_namespace() -> Result<String> {
for name in ["USER", "LOGNAME"] {
if let Some(value) = std::env::var_os(name) {
let value = value.to_string_lossy().trim().to_owned();
if !value.is_empty() {
return Ok(value);
}
}
}
bail!(
"there is no --into and no $USER to fall back on. Say which gunnar namespace the \
mirrors land in: `--into <your-account>`."
)
}
#[cfg(feature = "federation")]
/// The rows a `--pick` selection is resolved against.
struct Row {
number: usize,
repo: gunnar_forge::Repo,
}
#[cfg(feature = "federation")]
/// Read the choice from stdin: indices as printed, or names.
///
/// stdin rather than a terminal picker, so the choice is pipeable, keepable in
/// a file and re-runnable. Every token is resolved BEFORE anything is queued —
/// a typo that would have imported the wrong repository is a refusal, not a
/// mirror somebody has to find and remove.
fn pick_from_stdin(rows: &[Row]) -> Result<Vec<usize>> {
let mut buffer = String::new();
std::io::stdin()
.read_to_string(&mut buffer)
.context("reading the choice from stdin")?;
let mut chosen: Vec<usize> = Vec::new();
for line in buffer.lines() {
// `#` starts a comment, so the printed table can be pasted back in.
let line = line.split('#').next().unwrap_or("");
for token in line.split([',', ' ', '\t']).filter(|t| !t.is_empty()) {
let index = if let Ok(number) = token.parse::<usize>() {
rows.iter()
.position(|row| row.number == number)
.ok_or_else(|| {
anyhow::anyhow!(
"there is no row {number} in the list above; nothing was queued"
)
})?
} else {
rows.iter()
.position(|row| {
row.repo.full_name == token
|| row.repo.full_name.rsplit('/').next() == Some(token)
})
.ok_or_else(|| {
anyhow::anyhow!(
"{token:?} is not one of the repositories listed above; nothing was \
queued"
)
})?
};
if !chosen.contains(&index) {
chosen.push(index);
}
}
}
if chosen.is_empty() {
bail!("--pick read nothing from stdin, so nothing was chosen and nothing was queued");
}
// Table order, not the order they were typed: the summary then reads down
// the same list the operator was looking at.
chosen.sort_unstable();
Ok(chosen)
}
#[cfg(feature = "federation")]
fn size_column(repo: &gunnar_forge::Repo) -> String {
match repo.size_kib {
// GitLab reports nothing unless the caller is an administrator asking
// for statistics, so the absence is spelled out rather than shown as 0
// — which would make every GitLab row look empty.
None => "—".to_owned(),
Some(kib) => human_bytes(kib.saturating_mul(1024)),
}
}
#[cfg(feature = "federation")]
#[allow(clippy::too_many_arguments)]
fn import(
at: &str,
identity: Option<&Path>,
from: &str,
token_file: Option<&str>,
all: bool,
pick: bool,
into: Option<&str>,
ssh: bool,
_https: bool,
include_archived: bool,
adopt: bool,
) -> Result<()> {
let (forge, env_name, forge_host) = forge_from(from)?;
let token = forge_token(token_file, env_name)?;
let namespace = match into {
Some(into) => into.to_owned(),
None => default_namespace()?,
};
// ── the LIST, first, and a refusal here queues nothing ─────────────────
//
// Half an import is worse than none: it leaves an operator guessing which
// half. The listing is the first call precisely so a bad token costs one
// HTTP request and no writes at all.
let listed = forge.list_repos(&token).map_err(|err| {
let hint = if err.status() == Some(401) || err.status() == Some(403) {
"\n Nothing was queued. Re-issue the token (it needs read access to the \
repositories you want) and run this again."
} else if err.is_retryable() {
"\n Nothing was queued. This is worth retrying."
} else {
"\n Nothing was queued."
};
anyhow::anyhow!("{err}{hint}")
})?;
let archived = listed.iter().filter(|repo| repo.archived).count();
let rows: Vec<Row> = listed
.into_iter()
.filter(|repo| include_archived || !repo.archived)
.enumerate()
.map(|(index, repo)| Row {
number: index + 1,
repo,
})
.collect();
println!(
"{} {} repositories{}",
forge.kind(),
rows.len(),
match (archived, include_archived) {
(0, _) => String::new(),
(n, false) => format!(" ({n} archived, excluded — --include-archived keeps them)"),
(n, true) => format!(" ({n} archived, included)"),
}
);
if rows.is_empty() {
return Ok(());
}
println!();
let width = rows
.iter()
.map(|row| row.repo.full_name.len())
.max()
.unwrap_or(20)
.max(9);
println!(
" {:>3} {:<width$} {:<8} {:<7} size",
"#", "full_name", "branch", "private"
);
for row in &rows {
println!(
" {:>3} {:<width$} {:<8} {:<7} {}",
row.number,
row.repo.full_name,
if row.repo.default_branch.is_empty() {
"(none)"
} else {
&row.repo.default_branch
},
if row.repo.private { "yes" } else { "no" },
size_column(&row.repo),
);
}
println!();
// ── the choice ─────────────────────────────────────────────────────────
let chosen: Vec<usize> = if all {
(0..rows.len()).collect()
} else if pick {
pick_from_stdin(&rows)?
} else {
// Listing is the safe half and it is the default. Nothing is queued and
// the two ways to queue are named.
println!("Nothing was queued. `--all` mirrors every row above; `--pick` reads the");
println!("numbers or names you want from stdin:");
println!();
println!(
" echo '1 3 5' | gunnar control mirror import --from {from} --pick --into {namespace}"
);
return Ok(());
};
let mut plane = Plane::open(at, identity)?;
let mut queued = 0usize;
let mut failures: Vec<(String, String)> = Vec::new();
// **Said before anything is stored, not after.** The per-repository line
// below prints the URL once the registration has already happened, so on
// its own it tells an operator where their token went rather than where it
// is going. One line, before the first write, naming the one host every
// clone — and, over HTTPS, the token — is allowed to reach.
if !ssh {
println!();
println!("Your {from} token will be sent to {forge_host} and to nowhere else.");
println!();
}
for index in chosen {
let repo = &rows[index].repo;
let name = repo.full_name.rsplit('/').next().unwrap_or(&repo.full_name);
let path = format!("{namespace}/{name}");
let url = if ssh {
&repo.clone_url_ssh
} else {
&repo.clone_url_https
};
// **The destination is the forge's own answer**, so it is checked
// against the forge before it is written down or credentialled.
if let Err(err) = check_clone_origin(&forge_host, url) {
println!("REFUSED {path} <- {url}");
println!(" {err:#}");
failures.push((path, format!("{err:#}")));
continue;
}
// Per repository, and a failure does not stop the ones after it: an
// account with one repository gunnar cannot take is still an account
// worth moving.
match import_one(&mut plane, &path, url, ssh, &token, adopt) {
Ok(()) => {
println!("queued {path} <- {url}");
queued += 1;
}
Err(err) => {
println!("FAILED {path} <- {url}");
println!(" {err:#}");
failures.push((path, format!("{err:#}")));
}
}
}
println!();
println!("import {queued} queued, {} failed", failures.len());
if ssh && queued > 0 {
// The one thing this command owes an operator that it cannot do for
// them: the key belongs to the UPSTREAM forge, not to gunnar.
println!();
println!("--ssh stored NO credential. Each of these dials over ssh, so a deploy key");
println!("must be enrolled on the UPSTREAM repository — that forge's business, not");
println!("gunnar's — and gunnar given its private half:");
println!();
println!(" gunnar control mirror credential {namespace}/<name> --ssh-key <path>");
println!();
println!("Until then a sync of a PRIVATE upstream will fail on authentication.");
}
if !failures.is_empty() {
bail!(
"{} of the chosen repositories were not queued; the reasons are above",
failures.len()
);
}
Ok(())
}
#[cfg(feature = "federation")]
/// One repository: register, credential it if it needs one, enqueue.
///
/// `Add` on a mirror that is already there is a no-op and a second `Sync` while
/// one is live is the same job, so this is safe to run again over an account
/// that is already mirrored — which is what makes `import --all` the verb an
/// operator re-runs after adding one repository upstream.
fn import_one(
plane: &mut Plane,
path: &str,
url: &str,
ssh: bool,
token: &str,
adopt: bool,
) -> Result<()> {
plane.add(path, url, false, true, adopt)?;
if !ssh {
// The SAME token the listing used: the forge that enumerated these
// repositories is the forge that will serve the clone, so a second
// credential would be a second thing to rotate.
let repo = repo_path(path)?;
plane
.rt
.block_on(plane.client.set_credential(authed(
&plane.signer,
gunnar_proto::SetMirrorCredentialRequest {
repo: Some(repo),
kind: MirrorCredentialKind::Token as i32,
secret: token.to_owned(),
// The username every forge's personal access token wants
// over HTTP Basic. GitHub documents this exact literal;
// Codeberg and GitLab accept any non-empty user with a
// token as the password.
username: "x-access-token".to_owned(),
},
)?))
.map_err(|status| anyhow::anyhow!("{}", explain(&status)))?;
}
plane.enqueue(path)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// **The host, out of every shape a clone URL arrives in** — and out of the
/// parts that are not the host, which is where a comparison like this gets
/// fooled.
#[test]
fn a_clone_url_names_the_machine_it_would_be_fetched_from() {
for (url, want) in [
("https://github.com/octocat/x.git", "github.com"),
("http://GitHub.COM/octocat/x.git", "github.com"),
("git@github.com:octocat/x.git", "github.com"),
("ssh://git@github.com:2222/octocat/x.git", "github.com"),
("https://git.example.com:8443/ns/x.git", "git.example.com"),
("ssh://git@[2606:4700::1111]:22/ns/x.git", "2606:4700::1111"),
// The userinfo is not the host. A URL whose *user* is the forge's
// name is the oldest trick there is.
(
"https://github.com@attacker.example/x.git",
"attacker.example",
),
] {
assert_eq!(url_host(url).as_deref(), Some(want), "{url}");
}
assert_eq!(url_host("https:///x.git"), None, "no host at all");
}
/// **A clone URL off the forge is refused**, which is the whole finding:
/// the URL is copied verbatim out of the forge's JSON and the operator's
/// token is stored against it.
#[test]
fn a_clone_url_the_forge_invented_is_refused() {
check_clone_origin("github.com", "https://github.com/octocat/x.git")
.expect("the forge's own host");
check_clone_origin("github.com", "git@github.com:octocat/x.git").expect("over ssh");
for url in [
"https://attacker.example/octocat/x.git",
// A suffix is not the host, and neither is a subdomain.
"https://github.com.attacker.example/x.git",
"https://evil.github.com.attacker.example/x.git",
"git@attacker.example:octocat/x.git",
"https://github.com@attacker.example/x.git",
] {
let err = check_clone_origin("github.com", url)
.expect_err("a host the forge chose for itself");
let err = format!("{err:#}");
assert!(err.contains("github.com"), "{url}: {err}");
assert!(err.contains("Nothing was queued"), "{url}: {err}");
}
}
/// The host a forge's clones must be on, per `--from` spelling. Notably
/// **not** the API base: github.com answers its API on `api.github.com`.
#[test]
fn each_forge_names_the_host_its_clones_come_from() {
for (spec, want) in [
("github", "github.com"),
("codeberg", "codeberg.org"),
("gitlab", "gitlab.com"),
(
"github=https://github.example.com/api/v3",
"github.example.com",
),
("forgejo=https://git.example.com", "git.example.com"),
("gitlab=https://gitlab.example.com/", "gitlab.example.com"),
] {
let (_, _, host) = forge_from(spec).unwrap_or_else(|err| panic!("{spec}: {err}"));
assert_eq!(host, want, "{spec}");
}
}
}