server/lib/tests/every_znippy_download_streams.rs
//! **HOL-RSS part 2: every znippy format's download streams, and so does a delta chunk.**
//!
//! Part 1 (`a_streamed_download_is_the_whole_body_answer`) streamed pip, npm, Maven, apt
//! and dnf. This holds the other formats to the same rule: Helm, gem, Go, conda, Rust,
//! Composer, Docker and NuGet each seal a download that does not compress, one that does
//! (each at least `STREAM_MIN_BYTES`) and a small one; the large ones must stream, and
//! the streamed answer must be the whole-body door's answer (status, headers, bytes).
//!
//! And an archive with a delta map: a superseded generation stored as a delta against
//! the live one must stream too (its delta chunk rebuilt against the base, its checksum
//! checked), and so must the live generation beside it.
#![cfg(all(
feature = "fmt-helm",
feature = "fmt-gem",
feature = "fmt-go",
feature = "fmt-conda",
feature = "fmt-rust",
feature = "fmt-composer",
feature = "fmt-docker",
feature = "fmt-nuget",
feature = "fmt-python"
))]
use std::path::{Path, PathBuf};
use std::sync::Arc;
use traits::RepositoryBackendTrait;
const MIB: usize = 1024 * 1024;
const PIECE: usize = 256 * 1024;
fn noise(len: usize, seed: u64) -> Vec<u8> {
let mut x: u64 = 0x2545_F491_4F6C_DD1D ^ seed;
let mut v = Vec::with_capacity(len + 8);
while v.len() < len {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
v.extend_from_slice(&x.to_le_bytes());
}
v.truncate(len);
v
}
fn text(len: usize) -> Vec<u8> {
b"every znippy download streams, byte for byte\n"
.iter()
.copied()
.cycle()
.take(len)
.collect()
}
/// A sealed archive of `files` under `dir`.
fn seal(dir: &Path, files: &[(String, Vec<u8>)]) -> PathBuf {
let src = dir.join("src");
for (p, b) in files {
let f = src.join(p);
std::fs::create_dir_all(f.parent().unwrap()).unwrap();
std::fs::write(f, b).unwrap();
}
let archive = dir.join("store.znippy");
znippy_compress::compress_dir(&src, &archive, false, None, None, None).unwrap();
std::fs::remove_dir_all(&src).unwrap();
archive
}
/// `url` answered by both doors: the same status, headers and bytes, in pieces.
fn holds(b: &dyn RepositoryBackendTrait, url: &str, want: &[u8], what: &str) {
let (status, headers, whole) = b.handle_http2_request_bytes("GET", url, &[]).unwrap();
assert_eq!(status, 200, "{what}: the whole-body door serves {url}");
assert!(&whole[..] == want, "{what}: the whole body");
let st = b
.stream_get(url, PIECE)
.unwrap()
.unwrap_or_else(|| panic!("{what}: a {} byte download streams", want.len()));
assert_eq!(st.headers, headers, "{what}: the same headers");
assert_eq!(
st.body.len,
Some(want.len() as u64),
"{what}: its length up front"
);
let mut got = Vec::with_capacity(want.len());
for piece in st.body {
let piece = piece.unwrap();
assert!(
piece.len() <= PIECE,
"{what}: a piece is at most what was asked"
);
got.extend_from_slice(&piece);
}
assert!(
got == want,
"{what}: the pieces are the whole body, byte for byte"
);
}
type Open = fn(PathBuf) -> Arc<dyn RepositoryBackendTrait>;
#[test]
fn every_znippy_format_streams_its_downloads() {
// (format, the archive path of a download, its URL, the backend over an archive)
let formats: [(&str, &str, &str, Open); 8] = [
(
"helm",
"charts/demo-1.0.0.tgz",
"/helm/demo-1.0.0.tgz",
|a| {
Arc::new(
repository_znippy_helm::HelmRepoZnippy::with_archive("helm".into(), a).unwrap(),
)
},
),
(
"gem",
"gems/demo-1.0.0.gem",
"/gem/gems/demo-1.0.0.gem",
|a| {
Arc::new(
repository_znippy_gem::GemRepoZnippy::with_archive("gem".into(), a).unwrap(),
)
},
),
(
"go",
"example.com/demo/@v/v1.0.0.zip",
"/go/example.com/demo/@v/v1.0.0.zip",
|a| Arc::new(repository_znippy_go::GoRepoZnippy::with_archive("go".into(), a).unwrap()),
),
(
"conda",
"linux-64/demo-1.0-0.tar.bz2",
"/conda/linux-64/demo-1.0-0.tar.bz2",
|a| {
Arc::new(
repository_znippy_conda::CondaRepoZnippy::with_archive("conda".into(), a)
.unwrap(),
)
},
),
(
"rust",
"demo-1.0.0.crate",
"/crates/crates/demo/1.0.0/download",
|a| {
Arc::new(
repository_znippy_rust::RustRepoZnippy::with_archive("crates".into(), a)
.unwrap(),
)
},
),
(
"composer",
"dist/acme/demo/1.0.0.zip",
"/composer/dist/acme/demo/1.0.0.zip",
|a| {
Arc::new(
repository_znippy_composer::ComposerRepoZnippy::with_archive(
"composer".into(),
a,
)
.unwrap(),
)
},
),
(
"docker",
"v2/acme/demo/blobs/sha256:0123456789abcdef",
"/docker/v2/acme/demo/blobs/sha256:0123456789abcdef",
|a| {
Arc::new(
repository_znippy_docker::DockerRepoZnippy::with_archive("docker".into(), a)
.unwrap(),
)
},
),
(
"nuget",
"demo/1.0.0/demo.1.0.0.nupkg",
"/nuget/v3-flatcontainer/demo/1.0.0/demo.1.0.0.nupkg",
|a| {
Arc::new(
repository_znippy_nuget::NugetRepoZnippy::with_archive("nuget".into(), a)
.unwrap(),
)
},
),
];
for (format, rel, url, open) in formats {
for (what, bytes) in [
("stored", noise(3 * MIB + 17, 1)),
("text", text(5 * MIB + 3)),
] {
let what = format!("{format} {what}");
let dir = tempfile::tempdir().unwrap();
// a small file beside it, in the same directory, that is never streamed
let small_rel = format!("{rel}.small");
let b = open(seal(
dir.path(),
&[
(rel.to_string(), bytes.clone()),
(small_rel, noise(4096, 2)),
],
));
holds(b.as_ref(), url, &bytes, &what);
}
}
}
#[test]
fn a_delta_chunk_streams_rebuilt_against_its_base() {
use traits::znippy_common::meta_sink_append::{read_delta_map, supersede_as_delta};
let dir = tempfile::tempdir().unwrap();
let live = noise(3 * MIB + 5, 7);
let mut old = live.clone();
for i in (0..old.len()).step_by(64 * 1024) {
old[i] ^= 0x5a;
}
let (old_rel, live_rel) = (
"packages/demo/demo-1.0-py3-none-any.whl",
"packages/demo/demo-2.0-py3-none-any.whl",
);
let archive = seal(
dir.path(),
&[
(old_rel.into(), old.clone()),
(live_rel.into(), live.clone()),
],
);
supersede_as_delta(&archive, old_rel, live_rel, 0, 3).unwrap();
assert!(
read_delta_map(&archive)
.unwrap()
.iter()
.any(|(p, _, b)| p == old_rel && b == live_rel),
"the old generation is a delta against the live one"
);
let b = repository_znippy_python::PipRepoZnippy::with_archive("pypi".into(), archive).unwrap();
holds(
&b,
&format!("/pypi/{old_rel}"),
&old,
"the delta generation",
);
holds(
&b,
&format!("/pypi/{live_rel}"),
&live,
"the live generation beside it",
);
}