Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .minimal/minimal.toml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ path = "usr/share/microvm-rootfs/rootfs.img"
type = "oci-image"
packages = ["libkrun"]

[session]
packages = ["just", "protobuf"]

[tasks.vim]
profile = "demo"
state_key = ""
Expand Down
34 changes: 28 additions & 6 deletions crates/common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,18 +208,40 @@ pub fn hardlink_dir_contents(src: &Path, dst: &Path) -> Result<(), HardlinkError
} else if metadata.is_file() {
match fs::hard_link(&path, &dst_path) {
Ok(()) => Ok(()),
Err(e) => {
if e.kind() == std::io::ErrorKind::AlreadyExists {
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
warn!(
"Not linking {} => {}, already exists",
path.display(),
dst_path.display()
);
Ok(())
}
// The cache and destination can live on different filesystems
// (e.g. a per-VM `/state` volume vs. the rootfs holding
// `/home`), where hardlinks are impossible (`EXDEV`). Fall back
// to a copy so materialization still succeeds — slower and no
// longer deduplicated, but correct.
Err(e) if e.kind() == std::io::ErrorKind::CrossesDevices => {
// Every file in a cross-device tree hits EXDEV, so warn only
// on the first — a per-file log would flood with thousands
// of identical lines. Once-per-process is enough: the cause
// is a fixed filesystem-layout fact, not a per-file
// condition.
use std::sync::atomic::{AtomicBool, Ordering};
static WARNED: AtomicBool = AtomicBool::new(false);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Statics make me sad but I guess theres no way to do this cleaner without it being more verbose than its worth

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I don't really like it either. I don't think we want the log spammed with every single file being copied though. There's probably a better way to do this, but we don't have a ton of time today.

if !WARNED.swap(true, Ordering::Relaxed) {
warn!(
"Not linking {} => {}, already exists",
"Copying instead of hardlinking: cache and \
destination are on different filesystems; further \
cross-device copies this run are silent (first: {} \
=> {})",
path.display(),
dst_path.display()
);
Ok(())
} else {
Err(e)
}
fs::copy(&path, &dst_path).map(|_| ())

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Im also not sure how I feel about this, maybe we are far enough along that its better to be slow and fallback to a copy, but also my other thought is this might mask when your paths are wrong

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good point. This is really only required for nested sandboxes, and there may be a better way. We should probably change it to be a preference, but I'll cut an issue for it given timing.

}
Err(e) => Err(e),
}
.map_err(|e| HardlinkError::HardlinkFailed(path.to_path_buf(), dst_path, e))?;
} else if metadata.is_symlink() {
Expand Down
43 changes: 39 additions & 4 deletions crates/diagnostics/src/net.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,18 @@ pub async fn listening_sockets<W: BundleSink>(
/// The `/proc/net` socket tables, in the order a reader wants them.
pub const PROC_NET_SOCKET_TABLES: &[&str] = &["tcp", "tcp6", "udp", "udp6", "unix"];

/// The `/proc/net` table that stands in for `ip addr`/`ifconfig` when a host
/// ships no net tools: the interface list (`/proc/net/dev`). It carries names
/// and stats but no addresses or MACs — the picture a process has with nothing
/// installed, and the same table the in-microVM guest collector captures.
pub const PROC_NET_INTERFACE_TABLES: &[&str] = &["dev"];

/// The `/proc/net` tables that stand in for `ip route`/`netstat -rn` on a
/// stripped host: the IPv4 routing table plus the fib trie, which also carries
/// the local address picture `ip addr` gives where tools exist. Hex-encoded;
/// the dev team decodes.
pub const PROC_NET_ROUTE_TABLES: &[&str] = &["route", "fib_trie"];

/// `<dest>/net/<name>.txt`: the named raw `/proc/net` tables, concatenated
/// verbatim behind a `=== /proc/net/<table> ===` banner each.
///
Expand Down Expand Up @@ -99,8 +111,20 @@ pub async fn interfaces<W: BundleSink>(
} else {
&[("ifconfig", &["-a"])]
};
let (banner, out) = first_available(attempts).await?;
let text = format!("{banner}\n{}", mask_macs(&out));
let text = match first_available(attempts).await {
Ok((banner, out)) => format!("{banner}\n{}", mask_macs(&out)),
// A stripped host (no `ip`/`ifconfig`) still gets the interface picture
// `/proc/net` carries, mirroring `listening_sockets`. No MACs live in
// `/proc/net/dev`, so the mask pass is a no-op — kept to honor the
// `Redaction::Keys` label uniformly across both paths.
#[cfg(target_os = "linux")]
Err(cmd_err) => format!(
"(commands unavailable: {cmd_err})\n{}",
mask_macs(&proc_net_text(PROC_NET_INTERFACE_TABLES).await)
),
#[cfg(not(target_os = "linux"))]
Err(cmd_err) => return Err(cmd_err.into()),
};
w.add_bytes(
&format!("{dest}/net/interfaces.txt"),
text.as_bytes(),
Expand All @@ -121,10 +145,21 @@ pub async fn routes<W: BundleSink>(
} else {
&[("netstat", &["-rn"])]
};
let (banner, out) = first_available(attempts).await?;
let text = match first_available(attempts).await {
Ok((banner, out)) => format!("{banner}\n{out}"),
// On a stripped host fall back to the raw routing tables, mirroring
// `listening_sockets`; `fib_trie` also carries the local address picture.
#[cfg(target_os = "linux")]
Err(cmd_err) => format!(
"(commands unavailable: {cmd_err})\n{}",
proc_net_text(PROC_NET_ROUTE_TABLES).await
),
#[cfg(not(target_os = "linux"))]
Err(cmd_err) => return Err(cmd_err.into()),
};
w.add_bytes(
&format!("{dest}/net/routes.txt"),
format!("{banner}\n{out}").as_bytes(),
text.as_bytes(),
Redaction::None,
)
.await
Expand Down
18 changes: 18 additions & 0 deletions crates/minimal/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1873,9 +1873,27 @@ async fn attach_to_session(
let [strict, known_hosts_file] = host_key_opts(&sock.with_file_name(paths::KNOWN_HOSTS_FILE));

let mut ssh = std::process::Command::new("ssh");
// Pin the shell ssh uses to run the ProxyCommand. ssh launches a
// ProxyCommand via `$SHELL -c` and execs `$SHELL` with no PATH lookup, so a
// caller whose `$SHELL` is a bare name (`fish`) or points at a shell absent
// from this context fails with "<shell>: No such file or directory" and the
// transport dies at "banner exchange … Broken pipe". Our ProxyCommand is a
// full-path `min proxy …` that needs nothing but a POSIX `sh`, so force the
// always-present `/bin/sh` rather than inherit the user's interactive shell.
ssh.env("SHELL", "/bin/sh");
ssh.env("MINIMAL_SESSION_ID", id.to_string()).args([
"-o",
"SendEnv=MINIMAL_SESSION_ID",
// Forward the user's locale and timezone into the session, mirroring a
// conventional `SendEnv LANG LC_* TZ`. The daemon accepts only these
// (its `AcceptEnv` allowlist) and folds them in below any loadout.
// `TERM` needs no `SendEnv`: ssh always carries it in the PTY request.
"-o",
"SendEnv=LANG",
"-o",
"SendEnv=LC_*",
"-o",
"SendEnv=TZ",
"-o",
&format!("ProxyCommand={proxy_cmd}"),
"-o",
Expand Down
17 changes: 14 additions & 3 deletions crates/minimald/src/diag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,14 +322,25 @@ async fn build_bundle(
collect_step!(
w,
"net.interfaces",
diagnostics::net::proc_net_tables(&mut w, "", "interfaces", &["dev"])
diagnostics::net::proc_net_tables(
&mut w,
"",
"interfaces",
diagnostics::net::PROC_NET_INTERFACE_TABLES
)
);
// Routing *and* addresses: `fib_trie` carries the local address picture the
// host's `ip addr` gives on the other side of the switch.
// host's `ip addr` gives on the other side of the switch. Same tables the
// host collector falls back to, so the two captures stay comparable.
collect_step!(
w,
"net.routes",
diagnostics::net::proc_net_tables(&mut w, "", "routes", &["route", "fib_trie"])
diagnostics::net::proc_net_tables(
&mut w,
"",
"routes",
diagnostics::net::PROC_NET_ROUTE_TABLES
)
);
if s.in_microvm().await {
collect_step!(w, "net.gvproxy", gvproxy_probe(&mut w));
Expand Down
2 changes: 1 addition & 1 deletion crates/minimald/src/net/switch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,7 @@ fn blocked_syn(frame: &[u8], allowed: &HashSet<u16>) -> Option<(u16, SocketAddrV
return None;
}
let (syn, ack) = (pkt.tcp_flags & 0x02 != 0, pkt.tcp_flags & 0x10 != 0);
if !(syn && !ack) || allowed.contains(&pkt.dst.port()) {
if !syn || ack || allowed.contains(&pkt.dst.port()) {
return None;
}
Some((pkt.dst.port(), pkt.src))
Expand Down
112 changes: 109 additions & 3 deletions crates/minimald/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,21 @@ pub(crate) fn registry_name(record: &Record) -> String {
}
}

/// The server-side `AcceptEnv` allowlist: locale and timezone vars a client is
/// permitted to forward from its shell into the session (OpenSSH's default
/// `AcceptEnv LANG LC_*`, plus `TZ`). Everything else the client set on the
/// channel — e.g. `MINIMAL_SESSION_ID`, `TRACEPARENT` — is control plumbing and
/// must not leak into the shell environment, so it is filtered out here.
fn inherited_session_env(
channel_env: &std::collections::BTreeMap<String, String>,
) -> Vec<(String, String)> {
channel_env
.iter()
.filter(|(k, _)| k.as_str() == "LANG" || k.as_str() == "TZ" || k.starts_with("LC_"))
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
}

/// An error that occurred when attaching to a running session/its-shell.
#[derive(Debug)]
pub enum AttachError {
Expand Down Expand Up @@ -1093,6 +1108,31 @@ impl Session {
None => return Err(AttachError::NoPty),
});

// Capture the environment this attach contributes to the shell it may
// mint: the locale/timezone vars the client forwarded (folded as
// defaults below the composition) and the per-connection facts folded
// above it. Currently the only connection fact is `TERM`, from the
// client's PTY request.
//
// `SSH_TTY` and `SSH_CONNECTION`/`SSH_CLIENT` are intentionally omitted:
// the session sandbox has no host `/dev/pts` and the transport is a
// local Unix socket (no peer IP/port), so any value would name something
// that doesn't exist in-session and would only mislead audit logs,
// source-IP checks, or `$SSH_TTY` consumers.
let attach_env = {
let inherited = inherited_session_env(&config.env_vars);
let mut connection = Vec::new();
if let Some(pty) = config.pty.as_ref()
&& !pty.term.is_empty()
{
connection.push(("TERM".to_string(), pty.term.clone()));
}
session_host::AttachEnv {
inherited,
connection,
}
};

// A session that was created but never had its loadout configured has
// nothing in flight, so attaching to it shouldn't be an error: set it
// up now, with an empty contribution, and carry on into the attach.
Expand Down Expand Up @@ -1175,15 +1215,15 @@ impl Session {
};
match host {
None => {
self.mint_session_host(session_hnd, conn_username, channel, sz)
self.mint_session_host(session_hnd, conn_username, channel, sz, attach_env)
.await
}
Some((h, _)) => {
match h.attach(channel, sz).await {
Ok(()) => Ok(()),
Err((channel, sz)) => {
// session host is dead
self.mint_session_host(session_hnd, conn_username, channel, sz)
self.mint_session_host(session_hnd, conn_username, channel, sz, attach_env)
.await
}
}
Expand All @@ -1197,6 +1237,7 @@ impl Session {
conn_username: String,
channel: Channel<Msg>,
sz: WinSize,
attach_env: session_host::AttachEnv,
) -> Result<(), AttachError> {
let record = self.record.record().await.unwrap();
let paths = self.paths().await;
Expand All @@ -1206,7 +1247,9 @@ impl Session {
let control = SessionControl::new(self.manager.clone(), record.id);

// Spawn/setup the session in a closure that reports progress down the terminal.
let launcher = self.session_launcher(session_hnd, &record).await?;
let launcher = self
.session_launcher(session_hnd, &record, attach_env)
.await?;
let progress = ChannelProgress::new(channel, self.tracker.clone(), (sz.cols, sz.rows));
let (channel, spawned) = progress
.run(Box::pin(session_host::Host::spawn(
Expand Down Expand Up @@ -1253,6 +1296,7 @@ impl Session {
&mut self,
session: SessionHandle,
record: &Record,
attach_env: session_host::AttachEnv,
) -> Result<session_host::SandboxLauncher, AttachError> {
// R2.1: reject a policy that is incompatible with the network mode
// (e.g. egress on a non-`OwnIp` PTask) before launching the host.
Expand All @@ -1269,6 +1313,7 @@ impl Session {
.context(true)
.await
.map_err(AttachError::ContextCreationFailed)?,
attach_env,
network_mode,
net_switch: Arc::clone(&self.net_switch),
ingress,
Expand All @@ -1287,6 +1332,7 @@ impl Session {
&mut self,
_session: SessionHandle,
record: &Record,
_attach_env: session_host::AttachEnv,
) -> Result<session_host::MockLauncher, AttachError> {
// Mirror the production R2.1 gate so test launches reject a
// policy/network-mode mismatch the same way production does.
Expand Down Expand Up @@ -1722,6 +1768,66 @@ mod tests {

use crate::test_harness::{TestClient, TestServer, create_configured_session};

/// The `AcceptEnv` allowlist keeps locale + timezone vars and drops
/// everything else — critically the control-plane vars, which must never
/// reach the shell environment.
#[test]
fn inherited_session_env_keeps_only_locale_and_tz() {
let env: std::collections::BTreeMap<String, String> = [
("LANG", "en_US.UTF-8"),
("LC_CTYPE", "en_US.UTF-8"),
("LC_ALL", "C"),
("TZ", "America/New_York"),
("MINIMAL_SESSION_ID", "00000000-0000-0000-0000-000000000000"),
("TRACEPARENT", "00-abc-def-01"),
("PATH", "/evil/bin"),
("PS1", "# "),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();

let kept: std::collections::BTreeMap<String, String> =
super::inherited_session_env(&env).into_iter().collect();

assert_eq!(kept.get("LANG").map(String::as_str), Some("en_US.UTF-8"));
assert_eq!(
kept.get("LC_CTYPE").map(String::as_str),
Some("en_US.UTF-8")
);
assert_eq!(kept.get("LC_ALL").map(String::as_str), Some("C"));
assert_eq!(kept.get("TZ").map(String::as_str), Some("America/New_York"));
// Control-plane routing/tracing vars and everything else must be dropped.
assert!(!kept.contains_key("MINIMAL_SESSION_ID"));
assert!(!kept.contains_key("TRACEPARENT"));
assert!(!kept.contains_key("PATH"));
assert!(!kept.contains_key("PS1"));
assert_eq!(kept.len(), 4, "only LANG, LC_*, and TZ should survive");
}

/// A `LC_`-*prefixed* var is accepted, but a bare `LC` (or one that merely
/// contains `LC_`) is not — the filter is a prefix match, not a substring.
#[test]
fn inherited_session_env_prefix_not_substring() {
let env: std::collections::BTreeMap<String, String> = [
("LC_MESSAGES", "C"),
("LC", "nope"),
("MYLC_VAR", "nope"),
("XLANG", "nope"),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();

let kept: std::collections::BTreeMap<String, String> =
super::inherited_session_env(&env).into_iter().collect();

assert_eq!(
kept.keys().cloned().collect::<Vec<_>>(),
vec!["LC_MESSAGES"]
);
}

/// Reads the session record for `id`, or `None` once it has been deleted.
async fn record_exists(client: &mut TestClient, id: SessionId) -> bool {
client
Expand Down
Loading