From f940afa7baaf23a92f6c84490ee4fb3e754de833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=88=CE=BB=CE=BB=CE=B5=CE=BD=20=CE=95=CE=BC=CE=AF=CE=BB?= =?UTF-8?q?=CE=B9=CE=B1=20=CE=86=CE=BD=CE=BD=CE=B1=20Zscheile?= Date: Tue, 28 Apr 2026 17:17:45 +0200 Subject: [PATCH 1/2] feat(fs/mem): tarfs implementation This implements the internal Hermit image mode. The decompressed Hermit image gets passed from the bootloader (hermit-loader) or hypervisor (uhyve) to the kernel, the location is announced via the FDT property `/chosen/linux,initrd-*`, and the kernel parses the contained tar (expected in ustar format) file, placing its content at the filesystem root, similar to an initrd. The access permissions get copied from the tar archive, directories get default permissions. --- Cargo.lock | 12 +++++++ Cargo.toml | 8 +++++ src/fs/mem.rs | 88 +++++++++++++++++++++++++++++++++++++-------- src/fs/mod.rs | 16 ++++++++- src/mm/mod.rs | 72 +++++++++++++++++++++++++++++++++++-- xtask/src/clippy.rs | 2 +- 6 files changed, 178 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 567f8062c0..6d60240a01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,6 +885,7 @@ dependencies = [ "smoltcp", "take-static", "talc", + "tar-no-std", "thiserror", "time", "tock-registers 0.10.1", @@ -1846,6 +1847,17 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar-no-std" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715f9a4586706a61c571cb5ee1c3ac2bbb2cf63e15bce772307b95befef5f5ee" +dependencies = [ + "bitflags 2.13.0", + "log", + "num-traits", +] + [[package]] name = "thiserror" version = "2.0.19" diff --git a/Cargo.toml b/Cargo.toml index c73b89e5e8..62ef107d13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,6 +156,13 @@ semihosting = ["dep:semihosting"] ## _application processor_s (AP) from the _boot-strap processor_ (BSP). smp = ["acpi"] +## Enables (internal) hermit image support. +## +## This allows packaging a kernel together with all its static environment files into a .tar.gz archive, +## whose decompressed version gets handled by the kernel +## (the decompression gets handled by the bootloader / hypervisor). +hermit-image = ["dep:tar-no-std"] + #! ### Network Features ## Enables TCP support. @@ -346,6 +353,7 @@ simple-shell = { version = "0.0.1", optional = true } smallvec = { version = "1", features = ["const_new"] } take-static = "0.1" talc = { version = "5" } +tar-no-std = { version = "0.4", optional = true, features = ["alloc"] } thiserror = { version = "2", default-features = false } time = { version = "0.3", default-features = false } volatile = "0.6" diff --git a/src/fs/mem.rs b/src/fs/mem.rs index dde335f13c..91a888ec10 100644 --- a/src/fs/mem.rs +++ b/src/fs/mem.rs @@ -122,15 +122,6 @@ pub(crate) struct RamFileInner { pub attr: FileAttr, } -impl RamFileInner { - pub fn new(attr: FileAttr) -> Self { - Self { - data: Vec::new(), - attr, - } - } -} - pub struct RamFileInterface { /// Position within the file pos: Mutex, @@ -354,6 +345,10 @@ impl VfsNode for RamFile { impl RamFile { pub fn new(mode: AccessPermission) -> Self { + Self::new_with_data(Vec::new(), mode) + } + + fn new_with_data(data: Vec, mode: AccessPermission) -> Self { let microseconds = arch::kernel::systemtime::now_micros(); let t = timespec::from_usec(microseconds as i64); let attr = FileAttr { @@ -365,7 +360,7 @@ impl RamFile { }; Self { - data: Arc::new(RwLock::new(RamFileInner::new(attr))), + data: Arc::new(RwLock::new(RamFileInner { data, attr })), } } } @@ -468,6 +463,64 @@ impl MemDirectory { } } + #[cfg(feature = "hermit-image")] + pub fn try_from_image(image: &'static [u8]) -> io::Result { + let this = Self::new(AccessPermission::S_IRUSR); + + let tar_archive_ref = tar_no_std::TarArchiveRef::new(image).map_err(|e| { + error!("[Hermit image] Tar file has invalid format: {e:?}"); + Errno::Inval + })?; + + for entry in tar_archive_ref.entries() { + let filename = entry.filename(); + let filename = filename.as_str().map_err(|e| { + error!( + "[Hermit image] Tar entry has not supported filename (non UTF-8): {filename:?}; {e}", + ); + Errno::Inval + })?; + if filename.is_empty() { + continue; + } + debug!("[Hermit image] Processing tar entry: {filename}"); + + let mode = entry.posix_header().mode.to_flags().map_err(|e| { + error!( + "[Hermit image] Tar entry {filename:?} has invalid mode: {:?}; {e}", + entry.posix_header().mode, + ); + Errno::Inval + })?; + let mode = AccessPermission::from_bits(mode.bits() as u32).ok_or_else(|| { + error!("[Hermit image] Tar entry {filename:?} has invalid mode: {mode:?}"); + Errno::Inval + })?; + + for (i, _) in filename.match_indices("/") { + let part = &filename[..i]; + if this.traverse_lstat(part).is_err() { + this.traverse_mkdir( + part, + AccessPermission::S_IRUSR + | AccessPermission::S_IWUSR + | AccessPermission::S_IRGRP, + ) + .inspect_err(|e| { + error!("[Hermit image] Unable to mkdir {part:?}: {e}"); + })?; + } + } + + this.traverse_create_file(filename, entry.data(), mode) + .inspect_err(|e| { + error!("[Hermit image] Unable to write entry {filename:?}: {e}"); + })?; + } + + Ok(this) + } + async fn async_traverse_open( &self, path: &str, @@ -701,11 +754,16 @@ impl VfsNode for MemDirectory { return directory.traverse_create_file(rest, data, mode); } - let file = RomFile::new(data, mode); - self.inner - .write() - .await - .insert(component.to_owned(), Box::new(file)); + let file: Box = if mode.contains(AccessPermission::S_IWUSR) + || mode.contains(AccessPermission::S_IWGRP) + || mode.contains(AccessPermission::S_IWOTH) + { + Box::new(RamFile::new_with_data(data.to_vec(), mode)) + } else { + Box::new(RomFile::new(data, mode)) + }; + + self.inner.write().await.insert(component.to_owned(), file); Ok(()) }, None, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 8ef50ad0c7..e34da34856 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -307,7 +307,21 @@ pub(crate) fn init() { const VERSION: &str = env!("CARGO_PKG_VERSION"); const UTC_BUILT_TIME: &str = build_time::build_time_utc!(); - let root_filesystem = Filesystem::new(); + #[cfg_attr(not(feature = "hermit-image"), expect(unused_mut))] + let mut root_filesystem = Filesystem::new(); + + // Handle optional Hermit Image specified in FDT. + #[cfg(feature = "hermit-image")] + if let Some(tar_image) = crate::mm::hermit_tar_image() { + root_filesystem.root = + MemDirectory::try_from_image(tar_image).expect("Unable to parse Hermit Image"); + } + + if crate::mm::hermit_tar_image().is_some() { + error!( + "Kernel built without Hermit image support, but a Hermit image was supplied: ignoring" + ); + } root_filesystem .mkdir("/tmp", AccessPermission::from_bits(0o777).unwrap()) diff --git a/src/mm/mod.rs b/src/mm/mod.rs index e31e44e53f..068c4a03d3 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -46,6 +46,7 @@ mod physicalmem; mod virtualmem; use core::alloc::Layout; +use core::cmp; use core::mem::MaybeUninit; use core::ops::Range; @@ -110,9 +111,56 @@ pub(crate) fn claim_initial_heap() { } } +/// Physical and virtual address range of the Hermit image, in case it is present +/// (indicated via FDT). +static HERMIT_IMAGE_START_AND_LEN: Lazy> = Lazy::new(|| { + let fdt = crate::env::fdt()?; + + // per FDT specification, /chosen always exists + let chosen = fdt.find_node("/chosen").unwrap(); + + let fdt::node::NodeProperty { value: addr, .. } = chosen.property("linux,initrd-start")?; + + let fdt::node::NodeProperty { + value: addr_end, .. + } = chosen.property("linux,initrd-end")?; + + let (addr_, addr_end_) = (addr.try_into(), addr_end.try_into()); + + let ret = if let (Ok(addr), Ok(addr_end)) = (addr_, addr_end_) { + let addr = usize::from_be_bytes(addr); + let addr_end = usize::from_be_bytes(addr_end); + info!("Hermit image at {addr:x} - {addr_end:x}"); + addr_end.checked_sub(addr).map(|len| { + ( + VirtAddr::from_ptr(core::ptr::with_exposed_provenance::(addr)), + len, + ) + }) + } else { + None + }; + + if ret.is_none() { + error!( + "Hermit image supplied with invalid address range (#addr = {}, #addr_end = {})", + addr.len(), + addr_end.len(), + ); + } + + ret +}); + +pub(crate) fn hermit_tar_image() -> Option<&'static [u8]> { + // technically, the following is UB, because the kernel might be contained within... + HERMIT_IMAGE_START_AND_LEN + .map(|(addr, len)| unsafe { core::slice::from_raw_parts(addr.as_ptr(), len) }) +} + #[cfg(target_os = "none")] pub(crate) fn init() { - use crate::arch::mm::paging; + use arch::mm::paging; Lazy::force(&KERNEL_ADDR_RANGE); @@ -120,12 +168,30 @@ pub(crate) fn init() { arch::mm::init(); } + Lazy::force(&HERMIT_IMAGE_START_AND_LEN); + let total_mem = physicalmem::total_memory_size(); let kernel_addr_range = KERNEL_ADDR_RANGE.clone(); + let reserved_addr_range = if let Some((image_start, image_len)) = *HERMIT_IMAGE_START_AND_LEN { + cmp::min( + kernel_addr_range.start, + image_start.align_down(LargePageSize::SIZE), + ) + ..cmp::max( + kernel_addr_range.end, + (image_start + image_len).align_up(LargePageSize::SIZE), + ) + } else { + kernel_addr_range.clone() + }; info!("Total memory size: {} MiB", total_mem >> 20); info!( "Kernel region: {:p}..{:p}", - kernel_addr_range.start, kernel_addr_range.end + kernel_addr_range.start, kernel_addr_range.end, + ); + info!( + "Locally reserved region: {:p}..{:p}", + reserved_addr_range.start, reserved_addr_range.end, ); // we reserve physical memory for the required page tables @@ -158,7 +224,7 @@ pub(crate) fn init() { // we reserve at least 75% of the memory for the user space let reserve: usize = (avail_mem * 75) / 100; // 64 MB is enough as kernel heap - let reserve = core::cmp::min(reserve, 0x0400_0000); + let reserve = cmp::min(reserve, 0x0400_0000); let virt_size: usize = reserve.align_down(LargePageSize::SIZE as usize); let layout = PageLayout::from_size_align(virt_size, LargePageSize::SIZE as usize).unwrap(); diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index de1a6997e5..4448e910a6 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -21,7 +21,7 @@ impl Clippy { clippy().run()?; clippy().arg("--features=common-os").run()?; clippy() - .arg("--features=acpi,dns,fsgsbase,pci,smp,vga") + .arg("--features=acpi,dns,fsgsbase,pci,smp,vga,hermit-image") .run()?; clippy().arg("--no-default-features").run()?; clippy().arg("--all-features").run()?; From 5bf7fd4a9ced419a18cdcd7f139f5e9fa2f0c135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=88=CE=BB=CE=BB=CE=B5=CE=BD=20=CE=95=CE=BC=CE=AF=CE=BB?= =?UTF-8?q?=CE=B9=CE=B1=20=CE=86=CE=BD=CE=BD=CE=B1=20Zscheile?= Date: Tue, 14 Jul 2026 13:41:08 +0200 Subject: [PATCH 2/2] feat(xtask): initramfs building --- Cargo.lock | 279 ++++++++++++++++++++++++++++++++++++++++++++- xtask/Cargo.toml | 21 +++- xtask/src/ci/rs.rs | 87 +++++++++++++- 3 files changed, 382 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d60240a01..320a59ff29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,12 @@ dependencies = [ "tock-registers 0.10.1", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -168,6 +174,12 @@ dependencies = [ "paste", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-executor" version = "1.14.0" @@ -284,6 +296,17 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byte-unit" +version = "5.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -302,6 +325,39 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a57a50948117a233b27f9bf73ab74709ab90d245216c4707cc16eea067a50bb" +[[package]] +name = "camino" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.2.56" @@ -446,6 +502,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -610,6 +675,12 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -674,6 +745,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -827,8 +908,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77e69cdfe59e70c3c9b9b361e1fa0052aa8c08a746b6837a0258b7cfab46b67c" dependencies = [ "align-address 0.4.0", + "byte-unit", "const_parse", + "goblin", + "log", + "plain", + "serde", + "tar-no-std", "time", + "toml", ] [[package]] @@ -979,6 +1067,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "interrupt-mutex" version = "0.1.0" @@ -1172,6 +1270,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "nix" version = "0.28.0" @@ -1579,6 +1687,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3f2ad9f15a07f4a0e1677124f9120ce7e83ab7e1ca7186af0ca9da529b62e80" +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "num-traits", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1641,6 +1759,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sbi-rt" version = "0.0.4" @@ -1691,6 +1818,68 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8e4abf97879f4e80db69a9fba7bd64998e9bdad25f58ef045a778e191172fd4" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "sha2" version = "0.11.0" @@ -1714,6 +1903,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "simple-shell" version = "0.0.1" @@ -1853,7 +2048,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "715f9a4586706a61c571cb5ee1c3ac2bbb2cf63e15bce772307b95befef5f5ee" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "log", "num-traits", ] @@ -1908,6 +2103,45 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d2d250f87fb3fb6f225c907cf54381509f47b40b74b1d1f12d2dccbc915bdfe" +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "trapframe" version = "0.11.0" @@ -1987,6 +2221,12 @@ dependencies = [ "log", ] +[[package]] +name = "utf8-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -2074,6 +2314,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2159,6 +2409,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2357,6 +2616,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -2406,16 +2671,22 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "cargo_metadata", "clap", + "flate2", "goblin", + "hermit-entry", "home", "libc", "ovmf-prebuilt", "shlex 2.0.1", "sysinfo", + "tar", + "toml", "ureq", "vsock", "wait-timeout", + "walkdir", "xshell", ] @@ -2444,3 +2715,9 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 1f35ae34f9..b6e84dde91 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -5,20 +5,39 @@ edition = "2024" [features] default = ["ci"] -ci = ["dep:libc", "dep:ovmf-prebuilt", "dep:shlex", "dep:sysinfo", "dep:ureq", "dep:vsock", "dep:wait-timeout"] +ci = [ + "dep:cargo_metadata", + "dep:flate2", + "dep:hermit-entry", + "dep:libc", + "dep:ovmf-prebuilt", "dep:shlex", + "dep:sysinfo", + "dep:tar", + "dep:toml", + "dep:ureq", + "dep:vsock", + "dep:wait-timeout", + "dep:walkdir", +] [dependencies] anyhow = "1.0" +cargo_metadata = { version = "0.23", optional = true } clap = { version = "4", features = ["derive"] } +flate2 = { version = "1.1", optional = true } goblin = { version = "0.10", default-features = false, features = ["archive", "elf32", "elf64", "std"] } +hermit-entry = { version = "0.10", features = ["loader"], optional = true } home = "0.5" libc = { version = "0.2", optional = true } ovmf-prebuilt = { version = "0.2", optional = true } shlex = { version = "2", optional = true } sysinfo = { version = "0.39", optional = true } +tar = { version = "0.4", optional = true } +toml = { version = "1.1", optional = true } ureq = { version = "3", default-features = false, features = ["rustls"], optional = true } vsock = { version = "0.5", optional = true } wait-timeout = { version = "0.2", optional = true } +walkdir = { version = "2.5", optional = true } xshell = "0.2" [lints] diff --git a/xtask/src/ci/rs.rs b/xtask/src/ci/rs.rs index 3be615aa16..9cdc7a62a2 100644 --- a/xtask/src/ci/rs.rs +++ b/xtask/src/ci/rs.rs @@ -1,7 +1,9 @@ -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Args, Subcommand}; +use hermit_entry::config; use crate::cargo_build::CargoBuild; @@ -58,6 +60,7 @@ impl Rs { } let mut cargo = crate::cargo(); + let parent_root = super::parent_root(); if self.package.contains("rftrace") { cargo.env( @@ -67,7 +70,7 @@ impl Rs { }; cargo - .current_dir(super::parent_root()) + .current_dir(parent_root) .arg("build") .args(self.cargo_build.artifact.arch.ci_cargo_args()) .args(self.cargo_build.cargo_build_args()) @@ -77,10 +80,88 @@ impl Rs { let status = cargo.status()?; assert!(status.success()); + // discover possible initramfs seed + let manifest_dir = { + let cur_package = cargo_metadata::PackageName::new(self.package.clone()); + let mut cargo = cargo_metadata::MetadataCommand::new(); + cargo + .current_dir(parent_root) + .no_deps() + .verbose(true) + .exec()? + .packages + .iter() + .find(|i| i.name == cur_package) + .expect("unable to find current package in `cargo metadata` output") + // this path points to `Cargo.toml` + .manifest_path + .parent() + .unwrap() + .to_path_buf() + }; + eprintln!("MANIFEST_DIR = {manifest_dir}"); + let maybe_initramfs = { + let initramfs = manifest_dir.join("initramfs"); + if initramfs.is_dir() { + Some(initramfs) + } else { + None + } + }; + + let mut build_artifact = self.cargo_build.artifact.ci_image(&self.package); + + // handle possible initramfs + if let Some(initramfs) = maybe_initramfs { + eprintln!("discovered initramfs seed, creating initramfs."); + // find kernel name + let konfig = fs::read_to_string(initramfs.join(config::Config::DEFAULT_PATH))?; + let konfig: config::Config<'_> = toml::from_str(&konfig)?; + let kernel_name: &str = match &konfig { + config::Config::V1 { kernel, .. } => kernel, + }; + + let tar_artifact_path = build_artifact.with_extension("tar.gz"); + let mut tar_artifact = tar::Builder::new(flate2::write::GzEncoder::new( + fs::File::create(&tar_artifact_path)?, + flate2::Compression::default(), + )); + tar_artifact.mode(tar::HeaderMode::Deterministic); + + // NOTE: use tar ustar to create the image. + + // add kernel + eprintln!("- {kernel_name}"); + { + let mut header = tar::Header::new_ustar(); + let kernel_meta = fs::metadata(&build_artifact)?; + header.set_path(kernel_name).unwrap(); + header.set_size(kernel_meta.len()); + header.set_cksum(); + + tar_artifact.append(&header, fs::File::open(&build_artifact)?)?; + } + + // add rest + for entry in walkdir::WalkDir::new(&initramfs) { + let entry = entry?; + let entry_rel_path = entry.path().strip_prefix(&initramfs)?; + eprintln!("- {}", entry_rel_path.display()); + if entry_rel_path == Path::new(kernel_name) || entry.metadata()?.is_dir() { + continue; + } + + tar_artifact.append_path_with_name(entry.path(), entry_rel_path)?; + } + + tar_artifact.into_inner()?.finish()?.sync_all()?; + build_artifact = tar_artifact_path; + } + if super::in_ci() { eprintln!("::endgroup::"); } - Ok(self.cargo_build.artifact.ci_image(&self.package)) + Ok(build_artifact) } }