diff --git a/.cargo/config.toml b/.cargo/config.toml index 3a85f90dbb31dad18bfa7a439520412e40b9ae33..5a37c8104a35bc7dac8798cb3eccc0764be6fdeb 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,13 +1,5 @@ -[unstable] -# Keep in sync with CI! -build-std = ["core", "alloc"] -build-std-features = ["compiler-builtins-mem"] - -[build] -target = "targets/x86_64-unknown-none-hermitkernel.json" -rustflags = [ - "-Zmutable-noalias=no" -] +[alias] +xtask = "run --package xtask --" [target.x86_64-unknown-none-hermitkernel] runner = "tests/hermit_test_runner.py" diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index a562065dbe5931c7a3c763ce86fbc917e44d5508..8648183a746aa246bbaf6015b5f231ea4da515af 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -23,13 +23,12 @@ jobs: matrix: os: [ubuntu-latest] steps: - - name: Install cargo-binutils - run: cargo install cargo-binutils - name: Checkout rusty-hermit uses: actions/checkout@v3 with: repository: hermitcore/rusty-hermit submodules: true + ref: 04ca4407fd6823972a6950e3d59a157208be4f00 - name: Remove libhermit-rs submodule run: git rm -r libhermit-rs - name: Checkout libhermit-rs @@ -41,9 +40,9 @@ jobs: run: rustup show - name: Build minimal kernel working-directory: libhermit-rs - run: cargo build --no-default-features --target targets/aarch64-unknown-none-hermitkernel.json -Z build-std=core,alloc + run: cargo xtask build --arch aarch64 --no-default-features - name: Build dev profile - run: cargo build --target aarch64-unknown-hermit -p hello_world + run: cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target aarch64-unknown-hermit --package hello_world - name: Build loader run: make arch=aarch64 working-directory: loader diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 6decbbbbd99a83de0eff83be652fcadd198738a3..7c9d079779a2d48218e2b8020af7e127ddbe8137 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -10,6 +10,7 @@ on: env: CARGO_TERM_COLOR: always + RUSTFLAGS: -Dwarnings jobs: clippy: @@ -24,4 +25,4 @@ jobs: sudo apt-get update sudo apt-get install nasm - name: Clippy - run: cargo clippy -- -D warnings + run: cargo xtask clippy diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index f91b40e4c94f0fe7cbde6496fdad252c818b06bb..07d54237e21fff818082ed3834275fe74f807863 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -27,5 +27,5 @@ jobs: env: RUSTDOCFLAGS: -D warnings run: | - cargo doc --no-deps --document-private-items --target targets/x86_64-unknown-none-hermitkernel.json - cargo doc --no-deps --document-private-items --target targets/aarch64-unknown-none-hermitkernel.json + cargo doc -Z build-std=core,alloc --package rusty-hermit --no-deps --document-private-items --target targets/x86_64-unknown-none-hermitkernel.json + cargo doc -Z build-std=core,alloc --package rusty-hermit --no-deps --document-private-items --target targets/aarch64-unknown-none-hermitkernel.json diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml index a321eee80c6e19ceb1e68e9388e85b3f8d93ce7f..6ccabff2f0a45eca3431bbbbfd376d622a909eca 100644 --- a/.github/workflows/publish_docs.yml +++ b/.github/workflows/publish_docs.yml @@ -21,7 +21,7 @@ jobs: sudo apt-get update sudo apt-get install nasm - name: Generate documentation - run: cargo doc + run: cargo doc -Z build-std=core,alloc --package rusty-hermit - name: Generate index.html run: | cat > target/x86_64-unknown-none-hermitkernel/doc/index.html <<EOL diff --git a/.github/workflows/x86.yml b/.github/workflows/x86.yml index 8437b53c7fa83d90ee69d329df0638029edbc69c..dae160bfb4b4b1b3fc14e335c2e04adb9d5964ad 100644 --- a/.github/workflows/x86.yml +++ b/.github/workflows/x86.yml @@ -39,13 +39,12 @@ jobs: choco install qemu nasm make echo "C:\Program Files\qemu" >> $GITHUB_PATH echo "C:\Program Files\NASM" >> $GITHUB_PATH - - name: Install cargo-binutils - run: cargo install cargo-binutils - name: Checkout rusty-hermit uses: actions/checkout@v3 with: repository: hermitcore/rusty-hermit submodules: true + ref: 04ca4407fd6823972a6950e3d59a157208be4f00 - name: Remove libhermit-rs submodule run: git rm -r libhermit-rs - name: Checkout libhermit-rs @@ -57,9 +56,9 @@ jobs: run: rustup show - name: Build minimal kernel working-directory: libhermit-rs - run: cargo build --no-default-features -Z build-std=core,alloc + run: cargo xtask build --arch x86_64 --no-default-features - name: Build dev profile - run: cargo build + run: cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target x86_64-unknown-hermit - name: Unittests on host (ubuntu) if: ${{ matrix.os == 'ubuntu-latest' }} working-directory: libhermit-rs @@ -81,7 +80,7 @@ jobs: run: cargo test --tests --no-fail-fast -- --bootloader_path=../loader/target/x86_64-unknown-hermit-loader/release/rusty-loader continue-on-error: true - name: Build release profile - run: cargo build --release + run: cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target x86_64-unknown-hermit --release - name: Test release profile run: | qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio \ @@ -91,9 +90,7 @@ jobs: -initrd target/x86_64-unknown-hermit/release/rusty_demo - name: Build minimal profile if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - cargo clean - cargo build --no-default-features --release -p hello_world + run: cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target x86_64-unknown-hermit --no-default-features --release --package hello_world - name: Test minimal profile id: minimal if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc9367d8311df4bf44a05f65db8aa67e0cd2f4cf..14021e16433fd7033d56116b5f773a273e4a00e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,14 +37,15 @@ build:demo: - if [ -d "$HOME/tmp_libhermit-rs/target" ]; then rm -rf $HOME/tmp_libhermit-rs/target; fi - git clone https://github.com/hermitcore/rusty-hermit.git - cd rusty-hermit + - git checkout 04ca4407fd6823972a6950e3d59a157208be4f00 - echo "rusty-hermit at commit $(git rev-parse HEAD)" # Ensure that libhermit-rs is empty - This shouldn't be necessary since we don't initialize the submodules # But let's do it anyway to be safe - if [ -d "libhermit-rs" ]; then rm -rf libhermit-rs; fi - mkdir libhermit-rs - shopt -s dotglob nullglob && mv $HOME/tmp_libhermit-rs/* libhermit-rs/. - - cargo build -p rusty_demo - - cargo build -p rusty_demo --release + - cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target x86_64-unknown-hermit --package rusty_demo + - cargo build -Zbuild-std=core,alloc,std,panic_abort -Zbuild-std-features=compiler-builtins-mem --target x86_64-unknown-hermit --package rusty_demo --release artifacts: paths: - rusty-hermit/target/x86_64-unknown-hermit/debug/rusty_demo diff --git a/Cargo.toml b/Cargo.toml index c2f039e06bcbecae2c2e1ef146c4041e426a01d4..ba0f9125df8bc1e01308ef5334691fbeb05459f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,3 +84,8 @@ x86 = { version = "0.47", default-features = false } [target.'cfg(target_arch = "aarch64")'.dependencies.aarch64] version = "0.0.7" default-features = false + +[workspace] +members = [ + "xtask", +] diff --git a/Dockerfile b/Dockerfile index 1611a2d948c03a435c88039e8a4d102a8f53744e..6ca7782ef1d381355cb4afe75def14f09e5a0deb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,6 @@ RUN set -eux; \ # Build dependencies with stable toolchain channel FROM rust:bullseye as stable-deps RUN set -eux; \ - cargo install cargo-binutils; \ cargo install cargo-download; \ cargo install --git https://github.com/hermitcore/uhyve.git --locked uhyve; @@ -62,7 +61,6 @@ RUN set -eux; \ qemu-system-x86 \ ; \ rm -rf /var/lib/apt/lists/*; -COPY --from=stable-deps $CARGO_HOME/bin/rust-objcopy $CARGO_HOME/bin/rust-objcopy COPY --from=stable-deps $CARGO_HOME/bin/cargo-download $CARGO_HOME/bin/cargo-download COPY --from=stable-deps $CARGO_HOME/bin/uhyve $CARGO_HOME/bin/uhyve COPY --from=hermit-deps rusty-loader/target/x86_64-unknown-hermit-loader/release/rusty-loader /usr/local/bin/rusty-loader diff --git a/xtask/.gitignore b/xtask/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..ea8c4bf7f35f6f77f75d92ad8ce8349f6e81ddba --- /dev/null +++ b/xtask/.gitignore @@ -0,0 +1 @@ +/target diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..6a81a9e4b031d0e1afeea90578cd5fb7ecc9213f --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +goblin = { version = "0.5", default-features = false, features = ["archive", "std"] } +rustc_version = "0.4" +xflags = "0.2" +xshell = "0.2" diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs new file mode 100644 index 0000000000000000000000000000000000000000..cbce6df74e472d271b5f27007b760f8e1f01d7f4 --- /dev/null +++ b/xtask/src/flags.rs @@ -0,0 +1,84 @@ +use std::path::PathBuf; + +xflags::xflags! { + src "./src/flags.rs" + + /// Run custom build command. + cmd xtask { + default cmd help { + /// Print help information. + optional -h, --help + } + + /// Build the kernel. + cmd build + { + /// Build for the architecture. + required --arch arch: String + /// Directory for all generated artifacts. + optional --target-dir target_dir: PathBuf + /// Do not activate the `default` feature. + optional --no-default-features + /// Space or comma separated list of features to activate. + repeated --features features: String + /// Build artifacts in release mode, with optimizations. + optional -r, --release + /// Build artifacts with the specified profile. + optional --profile profile: String + /// Enable the `-Z instrument-mcount` flag. + optional --instrument-mcount + } + + /// Run clippy for all targets. + cmd clippy {} + } +} + +// generated start +// The following code is generated by `xflags` macro. +// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate. +#[derive(Debug)] +pub struct Xtask { + pub subcommand: XtaskCmd, +} + +#[derive(Debug)] +pub enum XtaskCmd { + Help(Help), + Build(Build), + Clippy(Clippy), +} + +#[derive(Debug)] +pub struct Help { + pub help: bool, +} + +#[derive(Debug)] +pub struct Build { + pub arch: String, + pub target_dir: Option<PathBuf>, + pub no_default_features: bool, + pub features: Vec<String>, + pub release: bool, + pub profile: Option<String>, + pub instrument_mcount: bool, +} + +#[derive(Debug)] +pub struct Clippy; + +impl Xtask { + pub const HELP: &'static str = Self::HELP_; + + #[allow(dead_code)] + pub fn from_env() -> xflags::Result<Self> { + Self::from_env_() + } + + #[allow(dead_code)] + pub fn from_vec(args: Vec<std::ffi::OsString>) -> xflags::Result<Self> { + Self::from_vec_(args) + } +} +// generated end diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..2221c2fa96d0421640eaa809166280a169e52fdf --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,270 @@ +//! See <https://github.com/matklad/cargo-xtask/>. + +mod flags; +mod rustc; + +use std::{ + env::{self, VarError}, + ffi::OsString, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Result}; +use goblin::archive::Archive; +use xshell::{cmd, Shell}; + +const RUSTFLAGS: &[&str] = &[ + // TODO: Re-enable mutable-noalias + // https://github.com/hermitcore/libhermit-rs/issues/200 + "-Zmutable-noalias=no", +]; + +const KERNEL_CARGO_ARGS: &[&str] = &[ + "-Zbuild-std=core,alloc", + "-Zbuild-std-features=compiler-builtins-mem", +]; + +fn main() -> Result<()> { + flags::Xtask::from_env()?.run() +} + +impl flags::Xtask { + fn run(self) -> Result<()> { + match self.subcommand { + flags::XtaskCmd::Help(_) => { + println!("{}", flags::Xtask::HELP); + Ok(()) + } + flags::XtaskCmd::Build(build) => build.run(), + flags::XtaskCmd::Clippy(clippy) => clippy.run(), + } + } +} + +impl flags::Build { + fn run(self) -> Result<()> { + let sh = sh()?; + + eprintln!("Building kernel"); + cmd!(sh, "cargo build") + .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) + .args(KERNEL_CARGO_ARGS) + .arg(target_arg(&self.arch)?) + .args(self.target_dir_args()) + .args(self.no_default_features_args()) + .args(self.features_args()) + .args(self.release_args()) + .args(self.profile_args()) + .run()?; + + eprintln!("Exporting symbols"); + self.export_syms()?; + + eprintln!("Kernel available at {}", self.dist_archive().display()); + Ok(()) + } + + fn cargo_encoded_rustflags(&self) -> Result<String> { + let outer_rustflags = match env::var("CARGO_ENCODED_RUSTFLAGS") { + Ok(s) => Some(s), + Err(VarError::NotPresent) => None, + Err(err) => return Err(err.into()), + }; + + let mut rustflags = outer_rustflags + .as_deref() + .map(|s| vec![s]) + .unwrap_or_default(); + rustflags.extend(RUSTFLAGS); + if self.instrument_mcount { + rustflags.push("-Zinstrument-mcount"); + } + Ok(rustflags.join("\x1f")) + } + + fn target_dir_args(&self) -> Vec<OsString> { + vec!["--target-dir".into(), self.target_dir().into()] + } + + fn no_default_features_args(&self) -> &[&str] { + if self.no_default_features { + &["--no-default-features"] + } else { + &[] + } + } + + fn features_args(&self) -> Vec<&str> { + if self.features.is_empty() { + vec![] + } else { + let mut features = vec!["--features"]; + features.extend(self.features.iter().map(String::as_str)); + features + } + } + + fn release_args(&self) -> &[&str] { + if self.release { + &["--release"] + } else { + &[] + } + } + + fn profile_args(&self) -> Vec<&str> { + match self.profile.as_deref() { + Some(profile) => vec!["--profile", profile], + None => vec![], + } + } + + fn export_syms(&self) -> Result<()> { + let sh = sh()?; + + let input = self.build_archive(); + let output = self.dist_archive(); + sh.create_dir(output.parent().unwrap())?; + sh.copy_file(&input, &output)?; + + let objcopy = binutil("objcopy")?; + + cmd!(sh, "{objcopy} --prefix-symbols=hermit_ {output}").run()?; + + let archive_bytes = sh.read_binary_file(&input)?; + let archive = Archive::parse(&archive_bytes)?; + + let sys_fns = archive + .summarize() + .into_iter() + .filter(|(member_name, _, _)| member_name.starts_with("hermit")) + .flat_map(|(_, _, symbols)| symbols) + .filter(|symbol| symbol.starts_with("sys_")); + + let explicit_exports = [ + "_start", + "__bss_start", + "__rg_oom", + "memcmp", + "memcpy", + "memmove", + "memset", + "runtime_entry", + ] + .into_iter(); + + let symbol_redefinitions = explicit_exports + .chain(sys_fns) + .map(|symbol| format!("hermit_{symbol} {symbol}\n")) + .collect::<String>(); + + let exported_syms = self.exported_syms(); + + sh.write_file(&exported_syms, &symbol_redefinitions)?; + + cmd!(sh, "{objcopy} --redefine-syms={exported_syms} {output}").run()?; + + Ok(()) + } + + fn profile(&self) -> &str { + self.profile + .as_deref() + .unwrap_or(if self.release { "release" } else { "dev" }) + } + + fn target_dir(&self) -> &Path { + self.target_dir + .as_deref() + .unwrap_or_else(|| Path::new("target")) + } + + fn out_dir(&self) -> PathBuf { + let mut out_dir = self.target_dir().to_path_buf(); + out_dir.push(target(&self.arch).unwrap()); + out_dir.push(match self.profile() { + "dev" => "debug", + profile => profile, + }); + out_dir + } + + fn dist_dir(&self) -> PathBuf { + let mut out_dir = self.target_dir().to_path_buf(); + out_dir.push(&self.arch); + out_dir.push(match self.profile() { + "dev" => "debug", + profile => profile, + }); + out_dir + } + + fn build_archive(&self) -> PathBuf { + let mut built_archive = self.out_dir(); + built_archive.push("libhermit.a"); + built_archive + } + + fn dist_archive(&self) -> PathBuf { + let mut dist_archive = self.dist_dir(); + dist_archive.push("libhermit.a"); + dist_archive + } + + fn exported_syms(&self) -> PathBuf { + let mut redefine_syms_path = self.dist_dir(); + redefine_syms_path.push("exported-syms"); + redefine_syms_path + } +} + +impl flags::Clippy { + fn run(self) -> Result<()> { + let sh = sh()?; + + // TODO: Enable clippy for aarch64 + // https://github.com/hermitcore/libhermit-rs/issues/381 + for target in ["x86_64"] { + cmd!(sh, "cargo clippy") + .args(KERNEL_CARGO_ARGS) + .arg(target_arg(target)?) + .run()?; + } + + cmd!(sh, "cargo clippy --package xtask").run()?; + + Ok(()) + } +} + +fn target(arch: &str) -> Result<&'static str> { + match arch { + "x86_64" => Ok("x86_64-unknown-none-hermitkernel"), + "aarch64" => Ok("aarch64-unknown-none-hermitkernel"), + arch => Err(anyhow!("Unsupported arch: {arch}")), + } +} + +fn target_arg(arch: &str) -> Result<String> { + let target = target(arch)?; + Ok(format!("--target=targets/{target}.json")) +} + +fn binutil(name: &str) -> Result<PathBuf> { + let exe_suffix = env::consts::EXE_SUFFIX; + let exe = format!("llvm-{name}{exe_suffix}"); + + let mut path = rustc::rustlib()?; + path.push(exe); + Ok(path) +} + +fn sh() -> Result<Shell> { + let sh = Shell::new()?; + sh.change_dir(project_root()); + Ok(sh) +} + +fn project_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap() +} diff --git a/xtask/src/rustc.rs b/xtask/src/rustc.rs new file mode 100644 index 0000000000000000000000000000000000000000..632cd91ce850684952e23eb5071f451e73c7d0b8 --- /dev/null +++ b/xtask/src/rustc.rs @@ -0,0 +1,25 @@ +//! Taken from <https://github.com/rust-embedded/cargo-binutils/blob/980607cf8e4bb1b7db5cc7a35aafa38463818f7e/src/rustc.rs>. + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +use anyhow::Result; + +pub fn sysroot() -> Result<String> { + let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let output = Command::new(rustc).arg("--print").arg("sysroot").output()?; + // Note: We must trim() to remove the `\n` from the end of stdout + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +// See: https://github.com/rust-lang/rust/blob/564758c4c329e89722454dd2fbb35f1ac0b8b47c/src/bootstrap/dist.rs#L2334-L2341 +pub fn rustlib() -> Result<PathBuf> { + let sysroot = sysroot()?; + let mut pathbuf = PathBuf::from(sysroot); + pathbuf.push("lib"); + pathbuf.push("rustlib"); + pathbuf.push(rustc_version::version_meta()?.host); // TODO: Prevent calling rustc_version::version_meta() multiple times + pathbuf.push("bin"); + Ok(pathbuf) +}