From db6292add6d664be6834b6ed4aab1e433d2c0442 Mon Sep 17 00:00:00 2001 From: Oleksandr Kozachuk Date: Sat, 4 Apr 2026 12:10:13 +0200 Subject: [PATCH] Implement --native flag for standalone executables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `wafer build --native` to produce self-contained native executables. The approach appends AOT-precompiled WASM and metadata to a copy of the wafer binary itself, requiring no Rust toolchain at build time. On startup, the binary checks for an appended payload (8-byte "WAFEREXE" magic trailer). If found, it deserializes the precompiled module and runs it directly, skipping CLI argument parsing entirely. Uses wasmtime's Engine::precompile_module() for AOT compilation at build time and Module::deserialize() at runtime — instant startup with no JIT. Binary layout: [wafer binary][precompiled wasm][metadata json][trailer] Trailer: payload_len(u64 LE) + metadata_len(u64 LE) + "WAFEREXE" Also refactored runner.rs: extracted shared run_module() to avoid duplication between run_wasm_bytes() and run_precompiled_bytes(). Made serialize_metadata() public for CLI use. --- Cargo.lock | 1 + crates/cli/Cargo.toml | 1 + crates/cli/src/main.rs | 151 ++++++++++++++++++++++++++++++++++---- crates/core/src/export.rs | 2 +- crates/core/src/runner.rs | 73 +++++++++++------- 5 files changed, 184 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0c7d8a..e216284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1523,6 +1523,7 @@ dependencies = [ "clap", "rustyline", "wafer-core", + "wasmtime", ] [[package]] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index ead7aec..7e1487f 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] wafer-core = { path = "../core", version = "0.1.0" } +wasmtime = { workspace = true } anyhow = { workspace = true } clap = { version = "4", features = ["derive"] } rustyline = "15" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index ce26637..37cd150 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,11 +1,17 @@ //! WAFER CLI: Interactive REPL, AOT compiler, and WASM runner for WAFER Forth. +use std::io::{Read, Seek, SeekFrom}; use std::path::Path; use clap::{Parser, Subcommand}; -use wafer_core::export::{ExportConfig, export_module}; +use wafer_core::export::{ExportConfig, export_module, serialize_metadata}; use wafer_core::outer::ForthVM; -use wafer_core::runner::run_wasm_file; +use wafer_core::runner::{run_precompiled_bytes, run_wasm_file}; + +/// 8-byte magic trailer identifying a native WAFER executable. +const NATIVE_MAGIC: &[u8; 8] = b"WAFEREXE"; +/// Size of the trailer: `payload_len`(8) + `metadata_len`(8) + magic(8). +const TRAILER_SIZE: u64 = 24; /// WAFER: WebAssembly Forth Engine in Rust #[derive(Parser, Debug)] @@ -25,7 +31,7 @@ enum Commands { /// Input Forth source file file: String, - /// Output .wasm file (default: input with .wasm extension) + /// Output file (default: input stem + .wasm, or no extension with --native) #[arg(short, long)] output: Option, @@ -36,6 +42,10 @@ enum Commands { /// Also generate a JS loader and HTML page for browser execution #[arg(long)] js: bool, + + /// Produce a standalone native executable (AOT-compiled) + #[arg(long)] + native: bool, }, /// Run a pre-compiled WASM module @@ -46,6 +56,15 @@ enum Commands { } fn main() -> anyhow::Result<()> { + // Check for embedded payload before CLI parsing. If this binary has + // an appended WASM payload (produced by --native), run it directly. + if let Some(output) = check_embedded_payload()? { + if !output.is_empty() { + print!("{output}"); + } + return Ok(()); + } + let cli = Cli::parse(); match cli.command { @@ -54,7 +73,8 @@ fn main() -> anyhow::Result<()> { output, entry, js, - }) => cmd_build(&file, output.as_deref(), entry, js), + native, + }) => cmd_build(&file, output.as_deref(), entry, js, native), Some(Commands::Run { file }) => cmd_run(&file), @@ -62,12 +82,57 @@ fn main() -> anyhow::Result<()> { } } -/// `wafer build program.fth -o program.wasm` +/// Check if this executable has an appended WAFER payload. +/// +/// Returns `Some(output)` if a payload was found and executed, +/// `None` if this is a normal wafer binary. +fn check_embedded_payload() -> anyhow::Result> { + let Ok(exe_path) = std::env::current_exe() else { + return Ok(None); + }; + let Ok(mut file) = std::fs::File::open(&exe_path) else { + return Ok(None); + }; + let file_len = file.seek(SeekFrom::End(0))?; + if file_len < TRAILER_SIZE { + return Ok(None); + } + + // Read the 24-byte trailer. + file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?; + let mut trailer = [0u8; TRAILER_SIZE as usize]; + file.read_exact(&mut trailer)?; + + // Check magic. + if &trailer[16..24] != NATIVE_MAGIC { + return Ok(None); + } + + let payload_len = u64::from_le_bytes(trailer[0..8].try_into().unwrap()); + let metadata_len = u64::from_le_bytes(trailer[8..16].try_into().unwrap()); + + // Read the payload and metadata. + let data_start = file_len - TRAILER_SIZE - metadata_len - payload_len; + file.seek(SeekFrom::Start(data_start))?; + + let mut payload = vec![0u8; payload_len as usize]; + file.read_exact(&mut payload)?; + + let mut metadata_bytes = vec![0u8; metadata_len as usize]; + file.read_exact(&mut metadata_bytes)?; + + let metadata_json = String::from_utf8_lossy(&metadata_bytes); + let output = run_precompiled_bytes(&payload, &metadata_json)?; + Ok(Some(output)) +} + +/// `wafer build program.fth -o program.wasm [--native]` fn cmd_build( file: &str, output: Option<&str>, entry: Option, js: bool, + native: bool, ) -> anyhow::Result<()> { let source = std::fs::read_to_string(file)?; @@ -92,18 +157,25 @@ fn cmd_build( .file_stem() .and_then(|s| s.to_str()) .unwrap_or("out"); - format!("{stem}.wasm") + if native { + stem.to_string() + } else { + format!("{stem}.wasm") + } } }; - std::fs::write(&out_path, &wasm_bytes)?; - - let word_count = vm.ir_words().len(); - let host_count = metadata.host_functions.len(); - eprintln!( - "Wrote {out_path} ({} bytes, {word_count} words, {host_count} host functions)", - wasm_bytes.len() - ); + if native { + build_native(&wasm_bytes, &metadata, &out_path)?; + } else { + std::fs::write(&out_path, &wasm_bytes)?; + let word_count = vm.ir_words().len(); + let host_count = metadata.host_functions.len(); + eprintln!( + "Wrote {out_path} ({} bytes, {word_count} words, {host_count} host functions)", + wasm_bytes.len() + ); + } if js { let out = Path::new(&out_path); @@ -129,6 +201,55 @@ fn cmd_build( Ok(()) } +/// Build a native executable by appending AOT-compiled WASM to the wafer binary. +fn build_native( + wasm_bytes: &[u8], + metadata: &wafer_core::export::ExportMetadata, + out_path: &str, +) -> anyhow::Result<()> { + // AOT precompile the WASM to native code. + let mut config = wasmtime::Config::new(); + config.cranelift_nan_canonicalization(false); + let engine = wasmtime::Engine::new(&config)?; + let precompiled = engine.precompile_module(wasm_bytes)?; + + // Read the current wafer binary. + let self_exe = std::env::current_exe()?; + let self_bytes = std::fs::read(&self_exe)?; + + // Serialize metadata. + let metadata_json = serialize_metadata(metadata); + let metadata_bytes = metadata_json.as_bytes(); + + // Assemble: wafer binary + precompiled payload + metadata + trailer. + let mut out = Vec::with_capacity( + self_bytes.len() + precompiled.len() + metadata_bytes.len() + TRAILER_SIZE as usize, + ); + out.extend_from_slice(&self_bytes); + out.extend_from_slice(&precompiled); + out.extend_from_slice(metadata_bytes); + out.extend_from_slice(&(precompiled.len() as u64).to_le_bytes()); + out.extend_from_slice(&(metadata_bytes.len() as u64).to_le_bytes()); + out.extend_from_slice(NATIVE_MAGIC); + + std::fs::write(out_path, &out)?; + + // Make executable on Unix. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(out_path, std::fs::Permissions::from_mode(0o755))?; + } + + eprintln!( + "Wrote {out_path} ({:.1} MB native executable, {:.0} KB precompiled WASM)", + out.len() as f64 / 1_048_576.0, + precompiled.len() as f64 / 1024.0 + ); + + Ok(()) +} + /// `wafer run program.wasm` fn cmd_run(file: &str) -> anyhow::Result<()> { let output = run_wasm_file(file)?; @@ -155,7 +276,7 @@ fn cmd_eval_or_repl(file: Option<&str>) -> anyhow::Result<()> { if !stdin_is_tty() { // Non-interactive: read all of stdin and evaluate let mut input = String::new(); - std::io::Read::read_to_string(&mut std::io::stdin(), &mut input)?; + Read::read_to_string(&mut std::io::stdin(), &mut input)?; for line in input.lines() { match vm.evaluate(line) { Ok(()) => { diff --git a/crates/core/src/export.rs b/crates/core/src/export.rs index 1a5ccd2..7b4e346 100644 --- a/crates/core/src/export.rs +++ b/crates/core/src/export.rs @@ -172,7 +172,7 @@ fn collect_external_calls(ops: &[IrOp], ir_ids: &HashSet, host_ids: &mut } /// Serialize export metadata to JSON (hand-rolled, no serde dependency). -fn serialize_metadata(m: &ExportMetadata) -> String { +pub fn serialize_metadata(m: &ExportMetadata) -> String { let mut s = String::from("{\n"); let _ = writeln!(s, " \"version\": {},", m.version); match m.entry_table_index { diff --git a/crates/core/src/runner.rs b/crates/core/src/runner.rs index 5dac34d..77ac2cd 100644 --- a/crates/core/src/runner.rs +++ b/crates/core/src/runner.rs @@ -10,14 +10,21 @@ use wasmtime::{ TableType, Val, ValType, }; -use crate::export::deserialize_metadata; +use crate::export::{ExportMetadata, deserialize_metadata}; use crate::memory::{CELL_SIZE, DATA_STACK_TOP, SYSVAR_BASE_VAR}; /// Host state for the runner (currently unused by wasmtime `Store` but /// required as the generic parameter). struct RunnerHost {} -/// Execute a pre-compiled `.wasm` module and return its output. +/// Create a wasmtime engine with the standard WAFER configuration. +fn make_engine() -> anyhow::Result { + let mut config = wasmtime::Config::new(); + config.cranelift_nan_canonicalization(false); + Engine::new(&config) +} + +/// Execute a pre-compiled `.wasm` module from a file path. pub fn run_wasm_file(path: &str) -> anyhow::Result { let wasm_bytes = std::fs::read(path)?; run_wasm_bytes(&wasm_bytes) @@ -25,20 +32,40 @@ pub fn run_wasm_file(path: &str) -> anyhow::Result { /// Execute WASM bytes directly (used by tests and the CLI). pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result { - // Parse the "wafer" custom section for metadata. let metadata_json = extract_custom_section(wasm_bytes, "wafer")?; let metadata = deserialize_metadata(&metadata_json)?; + let engine = make_engine()?; + let module = Module::new(&engine, wasm_bytes)?; + run_module(&engine, module, &metadata) +} - // Set up wasmtime runtime. - let mut config = wasmtime::Config::new(); - config.cranelift_nan_canonicalization(false); - let engine = Engine::new(&config)?; +/// Execute an AOT-precompiled module with separate metadata. +/// +/// The `precompiled` bytes must have been produced by +/// `Engine::precompile_module` with a compatible wasmtime version +/// and platform. +#[allow(unsafe_code)] +pub fn run_precompiled_bytes(precompiled: &[u8], metadata_json: &str) -> anyhow::Result { + let metadata = deserialize_metadata(metadata_json)?; + let engine = make_engine()?; + // SAFETY: precompiled bytes are produced by wafer build --native using + // the same wasmtime version. The caller guarantees compatibility. + let module = unsafe { Module::deserialize(&engine, precompiled)? }; + run_module(&engine, module, &metadata) +} +/// Shared runner logic: given a compiled module and metadata, set up the +/// six WASM imports, register host functions, call `_start`, return output. +fn run_module( + engine: &Engine, + module: Module, + metadata: &ExportMetadata, +) -> anyhow::Result { let output = Arc::new(Mutex::new(String::new())); - let mut store = Store::new(&engine, RunnerHost {}); + let mut store = Store::new(engine, RunnerHost {}); // Create the 6 imports the module expects. - let memory_pages = metadata.memory_size.div_ceil(65536).max(16); // at least 16 pages like the VM + let memory_pages = metadata.memory_size.div_ceil(65536).max(16); let memory = Memory::new(&mut store, MemoryType::new(memory_pages, None))?; let dsp = Global::new( @@ -57,24 +84,15 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result { Val::I32(metadata.fsp_init as i32), )?; - // Determine table size from the module's import. - let parsed = wasmparser::Parser::new(0).parse_all(wasm_bytes); - let mut table_min: u64 = 256; - for payload in parsed { - if let wasmparser::Payload::ImportSection(reader) = payload? { - for import in reader { - let import = import?; - if import.name == "table" - && let wasmparser::TypeRef::Table(t) = import.ty - { - table_min = t.initial; - } - } - } - } + // Determine table size from the module's imports. + let table_min: u32 = module + .imports() + .find(|i| i.name() == "table") + .and_then(|i| i.ty().table().map(|t| t.minimum() as u32)) + .unwrap_or(256); let table = Table::new( &mut store, - TableType::new(wasmtime::RefType::FUNCREF, table_min as u32, None), + TableType::new(wasmtime::RefType::FUNCREF, table_min, None), Ref::Func(None), )?; @@ -82,7 +100,7 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result { let out_ref = Arc::clone(&output); let emit_func = Func::new( &mut store, - FuncType::new(&engine, [ValType::I32], []), + FuncType::new(engine, [ValType::I32], []), move |_caller, params, _results| { let code = params[0].unwrap_i32(); if let Some(ch) = char::from_u32(code as u32) { @@ -93,7 +111,6 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result { ); // Instantiate the module. - let module = Module::new(&engine, wasm_bytes)?; let instance = wasmtime::Instance::new( &mut store, &module, @@ -109,7 +126,7 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result { // Register host functions in the table at the metadata-specified indices. for (idx, name) in &metadata.host_functions { - let func = create_host_func(&mut store, &engine, memory, dsp, &output, name); + let func = create_host_func(&mut store, engine, memory, dsp, &output, name); table.set(&mut store, *idx as u64, Ref::Func(Some(func)))?; }