Implement --native flag for standalone executables
Add `wafer build --native` to produce self-contained native executables. The approach appends AOT-precompiled WASM and metadata to a copy of the wafer binary itself, requiring no Rust toolchain at build time. On startup, the binary checks for an appended payload (8-byte "WAFEREXE" magic trailer). If found, it deserializes the precompiled module and runs it directly, skipping CLI argument parsing entirely. Uses wasmtime's Engine::precompile_module() for AOT compilation at build time and Module::deserialize() at runtime — instant startup with no JIT. Binary layout: [wafer binary][precompiled wasm][metadata json][trailer] Trailer: payload_len(u64 LE) + metadata_len(u64 LE) + "WAFEREXE" Also refactored runner.rs: extracted shared run_module() to avoid duplication between run_wasm_bytes() and run_precompiled_bytes(). Made serialize_metadata() public for CLI use.
This commit is contained in:
Generated
+1
@@ -1523,6 +1523,7 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"rustyline",
|
"rustyline",
|
||||||
"wafer-core",
|
"wafer-core",
|
||||||
|
"wasmtime",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ workspace = true
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
wafer-core = { path = "../core", version = "0.1.0" }
|
wafer-core = { path = "../core", version = "0.1.0" }
|
||||||
|
wasmtime = { workspace = true }
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
clap = { version = "4", features = ["derive"] }
|
clap = { version = "4", features = ["derive"] }
|
||||||
rustyline = "15"
|
rustyline = "15"
|
||||||
|
|||||||
+136
-15
@@ -1,11 +1,17 @@
|
|||||||
//! WAFER CLI: Interactive REPL, AOT compiler, and WASM runner for WAFER Forth.
|
//! WAFER CLI: Interactive REPL, AOT compiler, and WASM runner for WAFER Forth.
|
||||||
|
|
||||||
|
use std::io::{Read, Seek, SeekFrom};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use wafer_core::export::{ExportConfig, export_module};
|
use wafer_core::export::{ExportConfig, export_module, serialize_metadata};
|
||||||
use wafer_core::outer::ForthVM;
|
use wafer_core::outer::ForthVM;
|
||||||
use wafer_core::runner::run_wasm_file;
|
use wafer_core::runner::{run_precompiled_bytes, run_wasm_file};
|
||||||
|
|
||||||
|
/// 8-byte magic trailer identifying a native WAFER executable.
|
||||||
|
const NATIVE_MAGIC: &[u8; 8] = b"WAFEREXE";
|
||||||
|
/// Size of the trailer: `payload_len`(8) + `metadata_len`(8) + magic(8).
|
||||||
|
const TRAILER_SIZE: u64 = 24;
|
||||||
|
|
||||||
/// WAFER: WebAssembly Forth Engine in Rust
|
/// WAFER: WebAssembly Forth Engine in Rust
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
@@ -25,7 +31,7 @@ enum Commands {
|
|||||||
/// Input Forth source file
|
/// Input Forth source file
|
||||||
file: String,
|
file: String,
|
||||||
|
|
||||||
/// Output .wasm file (default: input with .wasm extension)
|
/// Output file (default: input stem + .wasm, or no extension with --native)
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
output: Option<String>,
|
output: Option<String>,
|
||||||
|
|
||||||
@@ -36,6 +42,10 @@ enum Commands {
|
|||||||
/// Also generate a JS loader and HTML page for browser execution
|
/// Also generate a JS loader and HTML page for browser execution
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
js: bool,
|
js: bool,
|
||||||
|
|
||||||
|
/// Produce a standalone native executable (AOT-compiled)
|
||||||
|
#[arg(long)]
|
||||||
|
native: bool,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Run a pre-compiled WASM module
|
/// Run a pre-compiled WASM module
|
||||||
@@ -46,6 +56,15 @@ enum Commands {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> anyhow::Result<()> {
|
fn main() -> anyhow::Result<()> {
|
||||||
|
// Check for embedded payload before CLI parsing. If this binary has
|
||||||
|
// an appended WASM payload (produced by --native), run it directly.
|
||||||
|
if let Some(output) = check_embedded_payload()? {
|
||||||
|
if !output.is_empty() {
|
||||||
|
print!("{output}");
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
match cli.command {
|
match cli.command {
|
||||||
@@ -54,7 +73,8 @@ fn main() -> anyhow::Result<()> {
|
|||||||
output,
|
output,
|
||||||
entry,
|
entry,
|
||||||
js,
|
js,
|
||||||
}) => cmd_build(&file, output.as_deref(), entry, js),
|
native,
|
||||||
|
}) => cmd_build(&file, output.as_deref(), entry, js, native),
|
||||||
|
|
||||||
Some(Commands::Run { file }) => cmd_run(&file),
|
Some(Commands::Run { file }) => cmd_run(&file),
|
||||||
|
|
||||||
@@ -62,12 +82,57 @@ fn main() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `wafer build program.fth -o program.wasm`
|
/// Check if this executable has an appended WAFER payload.
|
||||||
|
///
|
||||||
|
/// Returns `Some(output)` if a payload was found and executed,
|
||||||
|
/// `None` if this is a normal wafer binary.
|
||||||
|
fn check_embedded_payload() -> anyhow::Result<Option<String>> {
|
||||||
|
let Ok(exe_path) = std::env::current_exe() else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let Ok(mut file) = std::fs::File::open(&exe_path) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let file_len = file.seek(SeekFrom::End(0))?;
|
||||||
|
if file_len < TRAILER_SIZE {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the 24-byte trailer.
|
||||||
|
file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
|
||||||
|
let mut trailer = [0u8; TRAILER_SIZE as usize];
|
||||||
|
file.read_exact(&mut trailer)?;
|
||||||
|
|
||||||
|
// Check magic.
|
||||||
|
if &trailer[16..24] != NATIVE_MAGIC {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload_len = u64::from_le_bytes(trailer[0..8].try_into().unwrap());
|
||||||
|
let metadata_len = u64::from_le_bytes(trailer[8..16].try_into().unwrap());
|
||||||
|
|
||||||
|
// Read the payload and metadata.
|
||||||
|
let data_start = file_len - TRAILER_SIZE - metadata_len - payload_len;
|
||||||
|
file.seek(SeekFrom::Start(data_start))?;
|
||||||
|
|
||||||
|
let mut payload = vec![0u8; payload_len as usize];
|
||||||
|
file.read_exact(&mut payload)?;
|
||||||
|
|
||||||
|
let mut metadata_bytes = vec![0u8; metadata_len as usize];
|
||||||
|
file.read_exact(&mut metadata_bytes)?;
|
||||||
|
|
||||||
|
let metadata_json = String::from_utf8_lossy(&metadata_bytes);
|
||||||
|
let output = run_precompiled_bytes(&payload, &metadata_json)?;
|
||||||
|
Ok(Some(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `wafer build program.fth -o program.wasm [--native]`
|
||||||
fn cmd_build(
|
fn cmd_build(
|
||||||
file: &str,
|
file: &str,
|
||||||
output: Option<&str>,
|
output: Option<&str>,
|
||||||
entry: Option<String>,
|
entry: Option<String>,
|
||||||
js: bool,
|
js: bool,
|
||||||
|
native: bool,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let source = std::fs::read_to_string(file)?;
|
let source = std::fs::read_to_string(file)?;
|
||||||
|
|
||||||
@@ -92,18 +157,25 @@ fn cmd_build(
|
|||||||
.file_stem()
|
.file_stem()
|
||||||
.and_then(|s| s.to_str())
|
.and_then(|s| s.to_str())
|
||||||
.unwrap_or("out");
|
.unwrap_or("out");
|
||||||
format!("{stem}.wasm")
|
if native {
|
||||||
|
stem.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{stem}.wasm")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
std::fs::write(&out_path, &wasm_bytes)?;
|
if native {
|
||||||
|
build_native(&wasm_bytes, &metadata, &out_path)?;
|
||||||
let word_count = vm.ir_words().len();
|
} else {
|
||||||
let host_count = metadata.host_functions.len();
|
std::fs::write(&out_path, &wasm_bytes)?;
|
||||||
eprintln!(
|
let word_count = vm.ir_words().len();
|
||||||
"Wrote {out_path} ({} bytes, {word_count} words, {host_count} host functions)",
|
let host_count = metadata.host_functions.len();
|
||||||
wasm_bytes.len()
|
eprintln!(
|
||||||
);
|
"Wrote {out_path} ({} bytes, {word_count} words, {host_count} host functions)",
|
||||||
|
wasm_bytes.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if js {
|
if js {
|
||||||
let out = Path::new(&out_path);
|
let out = Path::new(&out_path);
|
||||||
@@ -129,6 +201,55 @@ fn cmd_build(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build a native executable by appending AOT-compiled WASM to the wafer binary.
|
||||||
|
fn build_native(
|
||||||
|
wasm_bytes: &[u8],
|
||||||
|
metadata: &wafer_core::export::ExportMetadata,
|
||||||
|
out_path: &str,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
// AOT precompile the WASM to native code.
|
||||||
|
let mut config = wasmtime::Config::new();
|
||||||
|
config.cranelift_nan_canonicalization(false);
|
||||||
|
let engine = wasmtime::Engine::new(&config)?;
|
||||||
|
let precompiled = engine.precompile_module(wasm_bytes)?;
|
||||||
|
|
||||||
|
// Read the current wafer binary.
|
||||||
|
let self_exe = std::env::current_exe()?;
|
||||||
|
let self_bytes = std::fs::read(&self_exe)?;
|
||||||
|
|
||||||
|
// Serialize metadata.
|
||||||
|
let metadata_json = serialize_metadata(metadata);
|
||||||
|
let metadata_bytes = metadata_json.as_bytes();
|
||||||
|
|
||||||
|
// Assemble: wafer binary + precompiled payload + metadata + trailer.
|
||||||
|
let mut out = Vec::with_capacity(
|
||||||
|
self_bytes.len() + precompiled.len() + metadata_bytes.len() + TRAILER_SIZE as usize,
|
||||||
|
);
|
||||||
|
out.extend_from_slice(&self_bytes);
|
||||||
|
out.extend_from_slice(&precompiled);
|
||||||
|
out.extend_from_slice(metadata_bytes);
|
||||||
|
out.extend_from_slice(&(precompiled.len() as u64).to_le_bytes());
|
||||||
|
out.extend_from_slice(&(metadata_bytes.len() as u64).to_le_bytes());
|
||||||
|
out.extend_from_slice(NATIVE_MAGIC);
|
||||||
|
|
||||||
|
std::fs::write(out_path, &out)?;
|
||||||
|
|
||||||
|
// Make executable on Unix.
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
std::fs::set_permissions(out_path, std::fs::Permissions::from_mode(0o755))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"Wrote {out_path} ({:.1} MB native executable, {:.0} KB precompiled WASM)",
|
||||||
|
out.len() as f64 / 1_048_576.0,
|
||||||
|
precompiled.len() as f64 / 1024.0
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// `wafer run program.wasm`
|
/// `wafer run program.wasm`
|
||||||
fn cmd_run(file: &str) -> anyhow::Result<()> {
|
fn cmd_run(file: &str) -> anyhow::Result<()> {
|
||||||
let output = run_wasm_file(file)?;
|
let output = run_wasm_file(file)?;
|
||||||
@@ -155,7 +276,7 @@ fn cmd_eval_or_repl(file: Option<&str>) -> anyhow::Result<()> {
|
|||||||
if !stdin_is_tty() {
|
if !stdin_is_tty() {
|
||||||
// Non-interactive: read all of stdin and evaluate
|
// Non-interactive: read all of stdin and evaluate
|
||||||
let mut input = String::new();
|
let mut input = String::new();
|
||||||
std::io::Read::read_to_string(&mut std::io::stdin(), &mut input)?;
|
Read::read_to_string(&mut std::io::stdin(), &mut input)?;
|
||||||
for line in input.lines() {
|
for line in input.lines() {
|
||||||
match vm.evaluate(line) {
|
match vm.evaluate(line) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ fn collect_external_calls(ops: &[IrOp], ir_ids: &HashSet<WordId>, host_ids: &mut
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Serialize export metadata to JSON (hand-rolled, no serde dependency).
|
/// Serialize export metadata to JSON (hand-rolled, no serde dependency).
|
||||||
fn serialize_metadata(m: &ExportMetadata) -> String {
|
pub fn serialize_metadata(m: &ExportMetadata) -> String {
|
||||||
let mut s = String::from("{\n");
|
let mut s = String::from("{\n");
|
||||||
let _ = writeln!(s, " \"version\": {},", m.version);
|
let _ = writeln!(s, " \"version\": {},", m.version);
|
||||||
match m.entry_table_index {
|
match m.entry_table_index {
|
||||||
|
|||||||
+45
-28
@@ -10,14 +10,21 @@ use wasmtime::{
|
|||||||
TableType, Val, ValType,
|
TableType, Val, ValType,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::export::deserialize_metadata;
|
use crate::export::{ExportMetadata, deserialize_metadata};
|
||||||
use crate::memory::{CELL_SIZE, DATA_STACK_TOP, SYSVAR_BASE_VAR};
|
use crate::memory::{CELL_SIZE, DATA_STACK_TOP, SYSVAR_BASE_VAR};
|
||||||
|
|
||||||
/// Host state for the runner (currently unused by wasmtime `Store` but
|
/// Host state for the runner (currently unused by wasmtime `Store` but
|
||||||
/// required as the generic parameter).
|
/// required as the generic parameter).
|
||||||
struct RunnerHost {}
|
struct RunnerHost {}
|
||||||
|
|
||||||
/// Execute a pre-compiled `.wasm` module and return its output.
|
/// Create a wasmtime engine with the standard WAFER configuration.
|
||||||
|
fn make_engine() -> anyhow::Result<Engine> {
|
||||||
|
let mut config = wasmtime::Config::new();
|
||||||
|
config.cranelift_nan_canonicalization(false);
|
||||||
|
Engine::new(&config)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Execute a pre-compiled `.wasm` module from a file path.
|
||||||
pub fn run_wasm_file(path: &str) -> anyhow::Result<String> {
|
pub fn run_wasm_file(path: &str) -> anyhow::Result<String> {
|
||||||
let wasm_bytes = std::fs::read(path)?;
|
let wasm_bytes = std::fs::read(path)?;
|
||||||
run_wasm_bytes(&wasm_bytes)
|
run_wasm_bytes(&wasm_bytes)
|
||||||
@@ -25,20 +32,40 @@ pub fn run_wasm_file(path: &str) -> anyhow::Result<String> {
|
|||||||
|
|
||||||
/// Execute WASM bytes directly (used by tests and the CLI).
|
/// Execute WASM bytes directly (used by tests and the CLI).
|
||||||
pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
||||||
// Parse the "wafer" custom section for metadata.
|
|
||||||
let metadata_json = extract_custom_section(wasm_bytes, "wafer")?;
|
let metadata_json = extract_custom_section(wasm_bytes, "wafer")?;
|
||||||
let metadata = deserialize_metadata(&metadata_json)?;
|
let metadata = deserialize_metadata(&metadata_json)?;
|
||||||
|
let engine = make_engine()?;
|
||||||
|
let module = Module::new(&engine, wasm_bytes)?;
|
||||||
|
run_module(&engine, module, &metadata)
|
||||||
|
}
|
||||||
|
|
||||||
// Set up wasmtime runtime.
|
/// Execute an AOT-precompiled module with separate metadata.
|
||||||
let mut config = wasmtime::Config::new();
|
///
|
||||||
config.cranelift_nan_canonicalization(false);
|
/// The `precompiled` bytes must have been produced by
|
||||||
let engine = Engine::new(&config)?;
|
/// `Engine::precompile_module` with a compatible wasmtime version
|
||||||
|
/// and platform.
|
||||||
|
#[allow(unsafe_code)]
|
||||||
|
pub fn run_precompiled_bytes(precompiled: &[u8], metadata_json: &str) -> anyhow::Result<String> {
|
||||||
|
let metadata = deserialize_metadata(metadata_json)?;
|
||||||
|
let engine = make_engine()?;
|
||||||
|
// SAFETY: precompiled bytes are produced by wafer build --native using
|
||||||
|
// the same wasmtime version. The caller guarantees compatibility.
|
||||||
|
let module = unsafe { Module::deserialize(&engine, precompiled)? };
|
||||||
|
run_module(&engine, module, &metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared runner logic: given a compiled module and metadata, set up the
|
||||||
|
/// six WASM imports, register host functions, call `_start`, return output.
|
||||||
|
fn run_module(
|
||||||
|
engine: &Engine,
|
||||||
|
module: Module,
|
||||||
|
metadata: &ExportMetadata,
|
||||||
|
) -> anyhow::Result<String> {
|
||||||
let output = Arc::new(Mutex::new(String::new()));
|
let output = Arc::new(Mutex::new(String::new()));
|
||||||
let mut store = Store::new(&engine, RunnerHost {});
|
let mut store = Store::new(engine, RunnerHost {});
|
||||||
|
|
||||||
// Create the 6 imports the module expects.
|
// Create the 6 imports the module expects.
|
||||||
let memory_pages = metadata.memory_size.div_ceil(65536).max(16); // at least 16 pages like the VM
|
let memory_pages = metadata.memory_size.div_ceil(65536).max(16);
|
||||||
let memory = Memory::new(&mut store, MemoryType::new(memory_pages, None))?;
|
let memory = Memory::new(&mut store, MemoryType::new(memory_pages, None))?;
|
||||||
|
|
||||||
let dsp = Global::new(
|
let dsp = Global::new(
|
||||||
@@ -57,24 +84,15 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
|||||||
Val::I32(metadata.fsp_init as i32),
|
Val::I32(metadata.fsp_init as i32),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
// Determine table size from the module's import.
|
// Determine table size from the module's imports.
|
||||||
let parsed = wasmparser::Parser::new(0).parse_all(wasm_bytes);
|
let table_min: u32 = module
|
||||||
let mut table_min: u64 = 256;
|
.imports()
|
||||||
for payload in parsed {
|
.find(|i| i.name() == "table")
|
||||||
if let wasmparser::Payload::ImportSection(reader) = payload? {
|
.and_then(|i| i.ty().table().map(|t| t.minimum() as u32))
|
||||||
for import in reader {
|
.unwrap_or(256);
|
||||||
let import = import?;
|
|
||||||
if import.name == "table"
|
|
||||||
&& let wasmparser::TypeRef::Table(t) = import.ty
|
|
||||||
{
|
|
||||||
table_min = t.initial;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let table = Table::new(
|
let table = Table::new(
|
||||||
&mut store,
|
&mut store,
|
||||||
TableType::new(wasmtime::RefType::FUNCREF, table_min as u32, None),
|
TableType::new(wasmtime::RefType::FUNCREF, table_min, None),
|
||||||
Ref::Func(None),
|
Ref::Func(None),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
@@ -82,7 +100,7 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
|||||||
let out_ref = Arc::clone(&output);
|
let out_ref = Arc::clone(&output);
|
||||||
let emit_func = Func::new(
|
let emit_func = Func::new(
|
||||||
&mut store,
|
&mut store,
|
||||||
FuncType::new(&engine, [ValType::I32], []),
|
FuncType::new(engine, [ValType::I32], []),
|
||||||
move |_caller, params, _results| {
|
move |_caller, params, _results| {
|
||||||
let code = params[0].unwrap_i32();
|
let code = params[0].unwrap_i32();
|
||||||
if let Some(ch) = char::from_u32(code as u32) {
|
if let Some(ch) = char::from_u32(code as u32) {
|
||||||
@@ -93,7 +111,6 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Instantiate the module.
|
// Instantiate the module.
|
||||||
let module = Module::new(&engine, wasm_bytes)?;
|
|
||||||
let instance = wasmtime::Instance::new(
|
let instance = wasmtime::Instance::new(
|
||||||
&mut store,
|
&mut store,
|
||||||
&module,
|
&module,
|
||||||
@@ -109,7 +126,7 @@ pub fn run_wasm_bytes(wasm_bytes: &[u8]) -> anyhow::Result<String> {
|
|||||||
|
|
||||||
// Register host functions in the table at the metadata-specified indices.
|
// Register host functions in the table at the metadata-specified indices.
|
||||||
for (idx, name) in &metadata.host_functions {
|
for (idx, name) in &metadata.host_functions {
|
||||||
let func = create_host_func(&mut store, &engine, memory, dsp, &output, name);
|
let func = create_host_func(&mut store, engine, memory, dsp, &output, name);
|
||||||
table.set(&mut store, *idx as u64, Ref::Func(Some(func)))?;
|
table.set(&mut store, *idx as u64, Ref::Func(Some(func)))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user