From 0f6285ddfc99766c0b1ebabd3852948330d6aac6 Mon Sep 17 00:00:00 2001
From: Oleksandr Kozachuk <ddeus.gh@mailnull.com>
Date: Tue, 7 Apr 2026 13:36:26 +0200
Subject: [PATCH] Fix optimizer bug: TailCall inside If not converted on inline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the tail-call pass converted a Call to TailCall inside an If branch,
and the inliner subsequently inlined that word, the TailCall was not
converted back to Call in nested control-flow bodies. The TailCall codegen
emits a Return instruction, which would exit the *caller* instead of just
the inlined callee — silently corrupting the return stack.

Root cause: the inliner only converted top-level TailCalls in the body
(line-by-line iteration), missing TailCalls nested inside If/DoLoop/Begin
structures.

Fix: add detailcall() that recursively walks the entire IR tree and
converts all TailCall ops back to Call before inlining.

This unblocks defining complex Forth words (like SM/REM, FM/MOD) that
use DABS → DNEGATE → D+ chains with return-stack operations inside
conditional branches.

426 tests pass (including new regression test).
---
 crates/core/src/optimizer.rs | 57 ++++++++++++++++++++++++++++++++----
 crates/core/src/outer.rs     | 12 ++++++++
 2 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/crates/core/src/optimizer.rs b/crates/core/src/optimizer.rs
index c7dc8c0..5957dc9 100644
--- a/crates/core/src/optimizer.rs
+++ b/crates/core/src/optimizer.rs
@@ -505,13 +505,10 @@ fn inline(ops: Vec<IrOp>, bodies: &HashMap<WordId, Vec<IrOp>>, max_size: usize)
                     && body.len() <= max_size
                     && !contains_call_to(body, *id)
                 {
-                    // Inline the body, converting TailCall back to Call
-                    // (tail position in the callee is not tail position in the caller)
+                    // Inline the body, recursively converting TailCall back to Call
+                    // (tail position in the callee is not tail position in the caller).
                     for inlined_op in body {
-                        match inlined_op {
-                            IrOp::TailCall(tid) => out.push(IrOp::Call(*tid)),
-                            other => out.push(other.clone()),
-                        }
+                        out.push(detailcall(inlined_op.clone()));
                     }
                     continue;
                 }
@@ -527,6 +524,54 @@ fn inline(ops: Vec<IrOp>, bodies: &HashMap<WordId, Vec<IrOp>>, max_size: usize)
     out
 }
 
+/// Recursively convert all `TailCall` ops to `Call` in an IR tree.
+///
+/// When inlining a callee, its tail-call positions are no longer tail positions
+/// in the caller. The `TailCall` codegen emits `Return` after the call, which
+/// would prematurely exit the caller's function. This must recurse into
+/// control-flow bodies (If, loops) where `convert_tail_call` may have placed
+/// `TailCall` ops.
+fn detailcall(op: IrOp) -> IrOp {
+    match op {
+        IrOp::TailCall(id) => IrOp::Call(id),
+        IrOp::If {
+            then_body,
+            else_body,
+        } => IrOp::If {
+            then_body: then_body.into_iter().map(detailcall).collect(),
+            else_body: else_body.map(|eb| eb.into_iter().map(detailcall).collect()),
+        },
+        IrOp::DoLoop { body, is_plus_loop } => IrOp::DoLoop {
+            body: body.into_iter().map(detailcall).collect(),
+            is_plus_loop,
+        },
+        IrOp::BeginUntil { body } => IrOp::BeginUntil {
+            body: body.into_iter().map(detailcall).collect(),
+        },
+        IrOp::BeginAgain { body } => IrOp::BeginAgain {
+            body: body.into_iter().map(detailcall).collect(),
+        },
+        IrOp::BeginWhileRepeat { test, body } => IrOp::BeginWhileRepeat {
+            test: test.into_iter().map(detailcall).collect(),
+            body: body.into_iter().map(detailcall).collect(),
+        },
+        IrOp::BeginDoubleWhileRepeat {
+            outer_test,
+            inner_test,
+            body,
+            after_repeat,
+            else_body,
+        } => IrOp::BeginDoubleWhileRepeat {
+            outer_test: outer_test.into_iter().map(detailcall).collect(),
+            inner_test: inner_test.into_iter().map(detailcall).collect(),
+            body: body.into_iter().map(detailcall).collect(),
+            after_repeat: after_repeat.into_iter().map(detailcall).collect(),
+            else_body: else_body.map(|eb| eb.into_iter().map(detailcall).collect()),
+        },
+        other => other,
+    }
+}
+
 /// Check if an IR body contains a direct call to the given word (recursion guard).
 fn contains_call_to(ops: &[IrOp], target: WordId) -> bool {
     for op in ops {
diff --git a/crates/core/src/outer.rs b/crates/core/src/outer.rs
index 1e5d38d..4dddd3a 100644
--- a/crates/core/src/outer.rs
+++ b/crates/core/src/outer.rs
@@ -7944,6 +7944,18 @@ mod tests {
         assert_eq!(eval_stack("-1 0 10 WITHIN"), vec![0]);
     }
 
+    #[test]
+    fn test_inline_tailcall_rstack_interaction() {
+        // Regression: inlining a word that had a TailCall inside an If branch
+        // caused the TailCall's Return to exit the *caller*, corrupting the
+        // return stack. The fix: detailcall() recursively converts TailCall
+        // back to Call inside all nested control-flow bodies when inlining.
+        assert_eq!(
+            eval_stack(": T 42 >R 99 >R -7 -1 DABS R> R> ; T"),
+            vec![42, 99, 0, 7]
+        );
+    }
+
     #[test]
     fn test_do_loop_with_i_and_step() {
         // +LOOP with step of 2