a potential solution that may fix vt processing (#13961)

# Description

This PR is related #11950 and serves as another potential fix alongside
rolling it back with https://github.com/nushell/nushell/pull/13959. The
idea here is to try and properly setup the input and output console
modes. I searched through a log of GitHub code to come up with this,
including deno, wezterm, conpty, among others. It seems to work but it
would be great if someone else would be able to test. I added comments
from the consoleapi.h from windows to know what the other flags are in
case we need to make other changes.

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

1 files changed, 53 insertions, 19 deletions

diff --git a/crates/nu-utils/src/utils.rs b/crates/nu-utils/src/utils.rs
index ee575b564..ea5df308b 100644
--- a/crates/nu-utils/src/utils.rs
+++ b/crates/nu-utils/src/utils.rs
@@ -1,32 +1,66 @@
+#[cfg(windows)]
+use crossterm_winapi::{ConsoleMode, Handle};
 use lscolors::LsColors;
 use std::io::{Result, Write};
 
 pub fn enable_vt_processing() -> Result<()> {
     #[cfg(windows)]
     {
-        use crossterm_winapi::{ConsoleMode, Handle};
+        let console_out_mode = ConsoleMode::from(Handle::current_out_handle()?);
+        let old_out_mode = console_out_mode.mode()?;
+        let console_in_mode = ConsoleMode::from(Handle::current_in_handle()?);
+        let old_in_mode = console_in_mode.mode()?;
 
-        pub const ENABLE_PROCESSED_OUTPUT: u32 = 0x0001;
-        pub const ENABLE_VIRTUAL_TERMINAL_INPUT: u32 = 0x0200;
-        pub const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
-        let mask = ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+        enable_vt_processing_input(console_in_mode, old_in_mode)?;
+        enable_vt_processing_output(console_out_mode, old_out_mode)?;
+    }
+    Ok(())
+}
 
-        let console_mode = ConsoleMode::from(Handle::current_out_handle()?);
-        let old_mode = console_mode.mode()?;
+#[cfg(windows)]
+fn enable_vt_processing_input(console_in_mode: ConsoleMode, mode: u32) -> Result<()> {
+    //
+    // Input Mode flags:
+    //
+    // #define ENABLE_PROCESSED_INPUT              0x0001
+    // #define ENABLE_LINE_INPUT                   0x0002
+    // #define ENABLE_ECHO_INPUT                   0x0004
+    // #define ENABLE_WINDOW_INPUT                 0x0008
+    // #define ENABLE_MOUSE_INPUT                  0x0010
+    // #define ENABLE_INSERT_MODE                  0x0020
+    // #define ENABLE_QUICK_EDIT_MODE              0x0040
+    // #define ENABLE_EXTENDED_FLAGS               0x0080
+    // #define ENABLE_AUTO_POSITION                0x0100
+    // #define ENABLE_VIRTUAL_TERMINAL_INPUT       0x0200
 
-        // researching odd ansi behavior in windows terminal repo revealed that
-        // enable_processed_output and enable_virtual_terminal_processing should be used
+    const ENABLE_PROCESSED_INPUT: u32 = 0x0001;
+    const ENABLE_LINE_INPUT: u32 = 0x0002;
+    const ENABLE_ECHO_INPUT: u32 = 0x0004;
+    const ENABLE_VIRTUAL_TERMINAL_INPUT: u32 = 0x0200;
 
-        if old_mode & mask == 0 {
-            console_mode.set_mode(
-                old_mode
-                    | ENABLE_PROCESSED_OUTPUT
-                    | ENABLE_VIRTUAL_TERMINAL_PROCESSING
-                    | ENABLE_VIRTUAL_TERMINAL_INPUT,
-            )?
-        }
-    }
-    Ok(())
+    console_in_mode.set_mode(
+        mode | ENABLE_VIRTUAL_TERMINAL_INPUT
+            & ENABLE_ECHO_INPUT
+            & ENABLE_LINE_INPUT
+            & ENABLE_PROCESSED_INPUT,
+    )
+}
+
+#[cfg(windows)]
+fn enable_vt_processing_output(console_out_mode: ConsoleMode, mode: u32) -> Result<()> {
+    //
+    // Output Mode flags:
+    //
+    // #define ENABLE_PROCESSED_OUTPUT             0x0001
+    // #define ENABLE_WRAP_AT_EOL_OUTPUT           0x0002
+    // #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
+    // #define DISABLE_NEWLINE_AUTO_RETURN         0x0008
+    // #define ENABLE_LVB_GRID_WORLDWIDE           0x0010
+
+    pub const ENABLE_PROCESSED_OUTPUT: u32 = 0x0001;
+    pub const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
+
+    console_out_mode.set_mode(mode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
 }
 
 pub fn stdout_write_all_and_flush<T>(output: T) -> Result<()>