diff --git a/README.md b/README.md index c204619..50d3c50 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Table of Contents * [Details](#details) * [Options](#options) * [HTCondor jobs](#htcondor-jobs) - * [Caveats](#caveats) + * [Interactive commands and signals](#interactive-commands-and-signals) * [bind_condor.sh](#bind_condorsh) * [Usage](#usage-1) * [Setting up bindings](#setting-up-bindings) @@ -171,14 +171,13 @@ or placed in a file `~/.callhostrc` (automatically detected and sourced by `call ``` * Using the `ENV()` function in the JDL file may not function as intended, since it will be evaluated on the host node, rather than inside the container with your environment set up. -### Caveats +### Interactive commands and signals -* Commands that require tty input (such as `nano` or `emacs -nw`) will not work with `call_host`. -* Occasionally, if a command fails (especially when calling multiple commands separated by semicolons), the pipe will break and the terminal will appear to hang. The message "Interrupted system call" may be shown. - It is necessary to exit and reenter the container (in order to create a new pipe) if this occurs. - To prevent this, chain multiple commands using logical operators (`&&` or `||`), or surround all the commands in `()` (thereby running them in a subshell). -* Stopping a command in progress with ctrl+C is disabled (to avoid breaking the pipe as described in the previous item). -* Suspending a command with ctrl+Z is not supported and may break the session. +* Commands that require tty input (such as `nano` or `emacs -nw`) now work with `call_host`, provided you are using `call_host` interactively (i.e. your terminal is a real tty). When both the input and output of `call_host` are a terminal, the host command is run under a pseudo-terminal (pty) so that full-screen and interactive programs behave normally. + * This requires `python3` on the host node (present on essentially all interactive nodes). If `python3` is unavailable, `script` is used as a fallback (interactive programs still work, but ctrl+C is delivered via the slower signal path described below); if neither is available, the command runs without a pty (as before). +* Your stdin is forwarded to the host command, so piping or redirecting input works, e.g. `echo data | call_host some_command` or `call_host some_command < input.txt`. +* Pressing ctrl+C now interrupts the running host command (and returns you to your prompt) instead of being ignored, in both interactive and non-interactive use. The container session is left intact. +* Pressing ctrl+Z does not suspend the host command (job-control suspension cannot be supported across the container boundary), but it no longer breaks the session or leaves a stray process on the host: the keystroke is simply ignored and the command keeps running. Use ctrl+C if you want to stop it. ## `bind_condor.sh` diff --git a/call_host.sh b/call_host.sh index c79b3f5..b813c16 100755 --- a/call_host.sh +++ b/call_host.sh @@ -204,20 +204,199 @@ export_func call_host_plugin_01 # concept based on https://stackoverflow.com/questions/32163955/how-to-run-shell-script-on-host-from-docker-container +# python helper that runs a command on the host under a pseudo-terminal (pty). +# This makes isatty() true for the host command (so full-screen tty programs like +# nano or emacs -nw work) and, crucially, routes the container terminal's control +# characters (e.g. ctrl+c -> 0x03) through the pty line discipline, which turns +# them into real signals (SIGINT) for the host command. python3 is used because it +# is present on essentially all interactive nodes and, unlike `script`, its pty +# correctly converts control bytes into signals. Stored as a string and executed +# via `python3 -c` so no temporary files need to be managed. +# shellcheck disable=SC2016 +read -r -d '' CALL_HOST_PTY_PY <<'CALL_HOST_PTY_PY_EOF' +import os, sys, pty, select, termios, struct, fcntl, signal +cmd = sys.argv[1] if len(sys.argv) > 1 else "" +rows = int(sys.argv[2]) if len(sys.argv) > 2 else 24 +cols = int(sys.argv[3]) if len(sys.argv) > 3 else 80 +# the bridge itself must survive a stray INT/QUIT; only the child should receive them +signal.signal(signal.SIGINT, signal.SIG_IGN) +signal.signal(signal.SIGQUIT, signal.SIG_IGN) +pid, fd = pty.fork() +if pid == 0: + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGQUIT, signal.SIG_DFL) + os.execv("/bin/bash", ["/bin/bash", "-c", cmd]) + os._exit(127) +try: + fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)) +except Exception: + pass +# Disable the pty's suspend character (ctrl+z). We cannot honour job-control +# suspension here -- there is no interactive shell on this side to background the +# job to -- and a stopped host command would hang this bridge and be orphaned when +# the container exits. Disabling VSUSP makes ctrl+z a harmless passthrough byte. +try: + attr = termios.tcgetattr(fd) + attr[6][termios.VSUSP] = b'\x00' + termios.tcsetattr(fd, termios.TCSANOW, attr) +except Exception: + pass +fds = [fd, 0] +while True: + try: + r, _, _ = select.select(fds, [], []) + except (OSError, select.error): + continue + if fd in r: + try: data = os.read(fd, 65536) + except OSError: data = b"" + if not data: + break + try: os.write(1, data) + except OSError: break + if 0 in r: + try: data = os.read(0, 65536) + except OSError: data = b"" + if not data: + fds = [fd] # stdin closed: stop polling it to avoid a busy loop + continue + try: os.write(fd, data) + except OSError: break +_, status = os.waitpid(pid, 0) +if os.WIFEXITED(status): + sys.exit(os.WEXITSTATUS(status)) +if os.WIFSIGNALED(status): + sys.exit(128 + os.WTERMSIG(status)) +sys.exit(0) +CALL_HOST_PTY_PY_EOF +export CALL_HOST_PTY_PY +# tty mode is available if the host can allocate a pty for the command, i.e. it has +# either python3 (preferred) or script. Exported so the in-container call_host knows +# whether to request a pty. +if command -v python3 >/dev/null 2>&1 || command -v script >/dev/null 2>&1; then + export APPTAINERENV_CALL_HOST_PTY_AVAIL=1 +else + export APPTAINERENV_CALL_HOST_PTY_AVAIL=0 +fi + +# A signal that is ignored (SIG_IGN) at the moment a program is exec'd cannot be +# reset by that program (POSIX). The listener must ignore SIGINT/SIGQUIT to keep +# its loop alive, and bash additionally forces SIG_IGN for SIGINT/SIGQUIT on any +# command started asynchronously (with &). Either way the host command would +# inherit "ignore" and could not be interrupted by a forwarded ctrl+c. To work +# around this we launch the (non-pty) host command through a small wrapper that +# resets those signals to their default disposition before running it. Prefer GNU +# coreutils' "env --default-signal" (fast, no extra process); fall back to a small +# python launcher (resets the handlers, then exec's, so no python stays resident). +# CALL_HOST_SIGDFL_MODE selects which mechanism run_on_host uses. +if env --default-signal=INT,QUIT true >/dev/null 2>&1; then + CALL_HOST_SIGDFL_MODE="env" +elif command -v python3 >/dev/null 2>&1; then + CALL_HOST_SIGDFL_MODE="python" +else + CALL_HOST_SIGDFL_MODE="none" +fi +# kept free of newlines so it can be passed as a single argument +CALL_HOST_SIGDFL_PY='import os,sys,signal; signal.signal(signal.SIGINT,signal.SIG_DFL); signal.signal(signal.SIGQUIT,signal.SIG_DFL); os.execvp(sys.argv[1],sys.argv[1:])' +export CALL_HOST_SIGDFL_MODE CALL_HOST_SIGDFL_PY + +# run a single command on the host, wiring up stdin, stdout/stderr, and signals. +# args: command want_tty rows cols inpipe contpipe sigpipe +# returns the command's exit status. +run_on_host(){ + local cmd="$1" want_tty="$2" rows="$3" cols="$4" inp="$5" cp="$6" sp="$7" + local child sigfwd rc=0 + # launch in a new session (setsid) so the command gets its own process group; + # this lets us deliver signals to the command (and its children) without + # affecting the listener itself. + if [ "$want_tty" = "1" ] && command -v python3 >/dev/null 2>&1; then + setsid python3 -c "$CALL_HOST_PTY_PY" "$cmd" "$rows" "$cols" <"$inp" >"$cp" 2>&1 & + elif [ "$want_tty" = "1" ] && command -v script >/dev/null 2>&1; then + # fallback: `script` provides a pty (tty programs work) but does not convert + # control bytes to signals, so ctrl+c relies on the signal pipe below. + setsid script -qfec "stty rows ${rows:-24} cols ${cols:-80} 2>/dev/null; $cmd" /dev/null <"$inp" >"$cp" 2>&1 & + else + # Non-pty command. Reset the inherited "ignore" disposition for SIGINT/SIGQUIT + # (see CALL_HOST_SIGDFL_MODE above) so a forwarded ctrl+c can interrupt it. + case "$CALL_HOST_SIGDFL_MODE" in + env) + setsid env --default-signal=INT,QUIT bash -c "$cmd" <"$inp" >"$cp" 2>&1 & + ;; + python) + setsid python3 -c "$CALL_HOST_SIGDFL_PY" bash -c "$cmd" <"$inp" >"$cp" 2>&1 & + ;; + *) + # no reset mechanism available: ctrl+c forwarding may not interrupt, + # but the command still runs correctly + setsid bash -c "$cmd" <"$inp" >"$cp" 2>&1 & + ;; + esac + fi + child=$! + # The command runs in its own session/process group (setsid above), which is + # necessary for targeted signal delivery but means it is NOT in the listener's + # process group -- so the container-exit cleanup (which kills the listener's + # group) would not reach a command that is still running (e.g. the user exits + # the container mid-command). Guard against that: if this runner is terminated + # (it IS in the listener's group), kill the command's whole process group first. + trap 'kill -TERM -- -"$child" 2>/dev/null; kill -KILL -- -"$child" 2>/dev/null; exit 143' TERM HUP + # forward signals received on the signal pipe to the command's process group. + # This is how a ctrl+c from a non-tty container session interrupts the host + # command. The loop exits once the command is gone. + ( + trap "" SIGINT SIGQUIT + while kill -0 "$child" 2>/dev/null; do + IFS= read -r sig <"$sp" || break + [ -n "$sig" ] && kill -"$sig" -- -"$child" 2>/dev/null + done + ) & + sigfwd=$! + wait "$child"; rc=$? + trap - TERM HUP + # tear down the signal forwarder (and unblock its read on the pipe) + kill "$sigfwd" 2>/dev/null + echo "" > "$sp" 2>/dev/null & + wait "$sigfwd" 2>/dev/null + return "$rc" +} +export_func run_on_host + # execute command sent to host pipe; send output to container pipe; store exit code +# args: hostpipe contpipe exitpipe inpipe sigpipe listenhost(){ + local hp="$1" cp="$2" ep="$3" inp="$4" sp="$5" + # make the listener immune to signals that could otherwise break the loop and + # leave the terminal hung ("Interrupted system call"); the command itself is + # signalled explicitly via run_on_host instead. + trap "" SIGINT SIGQUIT SIGPIPE # stop when host pipe is removed - while [ -e "$1" ]; do - # "|| true" is necessary to stop "Interrupted system call" when running commands like 'command1; command2; command3' - # now replaced with assignment of exit code to local variable (which also returns true) - # using { bash -c ... } >& is less fragile than eval - tmpexit=0 - cmd="$(cat "$1")" - call_host_debug_print "cmd: $cmd" - { - bash -c "$cmd" || tmpexit=$? - } >& "$2" - echo "$tmpexit" > "$3" + while [ -e "$hp" ]; do + local payload header cmd want_tty rows cols tmpexit=0 tok + # read one request; ignore spurious empty reads (e.g. writer reopened pipe) + payload="$(cat "$hp")" || continue + [ -z "$payload" ] && continue + # the first line is a header describing the request; the rest is the command + header="$(printf '%s\n' "$payload" | head -n 1)" + want_tty=0; rows=24; cols=80 + case "$header" in + CALL_HOST_HDR*) + cmd="$(printf '%s\n' "$payload" | tail -n +2)" + for tok in $header; do + case "$tok" in + TTY=*) want_tty="${tok#TTY=}" ;; + ROWS=*) rows="${tok#ROWS=}" ;; + COLS=*) cols="${tok#COLS=}" ;; + esac + done + ;; + *) + # backward compatibility: no header, whole payload is the command + cmd="$payload" + ;; + esac + call_host_debug_print "cmd: $cmd (tty=$want_tty)" + run_on_host "$cmd" "$want_tty" "$rows" "$cols" "$inp" "$cp" "$sp" || tmpexit=$? + echo "$tmpexit" > "$ep" done } export_func listenhost @@ -236,11 +415,23 @@ startpipe(){ HOSTPIPE=$(makepipe HOST) CONTPIPE=$(makepipe CONT) EXITPIPE=$(makepipe EXIT) + # INPIPE carries the container's stdin to the host command; + # SIGPIPE carries signals (e.g. INT from ctrl+c) to the host command. + INPIPE=$(makepipe IN) + SIGPIPE=$(makepipe SIG) # export pipes to apptainer - echo "export APPTAINERENV_HOSTPIPE=$HOSTPIPE; export APPTAINERENV_CONTPIPE=$CONTPIPE; export APPTAINERENV_EXITPIPE=$EXITPIPE" + echo "export APPTAINERENV_HOSTPIPE=$HOSTPIPE; export APPTAINERENV_CONTPIPE=$CONTPIPE; export APPTAINERENV_EXITPIPE=$EXITPIPE; export APPTAINERENV_INPIPE=$INPIPE; export APPTAINERENV_SIGPIPE=$SIGPIPE" } export_func startpipe +make_listener_script(){ + get_function call_host_debug_print + get_function run_on_host + get_function listenhost + printf '\nlistenhost "$@"\n' +} +export_func make_listener_script + # sends function to host, then listens for output, and provides exit code from function call_host(){ if [ "$CALL_HOST_STATUS" != "enable" ]; then @@ -251,9 +442,6 @@ call_host(){ return 1 fi - # disable ctrl+c to prevent "Interrupted system call" - trap "" SIGINT - # determine caller function name in a portable way CURFN="$(current_funcname)" if [ "$CURFN" = "call_host" ] || [ -z "$CURFN" ]; then @@ -266,8 +454,98 @@ call_host(){ # todo: evolve into full plugin system that executes detected functions/executables in order (like config.d) EXTRA="$(call_host_plugin_01)" - echo "cd $PWD; $EXTRA $FUNCTMP $*" > "$HOSTPIPE" - cat < "$CONTPIPE" + # decide whether to request a pty on the host: only useful (and only correct) + # when both our stdin and stdout are terminals, i.e. a genuinely interactive + # call. In that case the host command gets a real tty (so editors etc. work) + # and the pty converts our ctrl+c into a SIGINT for the host command. + CALL_HOST_TTY=0 + CALL_HOST_ROWS=24 + CALL_HOST_COLS=80 + if [ -t 0 ] && [ -t 1 ] && [ "$APPTAINERENV_CALL_HOST_PTY_AVAIL" = "1" ]; then + CALL_HOST_TTY=1 + CALL_HOST_SIZE="$(stty size 2>/dev/null)" + if [ -n "$CALL_HOST_SIZE" ]; then + CALL_HOST_ROWS="${CALL_HOST_SIZE%% *}" + CALL_HOST_COLS="${CALL_HOST_SIZE##* }" + fi + fi + + # Build the request: a header line (parsed by the host listener) followed by + # the actual command. The header is ignored by older host-side scripts. + { + printf 'CALL_HOST_HDR TTY=%s ROWS=%s COLS=%s\n' "$CALL_HOST_TTY" "$CALL_HOST_ROWS" "$CALL_HOST_COLS" + printf 'cd %s; %s %s %s\n' "$PWD" "$EXTRA" "$FUNCTMP" "$*" + } > "$HOSTPIPE" + + CALL_HOST_INCAT= + CALL_HOST_OUTCAT= + if [ "$CALL_HOST_TTY" = "1" ]; then + # Interactive path. Put our terminal in raw mode so every keystroke + # (including ctrl+c as a raw 0x03 byte) passes straight through to the host + # pty unmodified; the host pty's line discipline then turns ctrl+c into a + # real SIGINT for the host command. The output reader runs in the background + # while we forward stdin in the foreground, so the two never compete for the + # terminal. When the command ends the host closes CONTPIPE, the background + # reader exits, and we stop forwarding input. + CALL_HOST_STTY_SAVED="$(stty -g 2>/dev/null)" + stty raw -echo 2>/dev/null + # Run both helper readers inside one outer subshell. Two reasons: + # 1. Job control does not apply to commands started inside a subshell, so the + # interactive shell never prints "[1] " / "Terminated" notices (which + # would corrupt the display of full-screen programs like nano). + # 2. The subshell inherits our real terminal (fd 0/1), and because it is not + # interactive its background readers are not sent SIGTTIN when they read + # the terminal (which would otherwise stop the stdin forwarder). + # The subshell returns when the host closes CONTPIPE, i.e. when the command + # finishes; the stdin forwarder is then stopped. + ( + cat < "$CONTPIPE" & + CALL_HOST_OUT=$! + ( cat > "$INPIPE" 2>/dev/null ) <&0 & + CALL_HOST_IN=$! + wait "$CALL_HOST_OUT" 2>/dev/null + kill "$CALL_HOST_IN" 2>/dev/null + ) + [ -n "$CALL_HOST_STTY_SAVED" ] && stty "$CALL_HOST_STTY_SAVED" 2>/dev/null + else + # Non-interactive path. Forward our stdin to the host command in the + # background. A background command in a non-interactive shell has its stdin + # silently redirected from /dev/null (POSIX async behavior), so we duplicate + # our real stdin onto fd 3 and read from that explicit descriptor instead. + if [ -n "$INPIPE" ]; then + exec 3<&0 + ( cat <&3 > "$INPIPE" 2>/dev/null ) & + CALL_HOST_INCAT=$! + exec 3<&- + fi + # Catch ctrl+c locally and forward it as a SIGINT to the host command via + # SIGPIPE, instead of letting it tear down this function (which previously + # could break the pipe / hang the session). The output read is backgrounded + # and waited on so the trap is delivered promptly. + if [ -n "$SIGPIPE" ]; then + trap 'printf "INT\n" > "$SIGPIPE" 2>/dev/null &' SIGINT + else + # no signal pipe: fall back to old behavior of ignoring ctrl+c + trap "" SIGINT + fi + cat < "$CONTPIPE" & + CALL_HOST_OUTCAT=$! + while true; do + wait "$CALL_HOST_OUTCAT"; CALL_HOST_WRC=$? + # a return >128 means wait was interrupted by a signal (the trap ran); + # keep waiting for the host command to actually finish and close CONTPIPE. + [ "$CALL_HOST_WRC" -le 128 ] && break + done + trap - SIGINT + fi + + # stop forwarding stdin (wait reaps the process quietly, avoiding a stray + # "Terminated" job-control message) + if [ -n "$CALL_HOST_INCAT" ]; then + kill "$CALL_HOST_INCAT" 2>/dev/null + wait "$CALL_HOST_INCAT" 2>/dev/null + fi + return "$(cat < "$EXITPIPE")" } export_func call_host @@ -309,7 +587,18 @@ apptainer(){ # i.e. don't create more pipes/listeners for nested containers if [ -z "$APPTAINER_CONTAINER" ]; then eval "$(startpipe)" - listenhost "$APPTAINERENV_HOSTPIPE" "$APPTAINERENV_CONTPIPE" "$APPTAINERENV_EXITPIPE" & + # Start the listener in its own new session/process group (setsid) and + # detached from this shell's stdio. Two reasons: + # 1. The listener communicates only through the explicit pipes, so it must + # never hold the user's terminal/stdout open (otherwise a pipeline like + # "... | tail" would never see EOF after the container exits). + # 2. Its own process group lets us signal the whole subtree at cleanup. + # The listener reads each request with "$(cat "$hp")", which forks a + # subshell that forks cat; killing only the listener (or its direct + # children) would leave that cat grandchild blocked on the fifo and + # reparented to init. Killing the process group reaps all of them. + CALL_HOST_LISTENER_SCRIPT="$(make_listener_script)" + setsid bash -c "$CALL_HOST_LISTENER_SCRIPT" _ "$APPTAINERENV_HOSTPIPE" "$APPTAINERENV_CONTPIPE" "$APPTAINERENV_EXITPIPE" "$APPTAINERENV_INPIPE" "$APPTAINERENV_SIGPIPE" /dev/null 2>&1 & LISTENER=$! fi # actually run apptainer @@ -317,8 +606,15 @@ apptainer(){ # avoid dangling cat process after exiting container # (again, only on host) if [ -z "$APPTAINER_CONTAINER" ]; then - pkill -P "$LISTENER" - rm -f "$APPTAINERENV_HOSTPIPE" "$APPTAINERENV_CONTPIPE" "$APPTAINERENV_EXITPIPE" + # Tear down the listener and every descendant (the blocked "cat" reader, + # signal forwarders, detached command runners) by signalling its whole + # process group. setsid above made LISTENER the group leader, so its pgid + # equals its pid, and a negative pid signals the entire group. This is + # both necessary (the "cat" reader is a grandchild that a plain kill would + # orphan) and sufficient, so we deliberately avoid pkill -P / kill on the + # bare pid, which could hit an unrelated process if the pid were reused. + kill -- -"$LISTENER" 2>/dev/null + rm -f "$APPTAINERENV_HOSTPIPE" "$APPTAINERENV_CONTPIPE" "$APPTAINERENV_EXITPIPE" "$APPTAINERENV_INPIPE" "$APPTAINERENV_SIGPIPE" fi ) fi